dyncfg v2 (#16702)
* split rrdfunctions streaming and progress * simplified internal inline functions API * split rrdfunctions inflight management * split rrd functions exporters * renames * base dyncfg structure * config pluginsd * intercept dyncfg function calls * loading and saving of dyncfg metadata and data * save metadata and payload to a single file; added code to update the plugins with jobs and saved configs * basic working unit test * added payload to functions execution * removed old dyncfg code that is not needed any more * more cleanup * cleanup sender for functions with payload * dyncfg functions are not exposed as functions * remaining work to avoid indexing the \0 terminating character in dictionary keys * added back old dyncfg plugins.d commands as noop, to allow plugins continue working * working api; working streaming; * updated plugins.d documentation * aclk and http api requests share the same header parsing logic * added source type internal * fixed crashes * added god mode for tests * fixes * fixed messages * save host machine guids to configs * cleaner manipulation of supported commands * the functions event loop for external plugins can now process dyncfg requests * unified internal and external plugins dyncfg API * Netdata serves schema requests from /etc/netdata/schema.d and /var/lib/netdata/conf.d/schema.d * cleanup and various fixes; fixed bug in previous dyncfg implementation on streaming that was sending the paylod in a way that allowed other streaming commands to be multiplexed * internals go to a separate header file * fix duplicate ACLK requests sent by aclk queue mechanism * use fstat instead of stat * working api * plugin actions renamed to create and delete; dyncfg files are removed only from user actions * prevent deadlock by using the react callback * fix for string_strndupz() * better dyncfg unittests * more tests at the unittests * properly detect dyncfg functions * hide config functions from the UI * tree response improvements * send the initial update with payload * determine tty using stdout, not stderr * changes to statuses, cleanup and the code to bring all business logic into interception * do not crash when the status is empty * functions now propagate the source of the requests to plugins * avoid warning about unused functions * in the count at items for attention, do not count the orphan entries * save source into dyncfg * make the list null terminated * fixed invalid comparison * prevent memory leak on duplicated headers; log x-forwarded-for * more unit tests * added dyncfg unittests into the default unittests * more unit tests and fixes * more unit tests and fixes * fix dictionary unittests * config functions require admin access
Costa Tsaousis committed
Jan 11, 2024 at 16:56 UTC
f2b250a1f53af00241522db35f8c85f19ed282e1
121 files changed
+6597
-6064
.codacy.yml
-1
@@ -21,5 +21,4 @@ exclude_paths:
21
- web/server/h2o/libh2o/**
22
- build/**
23
- build_external/**
24
- - libnetdata/dyn_conf/tests/**
24
- packaging/**
CMakeLists.txt
+31
-3
@@ -144,7 +144,7 @@ if(NOT ${DISABLE_HARDENING})
144
endif()
145
146
if(NOT ${CMAKE_C_FLAGS} MATCHES "stack-clash-protection")
147
- check_c_compiler_flag("-fstack-clash-protection", HAVE_STACK_CLASH_FLAG)
147
+ check_c_compiler_flag("-fstack-clash-protection" HAVE_STACK_CLASH_FLAG)
148
if(HAVE_STACK_CLASH_FLAG)
149
set(EXTRA_HARDENING_FLAGS "${EXTRA_HARDENING_FLAGS} -fstack-clash-protection")
150
endif()
@@ -641,8 +641,10 @@ set(LIBNETDATA_FILES
641
libnetdata/http/http_access.h
642
libnetdata/http/http_defs.c
643
libnetdata/http/http_defs.h
644
- libnetdata/dyn_conf/dyn_conf.c
645
- libnetdata/dyn_conf/dyn_conf.h
644
+ libnetdata/http/content_type.c
645
+ libnetdata/http/content_type.h
646
+ libnetdata/config/dyncfg.c
647
+ libnetdata/config/dyncfg.h
648
)
649
650
if(ENABLE_PLUGIN_EBPF)
@@ -760,6 +762,15 @@ set(DAEMON_FILES
762
daemon/pipename.h
763
daemon/unit_test.c
764
daemon/unit_test.h
765
+ daemon/config/dyncfg.c
766
+ daemon/config/dyncfg.h
767
+ daemon/config/dyncfg-files.c
768
+ daemon/config/dyncfg-unittest.c
769
+ daemon/config/dyncfg-inline.c
770
+ daemon/config/dyncfg-echo.c
771
+ daemon/config/dyncfg-internals.h
772
+ daemon/config/dyncfg-intercept.c
773
+ daemon/config/dyncfg-tree.c
774
)
775
776
set(H2O_FILES
@@ -784,6 +795,10 @@ set(API_PLUGIN_FILES
795
web/api/web_api_v1.h
796
web/api/web_api_v2.c
797
web/api/web_api_v2.h
798
+ web/api/http_auth.c
799
+ web/api/http_auth.h
800
+ web/api/http_header.c
801
+ web/api/http_header.h
802
web/api/badges/web_buffer_svg.c
803
web/api/badges/web_buffer_svg.h
804
web/api/exporters/allmetrics.c
@@ -929,6 +944,12 @@ set(RRD_PLUGIN_FILES
944
database/rrdfamily.c
945
database/rrdfunctions.c
946
database/rrdfunctions.h
947
+ database/rrdfunctions-inline.c
948
+ database/rrdfunctions-inline.h
949
+ database/rrdfunctions-progress.c
950
+ database/rrdfunctions-progress.h
951
+ database/rrdfunctions-streaming.c
952
+ database/rrdfunctions-streaming.h
953
database/rrdhost.c
954
database/rrdlabels.c
955
database/rrd.c
@@ -965,6 +986,12 @@ set(RRD_PLUGIN_FILES
986
database/sqlite/dbdata.c
987
database/KolmogorovSmirnovDist.c
988
database/KolmogorovSmirnovDist.h
989
+ database/rrdfunctions-inflight.c
990
+ database/rrdfunctions-inflight.h
991
+ database/rrdfunctions-exporters.c
992
+ database/rrdfunctions-exporters.h
993
+ database/rrdfunctions-internals.h
994
+ database/rrdcollector-internals.h
995
)
996
997
if(ENABLE_DBENGINE)
@@ -1022,6 +1049,7 @@ set(SYSTEMD_JOURNAL_PLUGIN_FILES
1049
collectors/systemd-journal.plugin/systemd-journal-files.c
1050
collectors/systemd-journal.plugin/systemd-journal-fstat.c
1051
collectors/systemd-journal.plugin/systemd-journal-watcher.c
1052
+ collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
1053
)
1054
1055
set(STREAMING_PLUGIN_FILES
aclk/aclk_query.c
+33
-28
@@ -99,30 +99,49 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
99
BUFFER *local_buffer = NULL;
100
size_t size = 0;
101
size_t sent = 0;
102
+ usec_t dt_ut = 0;
103
104
int z_ret;
105
BUFFER *z_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
105
- char *start, *end;
106
107
struct web_client *w = web_client_get_from_cache();
108
+ web_client_set_conn_cloud(w);
109
w->acl = HTTP_ACL_ACLK;
110
+ w->access = HTTP_ACCESS_MEMBER; // the minimum access level for all requests from netdata cloud
111
+ web_client_flags_clear_auth(w);
112
+ web_client_flag_set(w, WEB_CLIENT_FLAG_AUTH_CLOUD);
113
+
114
w->mode = HTTP_REQUEST_MODE_GET;
115
w->timings.tv_in = query->created_tv;
116
117
w->interrupt.callback = aclk_web_client_interrupt_cb;
118
w->interrupt.callback_data = pending_req_list_add(query->msg_id);
119
115
- usec_t t;
120
+ buffer_flush(w->response.data);
121
+ buffer_strcat(w->response.data, query->data.http_api_v2.payload);
122
+
123
+ HTTP_VALIDATION validation = http_request_validate(w);
124
+ if(validation != HTTP_VALIDATION_OK) {
125
+ nd_log(NDLS_ACCESS, NDLP_ERR, "ACLK received request is not valid, code %d", validation);
126
+ retval = 1;
127
+ w->response.code = HTTP_RESP_BAD_REQUEST;
128
+ w->response.code = (short)aclk_http_msg_v2(query_thr->client, query->callback_topic, query->msg_id,
129
+ dt_ut, query->created, w->response.code,
130
+ NULL, 0);
131
+ goto cleanup;
132
+ }
133
+
134
web_client_timeout_checkpoint_set(w, query->timeout);
117
- if(web_client_timeout_checkpoint_and_check(w, &t)) {
118
- nd_log(NDLS_ACCESS, NDLP_ERR, "QUERY CANCELED: QUEUE TIME EXCEEDED %llu ms (LIMIT %d ms)", t / USEC_PER_MS, query->timeout);
135
+ if(web_client_timeout_checkpoint_and_check(w, &dt_ut)) {
136
+ nd_log(NDLS_ACCESS, NDLP_ERR,
137
+ "QUERY CANCELED: QUEUE TIME EXCEEDED %llu ms (LIMIT %d ms)",
138
+ dt_ut / USEC_PER_MS, query->timeout);
139
retval = 1;
140
w->response.code = HTTP_RESP_SERVICE_UNAVAILABLE;
141
aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_SND_TIMEOUT, CLOUD_EMSG_SND_TIMEOUT, NULL, 0);
142
goto cleanup;
143
}
144
125
- web_client_decode_path_and_query_string(w, query->data.http_api_v2.query);
145
char *path = (char *)buffer_tostring(w->url_path_decoded);
146
147
if (aclk_stats_enabled) {
@@ -134,41 +153,24 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
153
}
154
155
w->response.code = (short)web_client_api_request_with_node_selection(localhost, w, path);
137
- web_client_timeout_checkpoint_response_ready(w, &t);
156
+ web_client_timeout_checkpoint_response_ready(w, &dt_ut);
157
158
if (aclk_stats_enabled) {
159
ACLK_STATS_LOCK;
141
- aclk_metrics_per_sample.cloud_q_process_total += t;
160
+ aclk_metrics_per_sample.cloud_q_process_total += dt_ut;
161
aclk_metrics_per_sample.cloud_q_process_count++;
143
- if (aclk_metrics_per_sample.cloud_q_process_max < t)
144
- aclk_metrics_per_sample.cloud_q_process_max = t;
162
+ if (aclk_metrics_per_sample.cloud_q_process_max < dt_ut)
163
+ aclk_metrics_per_sample.cloud_q_process_max = dt_ut;
164
ACLK_STATS_UNLOCK;
165
}
166
167
size = w->response.data->len;
168
sent = size;
169
151
- // check if gzip encoding can and should be used
152
- if ((start = strstr((char *)query->data.http_api_v2.payload, WEB_HDR_ACCEPT_ENC))) {
153
- start += strlen(WEB_HDR_ACCEPT_ENC);
154
- end = strstr(start, "\x0D\x0A");
155
- start = strstr(start, "gzip");
156
-
157
- if (start && start < end) {
158
- w->response.zstream.zalloc = Z_NULL;
159
- w->response.zstream.zfree = Z_NULL;
160
- w->response.zstream.opaque = Z_NULL;
161
- if(deflateInit2(&w->response.zstream, web_gzip_level, Z_DEFLATED, 15 + 16, 8, web_gzip_strategy) == Z_OK) {
162
- w->response.zinitialized = true;
163
- w->response.zoutput = true;
164
- } else
165
- netdata_log_error("Failed to initialize zlib. Proceeding without compression.");
166
- }
167
- }
168
-
170
if (w->response.data->len && w->response.zinitialized) {
171
w->response.zstream.next_in = (Bytef *)w->response.data->buffer;
172
w->response.zstream.avail_in = w->response.data->len;
173
+
174
do {
175
w->response.zstream.avail_out = NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE;
176
w->response.zstream.next_out = w->response.zbuffer;
@@ -188,6 +190,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
190
memcpy(&z_buffer->buffer[z_buffer->len], w->response.zbuffer, bytes_to_cpy);
191
z_buffer->len += bytes_to_cpy;
192
} while(z_ret != Z_STREAM_END);
193
+
194
// so that web_client_build_http_header
195
// puts correct content length into header
196
buffer_free(w->response.data);
@@ -213,7 +216,9 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
216
}
217
218
// send msg.
216
- w->response.code = aclk_http_msg_v2(query_thr->client, query->callback_topic, query->msg_id, t, query->created, w->response.code, local_buffer->buffer, local_buffer->len);
219
+ w->response.code = (short)aclk_http_msg_v2(query_thr->client, query->callback_topic, query->msg_id,
220
+ dt_ut, query->created, w->response.code,
221
+ local_buffer->buffer, local_buffer->len);
222
223
cleanup:
224
web_client_log_completed_request(w, false);
aclk/aclk_query_queue.c
+2
-14
@@ -10,11 +10,9 @@ static netdata_mutex_t aclk_query_queue_mutex = NETDATA_MUTEX_INITIALIZER;
10
11
static struct aclk_query_queue {
12
aclk_query_t head;
13
- aclk_query_t tail;
13
int block_push;
14
} aclk_query_queue = {
15
.head = NULL,
17
- .tail = NULL,
16
.block_push = 0
17
};
18
@@ -31,15 +29,7 @@ static inline int _aclk_queue_query(aclk_query_t query)
29
aclk_query_free(query);
30
return 1;
31
}
34
- if (!aclk_query_queue.head) {
35
- aclk_query_queue.head = query;
36
- aclk_query_queue.tail = query;
37
- ACLK_QUEUE_UNLOCK;
38
- return 0;
39
- }
40
- // TODO deduplication
41
- aclk_query_queue.tail->next = query;
42
- aclk_query_queue.tail = query;
32
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(aclk_query_queue.head, query, prev, next);
33
ACLK_QUEUE_UNLOCK;
34
return 0;
35
@@ -77,9 +67,7 @@ aclk_query_t aclk_queue_pop(void)
67
return ret;
68
}
69
80
- aclk_query_queue.head = ret->next;
81
- if (unlikely(!aclk_query_queue.head))
82
- aclk_query_queue.tail = aclk_query_queue.head;
70
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(aclk_query_queue.head, ret, prev, next);
71
ACLK_QUEUE_UNLOCK;
72
73
ret->next = NULL;
aclk/aclk_query_queue.h
+1
-1
@@ -55,7 +55,7 @@ struct aclk_query {
55
struct timeval created_tv;
56
usec_t created;
57
int timeout;
58
- aclk_query_t next;
58
+ aclk_query_t prev, next;
59
60
// TODO maybe remove?
61
int version;
aclk/aclk_rx_msgs.c
+2
-2
@@ -101,7 +101,7 @@ static inline int aclk_v2_payload_get_query(const char *payload, char **query_ur
101
const char *start, *end;
102
103
// TODO better check of URL
104
- if(strncmp(payload, ACLK_CLOUD_REQ_V2_PREFIX, strlen(ACLK_CLOUD_REQ_V2_PREFIX))) {
104
+ if(strncmp(payload, ACLK_CLOUD_REQ_V2_PREFIX, strlen(ACLK_CLOUD_REQ_V2_PREFIX)) != 0) {
105
errno = 0;
106
netdata_log_error("Only accepting requests that start with \"%s\" from CLOUD.", ACLK_CLOUD_REQ_V2_PREFIX);
107
return 1;
@@ -196,7 +196,7 @@ int aclk_handle_cloud_cmd_message(char *payload)
196
197
// Originally we were expecting to have multiple types of 'cmd' message,
198
// but after the new protocol was designed we will ever only have 'http'
199
- if (strcmp(cloud_to_agent.type_id, "http")) {
199
+ if (strcmp(cloud_to_agent.type_id, "http") != 0) {
200
error_report("Only 'http' cmd message is supported");
201
goto err_cleanup;
202
}
collectors/apps.plugin/apps_plugin.c
+7
-5
@@ -4397,7 +4397,9 @@ static void apps_plugin_function_processes_help(const char *transaction) {
4397
buffer_json_add_array_item_double(wb, _tmp); \
4398
} while(0)
4399
4400
-static void function_processes(const char *transaction, char *function __maybe_unused, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) {
4400
+static void function_processes(const char *transaction, char *function __maybe_unused,
4401
+ usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused,
4402
+ BUFFER *payload __maybe_unused, const char *source __maybe_unused, void *data __maybe_unused) {
4403
struct pid_stat *p;
4404
4405
char *words[PLUGINSD_MAX_WORDS] = { NULL };
@@ -4459,8 +4461,8 @@ static void function_processes(const char *transaction, char *function __maybe_u
4461
return;
4462
}
4463
else {
4462
- char msg[PLUGINSD_LINE_MAX];
4463
- snprintfz(msg, PLUGINSD_LINE_MAX, "Invalid parameter '%s'", keyword);
4464
+ char msg[1024];
4465
+ snprintfz(msg, sizeof(msg), "Invalid parameter '%s'", keyword);
4466
pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST, msg);
4467
return;
4468
}
@@ -4472,7 +4474,7 @@ static void function_processes(const char *transaction, char *function __maybe_u
4474
unsigned int memory_divisor = 1024;
4475
unsigned int io_divisor = 1024 * RATES_DETAIL;
4476
4475
- BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
4477
+ BUFFER *wb = buffer_create(4096, NULL);
4478
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS);
4479
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
4480
buffer_json_member_add_string(wb, "type", "table");
@@ -5348,7 +5350,7 @@ int main(int argc, char **argv) {
5350
struct functions_evloop_globals *wg =
5351
functions_evloop_init(1, "APPS", &apps_and_stdout_mutex, &apps_plugin_exit);
5352
5351
- functions_evloop_add_function(wg, "processes", function_processes, PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT);
5353
+ functions_evloop_add_function(wg, "processes", function_processes, PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, NULL);
5354
5355
// ------------------------------------------------------------------------
5356
collectors/cgroups.plugin/cgroup-internals.h
+2
-17
@@ -454,23 +454,8 @@ static inline char *cgroup_chart_type(char *buffer, struct cgroup *cg) {
454
#define RRDFUNCTIONS_CGTOP_HELP "View running containers"
455
#define RRDFUNCTIONS_SYSTEMD_SERVICES_HELP "View systemd services"
456
457
-int cgroup_function_cgroup_top(uuid_t *transaction, BUFFER *wb,
458
- usec_t *stop_monotonic_ut, const char *function, void *collector_data,
459
- rrd_function_result_callback_t result_cb, void *result_cb_data,
460
- rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
461
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
462
- rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
463
- rrd_function_register_progresser_cb_t register_progresser_cb,
464
- void *register_progresser_cb_data);
465
-
466
-int cgroup_function_systemd_top(uuid_t *transaction, BUFFER *wb,
467
- usec_t *stop_monotonic_ut, const char *function, void *collector_data,
468
- rrd_function_result_callback_t result_cb, void *result_cb_data,
469
- rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
470
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
471
- rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
472
- rrd_function_register_progresser_cb_t register_progresser_cb,
473
- void *register_progresser_cb_data);
457
+int cgroup_function_cgroup_top(BUFFER *wb, const char *function);
458
+int cgroup_function_systemd_top(BUFFER *wb, const char *function);
459
460
void cgroup_netdev_link_init(void);
461
const DICTIONARY_ITEM *cgroup_netdev_get(struct cgroup *cg);
collectors/cgroups.plugin/cgroup-top.c
+4
-42
@@ -97,17 +97,7 @@ void cgroup_netdev_get_bandwidth(struct cgroup *cg, NETDATA_DOUBLE *received, NE
97
*sent = t->sent[slot];
98
}
99
100
-int cgroup_function_cgroup_top(uuid_t *transaction __maybe_unused, BUFFER *wb,
101
- usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
102
- void *collector_data __maybe_unused,
103
- rrd_function_result_callback_t result_cb, void *result_cb_data,
104
- rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
105
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
106
- rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
107
- void *register_canceller_cb_data __maybe_unused,
108
- rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
109
- void *register_progresser_cb_data __maybe_unused) {
110
-
100
+int cgroup_function_cgroup_top(BUFFER *wb, const char *function __maybe_unused) {
101
buffer_flush(wb);
102
wb->content_type = CT_APPLICATION_JSON;
103
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
@@ -334,29 +324,10 @@ int cgroup_function_cgroup_top(uuid_t *transaction __maybe_unused, BUFFER *wb,
324
buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + 1);
325
buffer_json_finalize(wb);
326
337
- int response = HTTP_RESP_OK;
338
- if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) {
339
- buffer_flush(wb);
340
- response = HTTP_RESP_CLIENT_CLOSED_REQUEST;
341
- }
342
-
343
- if(result_cb)
344
- result_cb(wb, response, result_cb_data);
345
-
346
- return response;
327
+ return HTTP_RESP_OK;
328
}
329
349
-int cgroup_function_systemd_top(uuid_t *transaction __maybe_unused, BUFFER *wb,
350
- usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
351
- void *collector_data __maybe_unused,
352
- rrd_function_result_callback_t result_cb, void *result_cb_data,
353
- rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
354
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
355
- rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
356
- void *register_canceller_cb_data __maybe_unused,
357
- rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
358
- void *register_progresser_cb_data __maybe_unused) {
359
-
330
+int cgroup_function_systemd_top(BUFFER *wb, const char *function __maybe_unused) {
331
buffer_flush(wb);
332
wb->content_type = CT_APPLICATION_JSON;
333
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
@@ -514,14 +485,5 @@ int cgroup_function_systemd_top(uuid_t *transaction __maybe_unused, BUFFER *wb,
485
buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + 1);
486
buffer_json_finalize(wb);
487
517
- int response = HTTP_RESP_OK;
518
- if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) {
519
- buffer_flush(wb);
520
- response = HTTP_RESP_CLIENT_CLOSED_REQUEST;
521
- }
522
-
523
- if(result_cb)
524
- result_cb(wb, response, result_cb_data);
525
-
526
- return response;
488
+ return HTTP_RESP_OK;
489
}
collectors/cgroups.plugin/sys_fs_cgroup.c
+6
-7
@@ -1671,16 +1671,15 @@ void *cgroups_main(void *ptr) {
1671
1672
// we register this only on localhost
1673
// for the other nodes, the origin server should register it
1674
- rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
1674
cgroup_netdev_link_init();
1675
1677
- rrd_function_add(localhost, NULL, "containers-vms", 10, RRDFUNCTIONS_PRIORITY_DEFAULT / 2,
1678
- RRDFUNCTIONS_CGTOP_HELP, "top", HTTP_ACCESS_ANY,
1679
- true, cgroup_function_cgroup_top, NULL);
1676
+ rrd_function_add_inline(localhost, NULL, "containers-vms", 10,
1677
+ RRDFUNCTIONS_PRIORITY_DEFAULT / 2, RRDFUNCTIONS_CGTOP_HELP,
1678
+ "top", HTTP_ACCESS_ANY, cgroup_function_cgroup_top);
1679
1681
- rrd_function_add(localhost, NULL, "systemd-services", 10, RRDFUNCTIONS_PRIORITY_DEFAULT / 3,
1682
- RRDFUNCTIONS_SYSTEMD_SERVICES_HELP, "top", HTTP_ACCESS_ANY,
1683
- true, cgroup_function_systemd_top, NULL);
1680
+ rrd_function_add_inline(localhost, NULL, "systemd-services", 10,
1681
+ RRDFUNCTIONS_PRIORITY_DEFAULT / 3, RRDFUNCTIONS_SYSTEMD_SERVICES_HELP,
1682
+ "top", HTTP_ACCESS_ANY, cgroup_function_systemd_top);
1683
1684
heartbeat_t hb;
1685
heartbeat_init(&hb);
collectors/diskspace.plugin/plugin_diskspace.c
+5
-25
@@ -636,17 +636,7 @@ static void diskspace_main_cleanup(void *ptr) {
636
#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
637
#endif
638
639
-int diskspace_function_mount_points(uuid_t *transaction __maybe_unused, BUFFER *wb,
640
- usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
641
- void *collector_data __maybe_unused,
642
- rrd_function_result_callback_t result_cb, void *result_cb_data,
643
- rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
644
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
645
- rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
646
- void *register_canceller_cb_data __maybe_unused,
647
- rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
648
- void *register_progresser_cb_data __maybe_unused) {
649
-
639
+int diskspace_function_mount_points(BUFFER *wb, const char *function __maybe_unused) {
640
buffer_flush(wb);
641
wb->content_type = CT_APPLICATION_JSON;
642
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
@@ -850,16 +840,7 @@ int diskspace_function_mount_points(uuid_t *transaction __maybe_unused, BUFFER *
840
buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + 1);
841
buffer_json_finalize(wb);
842
853
- int response = HTTP_RESP_OK;
854
- if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) {
855
- buffer_flush(wb);
856
- response = HTTP_RESP_CLIENT_CLOSED_REQUEST;
857
- }
858
-
859
- if(result_cb)
860
- result_cb(wb, response, result_cb_data);
861
-
862
- return response;
843
+ return HTTP_RESP_OK;
844
}
845
846
void *diskspace_main(void *ptr) {
@@ -868,10 +849,9 @@ void *diskspace_main(void *ptr) {
849
worker_register_job_name(WORKER_JOB_MOUNTPOINT, "mountpoint");
850
worker_register_job_name(WORKER_JOB_CLEANUP, "cleanup");
851
871
- rrd_collector_started();
872
- rrd_function_add(localhost, NULL, "mount-points", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSPACE_HELP,
873
- "top", HTTP_ACCESS_ANY,
874
- true, diskspace_function_mount_points, NULL);
852
+ rrd_function_add_inline(localhost, NULL, "mount-points", 10,
853
+ RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSPACE_HELP,
854
+ "top", HTTP_ACCESS_ANY, diskspace_function_mount_points);
855
856
netdata_thread_cleanup_push(diskspace_main_cleanup, ptr);
857
collectors/ebpf.plugin/ebpf_functions.c
+6
-3
@@ -277,7 +277,10 @@ void ebpf_socket_read_open_connections(BUFFER *buf, struct ebpf_module *em)
277
static void ebpf_function_socket_manipulation(const char *transaction,
278
char *function __maybe_unused,
279
usec_t *stop_monotonic_ut __maybe_unused,
280
- bool *cancelled __maybe_unused)
280
+ bool *cancelled __maybe_unused,
281
+ BUFFER *payload __maybe_unused,
282
+ const char *source __maybe_unused,
283
+ void *data __maybe_unused)
284
{
285
ebpf_module_t *em = &ebpf_modules[EBPF_MODULE_SOCKET_IDX];
286
@@ -434,7 +437,7 @@ for (int i = 1; i < PLUGINSD_MAX_WORDS; i++) {
437
}
438
pthread_mutex_unlock(&ebpf_exit_cleanup);
439
437
- BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
440
+ BUFFER *wb = buffer_create(4096, NULL);
441
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS);
442
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
443
buffer_json_member_add_string(wb, "type", "table");
@@ -684,7 +687,7 @@ void *ebpf_function_thread(void *ptr)
687
&ebpf_plugin_exit);
688
689
functions_evloop_add_function(
687
- wg, EBPF_FUNCTION_SOCKET, ebpf_function_socket_manipulation, PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT);
690
+ wg, EBPF_FUNCTION_SOCKET, ebpf_function_socket_manipulation, PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, NULL);
691
692
pthread_mutex_lock(&lock);
693
int i;
collectors/freeipmi.plugin/freeipmi_plugin.c
+6
-4
@@ -1471,10 +1471,12 @@ static const char *get_sensor_function_priority(struct sensor *sn) {
1471
}
1472
}
1473
1474
-static void freeimi_function_sensors(const char *transaction, char *function __maybe_unused, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) {
1474
+static void freeimi_function_sensors(const char *transaction, char *function __maybe_unused,
1475
+ usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused,
1476
+ BUFFER *payload __maybe_unused, const char *source __maybe_unused, void *data __maybe_unused) {
1477
time_t expires = now_realtime_sec() + update_every;
1478
1477
- BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
1479
+ BUFFER *wb = buffer_create(4096, NULL);
1480
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS);
1481
buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
1482
buffer_json_member_add_string(wb, "type", "table");
@@ -1973,7 +1975,7 @@ int main (int argc, char **argv) {
1975
size_t iteration = 0;
1976
usec_t step = 100 * USEC_PER_MS;
1977
bool global_chart_created = false;
1976
- bool tty = isatty(fileno(stderr)) == 1;
1978
+ bool tty = isatty(fileno(stdout)) == 1;
1979
1980
heartbeat_t hb;
1981
heartbeat_init(&hb);
@@ -2045,7 +2047,7 @@ int main (int argc, char **argv) {
2047
struct functions_evloop_globals *wg =
2048
functions_evloop_init(1, "FREEIPMI", &stdout_mutex, &function_plugin_should_exit);
2049
functions_evloop_add_function(
2048
- wg, "ipmi-sensors", freeimi_function_sensors, PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT);
2050
+ wg, "ipmi-sensors", freeimi_function_sensors, PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, NULL);
2051
FREEIPMI_GLOBAL_FUNCTION_SENSORS();
2052
}
2053
collectors/plugins.d/README.md
+184
-32
@@ -136,7 +136,8 @@ Netdata parses lines starting with:
136
- `FUNCTION` - define functions
137
- `FUNCTION_PROGRESS` - report the progress of a function execution
138
- `FUNCTION_RESULT_BEGIN` - to initiate the transmission of function results
139
-- `FUNCTION_RESULT_END` - to end the transmission of function results
139
+- `FUNCTION_RESULT_END` - to end the transmission of function result
140
+- `CONFIG` - to define dynamic configuration entities
141
142
a single program can produce any number of charts with any number of dimensions each.
143
@@ -147,7 +148,8 @@ Netdata may send the following commands to the plugin's `stdin`:
148
- `FUNCTION` - to call a specific function, with all parameters inline
149
- `FUNCTION_PAYLOAD` - to call a specific function, with a payload of parameters
150
- `FUNCTION_PAYLOAD_END` - to end the payload of parameters
150
-- `FUNCTION_CANCEL` - cancel a running function transaction
151
+- `FUNCTION_CANCEL` - to cancel a running function transaction - no response is required
152
+- `FUNCTION_PROGRESS` - to report that a user asked the progress of running function call - no response is required
153
154
### Command line parameters
155
@@ -471,43 +473,41 @@ The plugin can register functions to Netdata, like this:
473
> FUNCTION [GLOBAL] "name and parameters of the function" timeout "help string for users" "tags" "access"
474
475
- Tags currently recognized are either `top` or `logs` (or both, space separated).
474
-- Access is one of `any`, `members`, or `admins`.
476
+- Access is one of `any`, `member`, or `admin`:
477
+ - `any` to offer the function to all users of Netdata, even if they are not authenticated.
478
+ - `member` to offer the function to all authenticated members of Netdata.
479
+ - `admin` to offer the function only to authenticated administrators.
480
481
A function can be used by users to ask for more information from the collector. Netdata maintains a registry of functions in 2 levels:
482
483
- per node
484
- per chart
485
481
-Both node and chart functions are exactly the same, but chart functions allow Netdata to relate functions with charts and therefore present a context sensitive menu of functions related to the chart the user is using.
482
-
483
-A function is identified by a string. The allowed characters in the function definition are:
484
-
485
-| Character | Symbol | In Functions |
486
-|-------------------|:------:|:------------:|
487
-| UTF-8 character | UTF-8 | keep |
488
-| Lower case letter | [a-z] | keep |
489
-| Upper case letter | [A-Z] | keep |
490
-| Digit | [0-9] | keep |
491
-| Underscore | _ | keep |
492
-| Comma | , | keep |
493
-| Minus | - | keep |
494
-| Period | . | keep |
495
-| Colon | : | keep |
496
-| Slash | / | keep |
497
-| Space | ' ' | keep |
498
-| Semicolon | ; | : |
499
-| Equal | = | : |
500
-| Backslash | \ | / |
501
-| Anything else | | _ |
502
-
503
-Uses can get a list of all the registered functions using the `/api/v1/functions` end point of Netdata.
504
-
505
-Users can call functions using the `/api/v1/function` end point of Netdata.
486
+Both node and chart functions are exactly the same, but chart functions allow Netdata to relate functions with charts and therefore present a context-sensitive menu of functions related to the chart the user is using.
487
+
488
+Users can get a list of all the registered functions using the `/api/v1/functions` endpoint of Netdata and call functions using the `/api/v1/function` API call of Netdata.
489
+
490
Once a function is called, the plugin will receive at its standard input a command that looks like this:
491
508
-> FUNCTION transaction_id timeout "name and parameters of the function"
492
+> FUNCTION transaction_id timeout "name and parameters of the function as one quoted parameter" "source of request"
493
+
494
+When the function to be called is to receive a payload of parameters, the call looks like this:
495
+
496
+> FUNCTION_PAYLOAD transaction_id timeout "name and parameters of the function as one quoted parameter" "source of request" "content/type"
497
+> body of the payload, formatted according to content/type
498
+> FUNCTION PAYLOAD END
499
+
500
+In this case, Netdata will send:
501
510
-The plugin is expected to parse and validate `name and parameters of the function`. Netdata allows users to edit this string, append more parameters or even change the ones the plugin originally exposed. To minimize the security risk, Netdata guarantees that only the characters shown above are accepted in function definitions, but still the plugin should carefully inspect the `name and parameters of the function` to ensure that it is valid and not harmful.
502
+- A line starting with `FUNCTION_PAYLOAD` together with the required metadata for the function, like the transaction id, the function name and its parameters, the timeout and the content type. This line ends with a newline.
503
+- Then, the payload itself (which may or may not have newlines in it). The payload should be parsed according to the content type parameter.
504
+- Finally, a line starting with `FUNCTION_PAYLOAD_END`, so it is expected like `\nFUNCTION_PAYLOAD_END\n`.
505
+
506
+Note 1: The plugins.d protocol allows parameters without single or double quotes if they don't contain spaces. However, the plugin should be able to parse parameters even if they are enclosed in single or double quotes. If the first character of a parameter is a single quote, its last character should also be a single quote too, and similarly for double quotes.
507
+
508
+Note 2: Netdata always sends the function and its parameters enclosed in double quotes. If the function command and its parameters contain quotes, they are converted to single quotes.
509
+
510
+The plugin is expected to parse and validate `name and parameters of the function as one quotes parameter`. Netdata allows the user interface to manipulate this string by appending more parameters.
511
512
If the plugin rejects the request, it should respond with this:
513
@@ -522,12 +522,12 @@ FUNCTION_RESULT_END
522
523
If the plugin prepares a response, it should send (via its standard output, together with the collected data, but not interleaved with them):
524
525
-> FUNCTION_RESULT_BEGIN transaction_id http_error_code content_type expiration
525
+> FUNCTION_RESULT_BEGIN transaction_id http_response_code content_type expiration
526
527
Where:
528
529
- `transaction_id` is the transaction id that Netdata sent for this function execution
530
- - `http_error` is the http error code Netdata should respond with, 200 is the "ok" response
530
+ - `http_response_code` is the http error code Netdata should respond with, 200 is the "ok" response
531
- `content_type` is the content type of the response
532
- `expiration` is the absolute timestamp (number, unix epoch) this response expires
533
@@ -543,6 +543,158 @@ This defines the end of the message. `FUNCTION_RESULT_END` should appear in a li
543
544
After this line, Netdata resumes processing collected metrics from the plugin.
545
546
+The maximum uncompressed payload size Netdata will accept is 100MB.
547
+
548
+##### Functions cancellation
549
+
550
+Netdata is able to detect when a user made an API request, but abandoned it before it was completed. If this happens to an API called for a function served by the plugin, Netdata will generate a `FUNCTION_CANCEL` request to let the plugin know that it can stop processing the query.
551
+
552
+After receiving such a command, the plugin **must still send a response for the original function request**, to wake up any waiting threads before they timeout. The http response code is not important, since the response will be discarded, however for auditing reasons we suggest to send back a 499 http response code. This is not a standard response code according to the HTTP protocol, but web servers like `nginx` are using it to indicate that a request was abandoned by a user.
553
+
554
+##### Functions progress
555
+
556
+When a request takes too long to be processed, Netdata allows the plugin to report progress to Netdata, which in turn will report progress to the caller.
557
+
558
+The plugin can send `FUNCTION_PROGRESS` like this:
559
+
560
+> FUNCTION_PROGRESS transaction_id done all
561
+
562
+Where:
563
+
564
+- `transaction_id` is the transaction id of the function request
565
+- `done` is an integer value indicating the amount of work done
566
+- `all` is an integer value indicating the total amount of work to be done
567
+
568
+Netdata supports two kinds of progress:
569
+- progress as a percentage, which is calculated as `done * 100 / all`
570
+- progress without knowing the total amount of work to be done, which is enabled when the plugin reports `all` as zero.
571
+
572
+##### Functions timeout
573
+
574
+All functions calls specify a timeout, at which all the intermediate routing nodes (parents, web server threads) will time out and abort the call.
575
+
576
+However, all intermediate routing nodes are configured to extend the timeout when the caller asks for progress. This works like this:
577
+
578
+When a progress request is received, if the expected timeout of the request is less than or equal to 10 seconds, the expected timeout is extended by 10 seconds.
579
+
580
+Usually, the user interface asks for a progress every second. So, during the last 10 seconds of the timeout, every progress request made shifts the timeout 10 seconds to the future.
581
+
582
+To accomplish this, when Netdata receives a progress request by a user, it generates progress requests to the plugin, updating all the intermediate nodes to extend their timeout if necessary.
583
+
584
+The plugin will receive progress requests like this:
585
+
586
+> FUNCTION_PROGRESS transaction_id
587
+
588
+There is no need to respond to this command. It is only there to let the plugin know that a user is still waiting for the query to finish.
589
+
590
+#### CONFIG
591
+
592
+`CONFIG` commands sent from the plugin to Netdata define dynamic configuration entities. These configurable entities are exposed to the user interface, allowing users to change configuration at runtime.
593
+
594
+Dynamically configurations made this way are saved to disk by Netdata and are replayed automatically when Netdata or the plugin restarts.
595
+
596
+`CONFIG` commands look like this:
597
+
598
+> CONFIG id action ...
599
+
600
+Where:
601
+
602
+- `id` is a unique identifier for the configurable entity. This should by design be unique across Netdata. It should be something like `plugin:module:jobs`, e.g. `go.d:postgresql:jobs:masterdb`. This is assumed to be colon-separated with the last part (`masterdb` in our example), being the one displayed to users when there ano conflicts under the same configuration path.
603
+- `action` can be:
604
+ - `create`, to declare the dynamic configuration entity
605
+ - `delete`, to delete the dynamic configuration entity - this does not delete user configuration, we if an entity with the same id is created in the future, the saved configuration will be given to it.
606
+ - `status`, to update the dynamic configuration entity status
607
+
608
+> IMPORTANT:<br/>
609
+> The plugin should blindly create, delete and update the status of its dynamic configuration entities, without any special logic applied to it. Netdata needs to be updated of what is actually happening at the plugin. Keep in mind that creating dynamic configuration entities triggers responses from Netdata, depending on its type and status. Re-creating a job, triggers the same responses every time.
610
+
611
+
612
+When the `action` is `create`, the following additional parameters are expected:
613
+
614
+> CONFIG id action status type "path" source_type "source" "supported commands"
615
+
616
+Where:
617
+
618
+- `action` should be `create`
619
+- `status` can be:
620
+ - `accepted`, the plugin accepted the configuration, but it is not running yet.
621
+ - `running`, the plugin accepted and runs the configuration.
622
+ - `failed`, the plugin tries to run the configuration but it fails.
623
+ - `incomplete`, the plugin needs additional settings to run this configuration. This is usually used for the cases the plugin discovered a job, but important information is missing for it to work.
624
+ - `disabled`, the configuration has been disabled by a user.
625
+ - `orphan`, the configuration is not claimed by any plugin. This is used internally by Netdata to mark the configuration nodes available, for which there is no plugin related to them. Do not use in plugins directly.
626
+- `type` can be `single`, `template` or `job`:
627
+ - `single` is used when the configurable entity is fixed and users should never be able to add or delete it.
628
+ - `template` is used to define a template based on which users can add multiple configurations, like adding data collection jobs. So, the plugin defines the template of the jobs and users are presented with a `[+]` button to add such configuration jobs. The plugin can define multiple templates by giving different `id`s to them.
629
+ - `job` is used to define a job of a template. The plugin should always add all its jobs, independently of the way they have been discovered. It is important to note the relation between `template` and `job` when it comes it the `id`: The `id` of the template should be the prefix of the `job`'s `id`. For example, if the template is `go.d:postgresql:jobs`, then all its jobs be like `go.d:postgresql:jobs:jobname`.
630
+- `path` is the absolute path of the configurable entity inside the tree of Netdata configurations. Usually, this is should be `/collectors`.
631
+- `source` can be `internal`, `stock`, `user`, `discovered` or `dyncfg`:
632
+ - `internal` is used for configurations that are based on internal code settings
633
+ - `stock` is used for default configurations
634
+ - `discovered` is used for dynamic configurations the plugin discovers by its own
635
+ - `user` is used for user configurations, usually via a configuration file
636
+ - `dyncfg` is used for configuration received via this dynamic configuration mechanism
637
+- `source` should provide more details about the exact source of the configuration, like `line@file`, or `user@ip`, etc.
638
+- `supported_commands` is a space separated list of the following keywords, enclosed in single or double quotes. These commands are used by the user interface to determine the actions the users can take:
639
+ - `schema`, to expose the JSON schema for the user interface. This is mandatory for all configurable entities. When `schema` requests are received, Netdata will first attempt to load the schema from `/etc/netdata/schema.d/` and `/var/lib/netdata/conf.d/schema.d`. For jobs, it will serve the schema of their template. If no schema is found for the required `id`, the `schema` request will be forwarded to the plugin, which is expected to send back the relevant schema.
640
+ - `get`, to expose the current configuration values, according the schema defined. `templates` cannot support `get`, since they don't maintain any data.
641
+ - `update`, to receive configuration updates for this entity. `templates` cannot support `update`, since they don't maintain any data.
642
+ - `test`, like `update` but only test the configuration and report success or failure.
643
+ - `add`, to receive job creation commands for templates. Only `templates` should support this command.
644
+ - `remove`, to remove a configuration. Only `jobs` should support this command.
645
+ - `enable` and `disable`, to receive user requests to enable and disable this entity. Adding only one of `enable` or `disable` to the supported commands, Netdata will add both of them. The plugin should expose these commands on `templates` only when it wants to receive `enable` and `disable` commands for all the `jobs` of this `template`.
646
+ - `restart`, to restart a job.
647
+
648
+The plugin receives commands as if it had exposed a `FUNCTION` named `config`. Netdata formats all these calls like this:
649
+
650
+> config id command
651
+
652
+Where `id` is the unique id of the configurable entity and `command` is one of the supported commands the plugin sent to Netdata.
653
+
654
+The plugin will receive (for commands: `schema`, `get`, `remove`, `enable` and `disable`):
655
+
656
+```
657
+FUNCTION transaction_id timeout "config id command"
658
+```
659
+
660
+or (for commands: `update`, `add` and `test`):
661
+
662
+```
663
+FUNCTION_PAYLOAD transaction_id timeout "config id command" "content/type"
664
+body of the payload formatted according to content/type
665
+FUNCTION_PAYLOAD_END
666
+```
667
+
668
+Once received, the plugin should process it and respond accordingly.
669
+
670
+Immediately after the plugin adds a configuration entity, if the commands `enable` and `disable` are supported by it, Netdata will send either `enable` or `disable` for it, based on the last user action, which has been persisted to disk.
671
+
672
+Plugin responses follow the same format `FUNCTIONS` do:
673
+
674
+```
675
+FUNCTION_RESULT_BEGIN transaction_id http_response_code content/type expiration
676
+body of the response formatted according to content/type
677
+FUNCTION_RESULT_END
678
+```
679
+
680
+Successful responses (HTTP response code 200) to `schema` and `get` should send back the relevant JSON object.
681
+All other responses should have the following response body:
682
+
683
+```json
684
+{
685
+ "status" : 404,
686
+ "message" : "some text"
687
+}
688
+```
689
+
690
+The user interface presents the message to users, even when the response is successful (HTTP code 200).
691
+
692
+When responding to additions and updates, Netdata uses the following success response codes to derive additional information:
693
+
694
+- `200`, responding with 200, means the configuration has been accepted and it is running.
695
+- `202`, responding with 202, means the configuration has been accepted but it is not yet running. A subsequent `status` action will update it.
696
+- `299`, responding with 299, means the configuration has been accepted but a restart is required to apply it.
697
+
698
## Data collection
699
700
data collection is defined as a series of `BEGIN` -> `SET` -> `END` lines
collectors/plugins.d/gperf-config.txt
+20
-16
@@ -36,25 +36,29 @@ LABEL, 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_R
36
OVERWRITE, 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 18
37
SET, 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19
38
VARIABLE, 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 20
39
-DYNCFG_ENABLE, 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21
40
-DYNCFG_REGISTER_MODULE, 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22
41
-DYNCFG_REGISTER_JOB, 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23
42
-DYNCFG_RESET, 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24
43
-REPORT_JOB_STATUS, 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25
44
-DELETE_JOB, 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26
39
+CONFIG, 100, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 21
40
#
41
# Streaming only keywords
42
#
48
-CLAIMED_ID, 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 27
49
-BEGIN2, 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28
50
-SET2, 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29
51
-END2, 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30
43
+CLAIMED_ID, 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 22
44
+BEGIN2, 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23
45
+SET2, 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24
46
+END2, 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25
47
#
48
# Streaming Replication keywords
49
#
55
-CHART_DEFINITION_END, 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31
56
-RBEGIN, 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32
57
-RDSTATE, 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33
58
-REND, 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34
59
-RSET, 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35
60
-RSSTATE, 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36
50
+CHART_DEFINITION_END, 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26
51
+RBEGIN, 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27
52
+RDSTATE, 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28
53
+REND, 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29
54
+RSET, 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30
55
+RSSTATE, 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31
56
+#
57
+# obsolete - do nothing commands
58
+#
59
+DYNCFG_ENABLE, 901, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32
60
+DYNCFG_REGISTER_MODULE, 902, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33
61
+DYNCFG_REGISTER_JOB, 903, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34
62
+DYNCFG_RESET, 904, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35
63
+REPORT_JOB_STATUS, 905, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36
64
+DELETE_JOB, 906, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 37
collectors/plugins.d/gperf-hashtable.h
+95
-89
@@ -30,11 +30,11 @@
30
#endif
31
32
33
-#define GPERF_PARSER_TOTAL_KEYWORDS 36
33
+#define GPERF_PARSER_TOTAL_KEYWORDS 37
34
#define GPERF_PARSER_MIN_WORD_LENGTH 3
35
#define GPERF_PARSER_MAX_WORD_LENGTH 22
36
-#define GPERF_PARSER_MIN_HASH_VALUE 3
37
-#define GPERF_PARSER_MAX_HASH_VALUE 48
36
+#define GPERF_PARSER_MIN_HASH_VALUE 7
37
+#define GPERF_PARSER_MAX_HASH_VALUE 52
38
/* maximum key range = 46, duplicates = 0 */
39
40
#ifdef __GNUC__
@@ -49,116 +49,122 @@ gperf_keyword_hash_function (register const char *str, register size_t len)
49
{
50
static unsigned char asso_values[] =
51
{
52
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
53
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
54
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
55
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
56
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
57
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
58
- 49, 49, 49, 49, 49, 23, 29, 0, 0, 0,
59
- 0, 49, 9, 0, 49, 49, 20, 49, 0, 8,
60
- 49, 49, 1, 12, 49, 23, 6, 49, 2, 0,
61
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
62
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
63
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
64
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
65
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
66
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
67
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
68
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
69
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
70
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
71
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
72
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
73
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
74
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
75
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
76
- 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
77
- 49, 49, 49, 49, 49, 49
52
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
53
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
54
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
55
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
56
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
57
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
58
+ 53, 53, 53, 53, 53, 6, 24, 3, 9, 6,
59
+ 0, 53, 3, 27, 53, 53, 33, 53, 42, 0,
60
+ 53, 53, 0, 30, 53, 12, 3, 53, 9, 0,
61
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
62
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
63
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
64
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
65
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
66
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
67
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
68
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
69
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
70
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
71
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
72
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
73
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
74
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
75
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
76
+ 53, 53, 53, 53, 53, 53, 53, 53, 53, 53,
77
+ 53, 53, 53, 53, 53, 53
78
};
79
return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];
80
}
81
82
static PARSER_KEYWORD gperf_keywords[] =
83
{
84
+ {(char*)0}, {(char*)0}, {(char*)0}, {(char*)0},
85
{(char*)0}, {(char*)0}, {(char*)0},
85
-#line 30 "gperf-config.txt"
86
- {"END", 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13},
87
-#line 51 "gperf-config.txt"
88
- {"END2", 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30},
89
-#line 58 "gperf-config.txt"
90
- {"REND", 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34},
91
-#line 17 "gperf-config.txt"
92
- {"EXIT", 99, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3},
93
-#line 16 "gperf-config.txt"
94
- {"DISABLE", 98, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2},
95
-#line 57 "gperf-config.txt"
96
- {"RDSTATE", 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33},
97
-#line 29 "gperf-config.txt"
98
- {"DIMENSION", 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 12},
99
-#line 44 "gperf-config.txt"
100
- {"DELETE_JOB", 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26},
86
+#line 18 "gperf-config.txt"
87
+ {"HOST", 71, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 4},
88
{(char*)0},
102
-#line 42 "gperf-config.txt"
103
- {"DYNCFG_RESET", 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24},
89
#line 39 "gperf-config.txt"
105
- {"DYNCFG_ENABLE", 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21},
90
+ {"CONFIG", 100, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 21},
91
+#line 53 "gperf-config.txt"
92
+ {"REND", 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29},
93
#line 26 "gperf-config.txt"
94
{"CHART", 32, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 9},
108
-#line 37 "gperf-config.txt"
109
- {"SET", 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19},
110
-#line 50 "gperf-config.txt"
111
- {"SET2", 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29},
112
-#line 59 "gperf-config.txt"
113
- {"RSET", 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35},
114
-#line 43 "gperf-config.txt"
115
- {"REPORT_JOB_STATUS", 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25},
116
-#line 41 "gperf-config.txt"
117
- {"DYNCFG_REGISTER_JOB", 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23},
118
-#line 60 "gperf-config.txt"
119
- {"RSSTATE", 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36},
120
-#line 18 "gperf-config.txt"
121
- {"HOST", 71, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 4},
122
-#line 40 "gperf-config.txt"
123
- {"DYNCFG_REGISTER_MODULE", 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22},
95
#line 36 "gperf-config.txt"
96
{"OVERWRITE", 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 18},
126
- {(char*)0},
127
-#line 15 "gperf-config.txt"
128
- {"FLUSH", 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1},
129
-#line 27 "gperf-config.txt"
130
- {"CLABEL", 34, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 10},
97
#line 21 "gperf-config.txt"
98
{"HOST_LABEL", 74, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 7},
99
#line 19 "gperf-config.txt"
100
{"HOST_DEFINE", 72, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 5},
135
-#line 55 "gperf-config.txt"
136
- {"CHART_DEFINITION_END", 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31},
137
-#line 48 "gperf-config.txt"
138
- {"CLAIMED_ID", 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 27},
139
-#line 31 "gperf-config.txt"
140
- {"FUNCTION", 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14},
141
-#line 20 "gperf-config.txt"
142
- {"HOST_DEFINE_END", 73, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 6},
143
-#line 28 "gperf-config.txt"
144
- {"CLABEL_COMMIT", 35, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 11},
145
-#line 25 "gperf-config.txt"
146
- {"BEGIN", 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8},
147
-#line 49 "gperf-config.txt"
148
- {"BEGIN2", 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28},
149
-#line 56 "gperf-config.txt"
150
- {"RBEGIN", 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32},
101
+ {(char*)0},
102
+#line 52 "gperf-config.txt"
103
+ {"RDSTATE", 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28},
104
#line 38 "gperf-config.txt"
105
{"VARIABLE", 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 20},
153
- {(char*)0}, {(char*)0},
106
+#line 20 "gperf-config.txt"
107
+ {"HOST_DEFINE_END", 73, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 6},
108
+#line 17 "gperf-config.txt"
109
+ {"EXIT", 99, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3},
110
+#line 31 "gperf-config.txt"
111
+ {"FUNCTION", 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14},
112
+#line 62 "gperf-config.txt"
113
+ {"DYNCFG_RESET", 904, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35},
114
+#line 59 "gperf-config.txt"
115
+ {"DYNCFG_ENABLE", 901, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32},
116
+#line 63 "gperf-config.txt"
117
+ {"REPORT_JOB_STATUS", 905, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36},
118
+ {(char*)0},
119
+#line 64 "gperf-config.txt"
120
+ {"DELETE_JOB", 906, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 37},
121
+#line 50 "gperf-config.txt"
122
+ {"CHART_DEFINITION_END", 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26},
123
+ {(char*)0},
124
+#line 61 "gperf-config.txt"
125
+ {"DYNCFG_REGISTER_JOB", 903, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34},
126
#line 33 "gperf-config.txt"
127
{"FUNCTION_PROGRESS", 43, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16},
156
- {(char*)0}, {(char*)0}, {(char*)0},
128
+#line 51 "gperf-config.txt"
129
+ {"RBEGIN", 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27},
130
+#line 60 "gperf-config.txt"
131
+ {"DYNCFG_REGISTER_MODULE", 902, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33},
132
+ {(char*)0},
133
#line 32 "gperf-config.txt"
134
{"FUNCTION_RESULT_BEGIN", 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15},
159
- {(char*)0}, {(char*)0}, {(char*)0},
135
+#line 54 "gperf-config.txt"
136
+ {"RSET", 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30},
137
+#line 25 "gperf-config.txt"
138
+ {"BEGIN", 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8},
139
+#line 44 "gperf-config.txt"
140
+ {"BEGIN2", 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23},
141
+#line 55 "gperf-config.txt"
142
+ {"RSSTATE", 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31},
143
+#line 15 "gperf-config.txt"
144
+ {"FLUSH", 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1},
145
+#line 37 "gperf-config.txt"
146
+ {"SET", 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19},
147
+#line 45 "gperf-config.txt"
148
+ {"SET2", 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24},
149
+ {(char*)0},
150
+#line 27 "gperf-config.txt"
151
+ {"CLABEL", 34, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 10},
152
+#line 16 "gperf-config.txt"
153
+ {"DISABLE", 98, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2},
154
#line 35 "gperf-config.txt"
161
- {"LABEL", 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17}
155
+ {"LABEL", 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17},
156
+#line 29 "gperf-config.txt"
157
+ {"DIMENSION", 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 12},
158
+#line 43 "gperf-config.txt"
159
+ {"CLAIMED_ID", 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 22},
160
+ {(char*)0}, {(char*)0},
161
+#line 28 "gperf-config.txt"
162
+ {"CLABEL_COMMIT", 35, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 11},
163
+ {(char*)0},
164
+#line 30 "gperf-config.txt"
165
+ {"END", 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13},
166
+#line 46 "gperf-config.txt"
167
+ {"END2", 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25}
168
};
169
170
PARSER_KEYWORD *
collectors/plugins.d/plugins_d.h
-11
@@ -10,14 +10,6 @@
10
#define PLUGINSD_CMD_MAX (FILENAME_MAX*2)
11
#define PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH 0
12
13
-#define PLUGINSD_KEYWORD_DYNCFG_ENABLE "DYNCFG_ENABLE"
14
-#define PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE "DYNCFG_REGISTER_MODULE"
15
-#define PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB "DYNCFG_REGISTER_JOB"
16
-#define PLUGINSD_KEYWORD_DYNCFG_RESET "DYNCFG_RESET"
17
-
18
-#define PLUGINSD_KEYWORD_REPORT_JOB_STATUS "REPORT_JOB_STATUS"
19
-#define PLUGINSD_KEYWORD_DELETE_JOB "DELETE_JOB"
20
-
13
#define PLUGINSD_MAX_DIRECTORIES 20
14
extern char *plugin_directories[PLUGINSD_MAX_DIRECTORIES];
15
@@ -47,9 +39,6 @@ struct plugind {
39
40
time_t started_t;
41
50
- const DICTIONARY_ITEM *cfg_dict_item;
51
- struct configurable_plugin *configuration;
52
-
42
struct plugind *prev;
43
struct plugind *next;
44
};
collectors/plugins.d/pluginsd_dyncfg.c
+49
-571
@@ -2,584 +2,62 @@
2
3
#include "pluginsd_dyncfg.h"
4
5
-struct mutex_cond {
6
- pthread_mutex_t lock;
7
- pthread_cond_t cond;
8
- int rc;
9
-};
5
11
-static void virt_fnc_got_data_cb(BUFFER *wb __maybe_unused, int code, void *callback_data)
12
-{
13
- struct mutex_cond *ctx = callback_data;
14
- pthread_mutex_lock(&ctx->lock);
15
- ctx->rc = code;
16
- pthread_cond_broadcast(&ctx->cond);
17
- pthread_mutex_unlock(&ctx->lock);
18
-}
19
-
20
-#define VIRT_FNC_TIMEOUT_S 10
21
-#define VIRT_FNC_BUF_SIZE (4096)
22
-void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data) {
23
- PARSER *parser = NULL;
24
-
25
- //TODO simplify (as we really need only first parameter to get plugin name maybe we can avoid parsing all)
26
- char *words[PLUGINSD_MAX_WORDS];
27
- char *function_with_params = strdupz(name);
28
- size_t num_words = quoted_strings_splitter(function_with_params, words, PLUGINSD_MAX_WORDS, isspace_map_pluginsd);
29
-
30
- if (num_words < 2) {
31
- netdata_log_error("PLUGINSD: virtual function name is empty.");
32
- freez(function_with_params);
33
- return;
34
- }
35
-
36
- const DICTIONARY_ITEM *cpi = dictionary_get_and_acquire_item(host->configurable_plugins, get_word(words, num_words, 1));
37
- if (unlikely(cpi == NULL)) {
38
- netdata_log_error("PLUGINSD: virtual function plugin '%s' not found.", name);
39
- freez(function_with_params);
40
- return;
41
- }
42
- struct configurable_plugin *cp = dictionary_acquired_item_value(cpi);
43
- parser = (PARSER *)cp->cb_usr_ctx;
44
-
45
- BUFFER *function_out = buffer_create(VIRT_FNC_BUF_SIZE, NULL);
46
- // if we are forwarding this to a plugin (as opposed to streaming/child) we have to remove the first parameter (plugin_name)
47
- buffer_strcat(function_out, get_word(words, num_words, 0));
48
- for (size_t i = 1; i < num_words; i++) {
49
- if (i == 1 && SERVING_PLUGINSD(parser))
50
- continue;
51
- buffer_sprintf(function_out, " %s", get_word(words, num_words, i));
52
- }
53
- freez(function_with_params);
54
-
55
- usec_t now_ut = now_monotonic_usec();
56
-
57
- struct inflight_function tmp = {
58
- .started_monotonic_ut = now_ut,
59
- .result_body_wb = wb,
60
- .timeout_s = VIRT_FNC_TIMEOUT_S,
61
- .function = string_strdupz(buffer_tostring(function_out)),
62
- .payload = payload != NULL ? strdupz(payload) : NULL,
63
- .virtual = true,
64
-
65
- .result = {
66
- .cb = callback,
67
- .data = callback_data,
68
- },
69
- .dyncfg = {
70
- .stop_monotonic_ut = now_ut + VIRT_FNC_TIMEOUT_S * USEC_PER_SEC,
71
- }
72
- };
73
- tmp.stop_monotonic_ut = &tmp.dyncfg.stop_monotonic_ut;
74
- buffer_free(function_out);
75
-
76
- uuid_generate_time(tmp.transaction);
77
- char key[UUID_COMPACT_STR_LEN];
78
- uuid_unparse_lower_compact(tmp.transaction, key);
79
-
80
- dictionary_write_lock(parser->inflight.functions);
81
-
82
- // if there is any error, our dictionary callbacks will call the caller callback to notify
83
- // the caller about the error - no need for error handling here.
84
- dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function));
85
-
86
- if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut)
87
- parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
88
-
89
- // garbage collect stale inflight functions
90
- if(parser->inflight.smaller_monotonic_timeout_ut < now_ut)
91
- pluginsd_inflight_functions_garbage_collect(parser, now_ut);
92
-
93
- dictionary_write_unlock(parser->inflight.functions);
94
-}
95
-
96
-
97
-dyncfg_config_t call_virtual_function_blocking(PARSER *parser, const char *name, int *rc, const char *payload) {
98
- usec_t now_ut = now_monotonic_usec();
99
- BUFFER *wb = buffer_create(VIRT_FNC_BUF_SIZE, NULL);
100
-
101
- struct mutex_cond cond = {
102
- .lock = PTHREAD_MUTEX_INITIALIZER,
103
- .cond = PTHREAD_COND_INITIALIZER
104
- };
105
-
106
- struct inflight_function tmp = {
107
- .started_monotonic_ut = now_ut,
108
- .result_body_wb = wb,
109
- .timeout_s = VIRT_FNC_TIMEOUT_S,
110
- .function = string_strdupz(name),
111
- .payload = payload != NULL ? strdupz(payload) : NULL,
112
- .virtual = true,
113
-
114
- .result = {
115
- .cb = virt_fnc_got_data_cb,
116
- .data = &cond,
117
- },
118
- .dyncfg = {
119
- .stop_monotonic_ut = now_ut + VIRT_FNC_TIMEOUT_S * USEC_PER_SEC,
120
- }
121
- };
122
- tmp.stop_monotonic_ut = &tmp.dyncfg.stop_monotonic_ut;
123
-
124
- uuid_generate_time(tmp.transaction);
125
-
126
- char key[UUID_COMPACT_STR_LEN];
127
- uuid_unparse_lower_compact(tmp.transaction, key);
128
-
129
- dictionary_write_lock(parser->inflight.functions);
130
-
131
- // if there is any error, our dictionary callbacks will call the caller callback to notify
132
- // the caller about the error - no need for error handling here.
133
- dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function));
134
-
135
- if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut)
136
- parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
137
-
138
- // garbage collect stale inflight functions
139
- if(parser->inflight.smaller_monotonic_timeout_ut < now_ut)
140
- pluginsd_inflight_functions_garbage_collect(parser, now_ut);
141
-
142
- dictionary_write_unlock(parser->inflight.functions);
143
-
144
- struct timespec tp;
145
- clock_gettime(CLOCK_REALTIME, &tp);
146
- tp.tv_sec += (time_t)VIRT_FNC_TIMEOUT_S;
147
-
148
- pthread_mutex_lock(&cond.lock);
149
-
150
- int ret = pthread_cond_timedwait(&cond.cond, &cond.lock, &tp);
151
- if (ret == ETIMEDOUT)
152
- netdata_log_error("PLUGINSD: DYNCFG virtual function %s timed out", name);
153
-
154
- pthread_mutex_unlock(&cond.lock);
155
-
156
- dyncfg_config_t cfg;
157
- cfg.data = strdupz(buffer_tostring(wb));
158
- cfg.data_size = buffer_strlen(wb);
159
-
160
- if (rc != NULL)
161
- *rc = cond.rc;
162
-
163
- buffer_free(wb);
164
- return cfg;
165
-}
166
-
167
-#define CVF_MAX_LEN (1024)
168
-static dyncfg_config_t get_plugin_config_cb(void *usr_ctx, const char *plugin_name)
169
-{
170
- PARSER *parser = usr_ctx;
171
-
172
- if (SERVING_STREAMING(parser)) {
173
- char buf[CVF_MAX_LEN + 1];
174
- snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG " %s", plugin_name);
175
- return call_virtual_function_blocking(parser, buf, NULL, NULL);
176
- }
177
-
178
- return call_virtual_function_blocking(parser, FUNCTION_NAME_GET_PLUGIN_CONFIG, NULL, NULL);
179
-}
180
-
181
-static dyncfg_config_t get_plugin_config_schema_cb(void *usr_ctx, const char *plugin_name)
182
-{
183
- PARSER *parser = usr_ctx;
184
-
185
- if (SERVING_STREAMING(parser)) {
186
- char buf[CVF_MAX_LEN + 1];
187
- snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA " %s", plugin_name);
188
- return call_virtual_function_blocking(parser, buf, NULL, NULL);
189
- }
190
-
191
- return call_virtual_function_blocking(parser, "get_plugin_config_schema", NULL, NULL);
192
-}
193
-
194
-static dyncfg_config_t get_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
195
-{
196
- PARSER *parser = usr_ctx;
197
- BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
198
-
199
- buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG);
200
- if (SERVING_STREAMING(parser))
201
- buffer_sprintf(wb, " %s", plugin_name);
202
-
203
- buffer_sprintf(wb, " %s", module_name);
204
-
205
- dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
206
-
207
- buffer_free(wb);
208
-
209
- return ret;
210
-}
211
-
212
-static dyncfg_config_t get_module_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
213
-{
214
- PARSER *parser = usr_ctx;
215
- BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
216
-
217
- buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA);
218
- if (SERVING_STREAMING(parser))
219
- buffer_sprintf(wb, " %s", plugin_name);
220
-
221
- buffer_sprintf(wb, " %s", module_name);
222
-
223
- dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
224
-
225
- buffer_free(wb);
226
-
227
- return ret;
228
-}
229
-
230
-static dyncfg_config_t get_job_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
231
-{
232
- PARSER *parser = usr_ctx;
233
- BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
234
-
235
- buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA);
236
-
237
- if (SERVING_STREAMING(parser))
238
- buffer_sprintf(wb, " %s", plugin_name);
239
-
240
- buffer_sprintf(wb, " %s", module_name);
241
-
242
- dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
243
-
244
- buffer_free(wb);
245
-
246
- return ret;
247
-}
248
-
249
-static dyncfg_config_t get_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char* job_name)
250
-{
251
- PARSER *parser = usr_ctx;
252
- BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
253
-
254
- buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG);
255
-
256
- if (SERVING_STREAMING(parser))
257
- buffer_sprintf(wb, " %s", plugin_name);
258
-
259
- buffer_sprintf(wb, " %s %s", module_name, job_name);
260
-
261
- dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
262
-
263
- buffer_free(wb);
264
-
265
- return ret;
266
-}
267
-
268
-enum set_config_result set_plugin_config_cb(void *usr_ctx, const char *plugin_name, dyncfg_config_t *cfg)
269
-{
270
- PARSER *parser = usr_ctx;
271
- BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
272
-
273
- buffer_strcat(wb, FUNCTION_NAME_SET_PLUGIN_CONFIG);
274
-
275
- if (SERVING_STREAMING(parser))
276
- buffer_sprintf(wb, " %s", plugin_name);
277
-
278
- int rc;
279
- call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
280
-
281
- buffer_free(wb);
282
- if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
283
- return SET_CONFIG_REJECTED;
284
- return SET_CONFIG_ACCEPTED;
285
-}
286
-
287
-enum set_config_result set_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, dyncfg_config_t *cfg)
288
-{
289
- PARSER *parser = usr_ctx;
290
- BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
291
-
292
- buffer_strcat(wb, FUNCTION_NAME_SET_MODULE_CONFIG);
293
-
294
- if (SERVING_STREAMING(parser))
295
- buffer_sprintf(wb, " %s", plugin_name);
296
-
297
- buffer_sprintf(wb, " %s", module_name);
298
-
299
- int rc;
300
- call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
301
-
302
- buffer_free(wb);
303
-
304
- if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
305
- return SET_CONFIG_REJECTED;
306
- return SET_CONFIG_ACCEPTED;
307
-}
308
-
309
-enum set_config_result set_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name, dyncfg_config_t *cfg)
310
-{
311
- PARSER *parser = usr_ctx;
312
- BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
313
-
314
- buffer_strcat(wb, FUNCTION_NAME_SET_JOB_CONFIG);
315
-
316
- if (SERVING_STREAMING(parser))
317
- buffer_sprintf(wb, " %s", plugin_name);
318
-
319
- buffer_sprintf(wb, " %s %s", module_name, job_name);
320
-
321
- int rc;
322
- call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
323
-
324
- buffer_free(wb);
325
-
326
- if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
327
- return SET_CONFIG_REJECTED;
328
- return SET_CONFIG_ACCEPTED;
329
-}
330
-
331
-enum set_config_result delete_job_cb(void *usr_ctx, const char *plugin_name ,const char *module_name, const char *job_name)
332
-{
333
- PARSER *parser = usr_ctx;
334
- BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
335
-
336
- buffer_strcat(wb, FUNCTION_NAME_DELETE_JOB);
337
-
338
- if (SERVING_STREAMING(parser))
339
- buffer_sprintf(wb, " %s", plugin_name);
340
-
341
- buffer_sprintf(wb, " %s %s", module_name, job_name);
342
-
343
- int rc;
344
- call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, NULL);
345
-
346
- buffer_free(wb);
347
-
348
- if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
349
- return SET_CONFIG_REJECTED;
350
- return SET_CONFIG_ACCEPTED;
351
-}
352
-
353
-
354
-PARSER_RC pluginsd_register_plugin(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
355
- netdata_log_info("PLUGINSD: DYNCFG_ENABLE");
356
-
357
- if (unlikely (num_words != 2))
358
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "missing name parameter");
359
-
360
- struct configurable_plugin *cfg = callocz(1, sizeof(struct configurable_plugin));
361
-
362
- cfg->name = strdupz(words[1]);
363
- cfg->set_config_cb = set_plugin_config_cb;
364
- cfg->get_config_cb = get_plugin_config_cb;
365
- cfg->get_config_schema_cb = get_plugin_config_schema_cb;
366
- cfg->cb_usr_ctx = parser;
367
-
368
- const DICTIONARY_ITEM *di = register_plugin(parser->user.host->configurable_plugins, cfg, SERVING_PLUGINSD(parser));
369
- if (unlikely(di == NULL)) {
370
- freez(cfg->name);
371
- freez(cfg);
372
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "error registering plugin");
373
- }
374
-
375
- if (SERVING_PLUGINSD(parser)) {
376
- // this is optimization for pluginsd to avoid extra dictionary lookup
377
- // as we know which plugin is comunicating with us
378
- parser->user.cd->cfg_dict_item = di;
379
- parser->user.cd->configuration = cfg;
380
- } else {
381
- // register_plugin keeps the item acquired, so we need to release it
382
- dictionary_acquired_item_release(parser->user.host->configurable_plugins, di);
383
- }
384
-
385
- rrdpush_send_dyncfg_enable(parser->user.host, cfg->name);
386
-
387
- return PARSER_RC_OK;
388
-}
389
-
390
-#define LOG_MSG_SIZE (1024)
391
-#define MODULE_NAME_IDX (SERVING_PLUGINSD(parser) ? 1 : 2)
392
-#define MODULE_TYPE_IDX (SERVING_PLUGINSD(parser) ? 2 : 3)
393
-
394
-PARSER_RC pluginsd_register_module(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
395
- netdata_log_info("PLUGINSD: DYNCFG_REG_MODULE");
396
-
397
- size_t expected_num_words = SERVING_PLUGINSD(parser) ? 3 : 4;
398
-
399
- if (unlikely(num_words != expected_num_words)) {
400
- char log[LOG_MSG_SIZE + 1];
401
- snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name module_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name ");
402
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, log);
6
+// ----------------------------------------------------------------------------
7
+
8
+PARSER_RC pluginsd_config(char **words, size_t num_words, PARSER *parser) {
9
+ RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_CONFIG);
10
+ if(!host) return PARSER_RC_ERROR;
11
+
12
+ size_t i = 1;
13
+ char *id = get_word(words, num_words, i++);
14
+ char *action = get_word(words, num_words, i++);
15
+
16
+ if(strcmp(action, PLUGINSD_KEYWORD_CONFIG_ACTION_CREATE) == 0) {
17
+ char *status_str = get_word(words, num_words, i++);
18
+ char *type_str = get_word(words, num_words, i++);
19
+ char *path = get_word(words, num_words, i++);
20
+ char *source_type_str = get_word(words, num_words, i++);
21
+ char *source = get_word(words, num_words, i++);
22
+ char *supported_cmds_str = get_word(words, num_words, i++);
23
+
24
+ DYNCFG_STATUS status = dyncfg_status2id(status_str);
25
+ DYNCFG_TYPE type = dyncfg_type2id(type_str);
26
+ DYNCFG_SOURCE_TYPE source_type = dyncfg_source_type2id(source_type_str);
27
+ DYNCFG_CMDS cmds = dyncfg_cmds2id(supported_cmds_str);
28
+
29
+ if(!dyncfg_add_low_level(
30
+ host,
31
+ id,
32
+ path,
33
+ status,
34
+ type,
35
+ source_type,
36
+ source,
37
+ cmds,
38
+ 0,
39
+ 0,
40
+ false,
41
+ pluginsd_function_execute_cb,
42
+ parser))
43
+ return PARSER_RC_ERROR;
44
+ }
45
+ else if(strcmp(action, PLUGINSD_KEYWORD_CONFIG_ACTION_DELETE) == 0) {
46
+ dyncfg_del_low_level(host, id);
47
+ }
48
+ else if(strcmp(action, PLUGINSD_KEYWORD_CONFIG_ACTION_STATUS) == 0) {
49
+ char *status_str = get_word(words, num_words, i++);
50
+ dyncfg_status_low_level(host, id, dyncfg_status2id(status_str));
51
}
404
-
405
- struct configurable_plugin *plug_cfg;
406
- const DICTIONARY_ITEM *di = NULL;
407
- if (SERVING_PLUGINSD(parser)) {
408
- plug_cfg = parser->user.cd->configuration;
409
- if (unlikely(plug_cfg == NULL))
410
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "you have to enable dynamic configuration first using " PLUGINSD_KEYWORD_DYNCFG_ENABLE);
411
- } else {
412
- di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]);
413
- if (unlikely(di == NULL))
414
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "plugin not found");
415
-
416
- plug_cfg = (struct configurable_plugin *)dictionary_acquired_item_value(di);
417
- }
418
-
419
- struct module *mod = callocz(1, sizeof(struct module));
420
-
421
- mod->type = str2_module_type(words[MODULE_TYPE_IDX]);
422
- if (unlikely(mod->type == MOD_TYPE_UNKNOWN)) {
423
- freez(mod);
424
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "unknown module type (allowed: job_array, single)");
425
- }
426
-
427
- mod->name = strdupz(words[MODULE_NAME_IDX]);
428
-
429
- mod->set_config_cb = set_module_config_cb;
430
- mod->get_config_cb = get_module_config_cb;
431
- mod->get_config_schema_cb = get_module_config_schema_cb;
432
- mod->config_cb_usr_ctx = parser;
433
-
434
- mod->get_job_config_cb = get_job_config_cb;
435
- mod->get_job_config_schema_cb = get_job_config_schema_cb;
436
- mod->set_job_config_cb = set_job_config_cb;
437
- mod->delete_job_cb = delete_job_cb;
438
- mod->job_config_cb_usr_ctx = parser;
439
-
440
- register_module(parser->user.host->configurable_plugins, plug_cfg, mod, SERVING_PLUGINSD(parser));
441
-
442
- if (di != NULL)
443
- dictionary_acquired_item_release(parser->user.host->configurable_plugins, di);
444
-
445
- rrdpush_send_dyncfg_reg_module(parser->user.host, plug_cfg->name, mod->name, mod->type);
446
-
447
- return PARSER_RC_OK;
448
-}
449
-
450
-static inline PARSER_RC pluginsd_register_job_common(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused, const char *plugin_name) {
451
- const char *module_name = words[0];
452
- const char *job_name = words[1];
453
- const char *job_type_str = words[2];
454
- const char *flags_str = words[3];
455
-
456
- long f = str2l(flags_str);
457
-
458
- if (f < 0)
459
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "invalid flags received");
460
-
461
- dyncfg_job_flg_t flags = f;
462
-
463
- if (SERVING_PLUGINSD(parser))
464
- flags |= JOB_FLG_PLUGIN_PUSHED;
52
else
466
- flags |= JOB_FLG_STREAMING_PUSHED;
467
-
468
- enum job_type job_type = dyncfg_str2job_type(job_type_str);
469
- if (job_type == JOB_TYPE_UNKNOWN)
470
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "unknown job type");
471
-
472
- if (SERVING_PLUGINSD(parser) && job_type == JOB_TYPE_USER)
473
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "plugins cannot push jobs of type \"user\" (this is allowed only in streaming)");
474
-
475
- if (register_job(parser->user.host->configurable_plugins, plugin_name, module_name, job_name, job_type, flags, 0)) // ignore existing is off as this is explicitly called register job
476
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "error registering job");
477
-
478
- rrdpush_send_dyncfg_reg_job(parser->user.host, plugin_name, module_name, job_name, job_type, flags);
53
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "DYNCFG: unknown action '%s' received from plugin", action);
54
55
+ parser->user.data_collections_count++;
56
return PARSER_RC_OK;
57
}
58
483
-PARSER_RC pluginsd_register_job(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
484
- size_t expected_num_words = SERVING_PLUGINSD(parser) ? 5 : 6;
485
-
486
- if (unlikely(num_words != expected_num_words)) {
487
- char log[LOG_MSG_SIZE + 1];
488
- snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name job_name job_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name ");
489
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, log);
490
- }
491
-
492
- if (SERVING_PLUGINSD(parser)) {
493
- return pluginsd_register_job_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name);
494
- }
495
- return pluginsd_register_job_common(&words[2], num_words - 2, parser, words[1]);
496
-}
497
-
498
-PARSER_RC pluginsd_dyncfg_reset(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
499
- if (unlikely(num_words != (SERVING_PLUGINSD(parser) ? 1 : 2)))
500
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, SERVING_PLUGINSD(parser) ? "expected 0 parameters" : "expected 1 parameter: plugin_name");
501
-
502
- if (SERVING_PLUGINSD(parser)) {
503
- unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item);
504
- rrdpush_send_dyncfg_reset(parser->user.host, parser->user.cd->configuration->name);
505
- parser->user.cd->configuration = NULL;
506
- } else {
507
- const DICTIONARY_ITEM *di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]);
508
- if (unlikely(di == NULL))
509
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, "plugin not found");
510
- unregister_plugin(parser->user.host->configurable_plugins, di);
511
- rrdpush_send_dyncfg_reset(parser->user.host, words[1]);
512
- }
59
+// ----------------------------------------------------------------------------
60
61
+PARSER_RC pluginsd_dyncfg_noop(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
62
return PARSER_RC_OK;
63
}
516
-
517
-static inline PARSER_RC pluginsd_job_status_common(char **words, size_t num_words, PARSER *parser, const char *plugin_name) {
518
- int state = str2i(words[3]);
519
-
520
- enum job_status status = str2job_state(words[2]);
521
- if (unlikely(SERVING_PLUGINSD(parser) && status == JOB_STATUS_UNKNOWN))
522
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "unknown job status");
523
-
524
- char *message = NULL;
525
- if (num_words == 5 && strlen(words[4]) > 0)
526
- message = words[4];
527
-
528
- const DICTIONARY_ITEM *plugin_item;
529
- DICTIONARY *job_dict;
530
- const DICTIONARY_ITEM *job_item = report_job_status_acq_lock(parser->user.host->configurable_plugins, &plugin_item, &job_dict, plugin_name, words[0], words[1], status, state, message);
531
-
532
- if (job_item != NULL) {
533
- struct job *job = dictionary_acquired_item_value(job_item);
534
- rrdpush_send_job_status_update(parser->user.host, plugin_name, words[0], job);
535
-
536
- pthread_mutex_unlock(&job->lock);
537
- dictionary_acquired_item_release(job_dict, job_item);
538
- dictionary_acquired_item_release(parser->user.host->configurable_plugins, plugin_item);
539
- }
540
-
541
- return PARSER_RC_OK;
542
-}
543
-
544
-// job_status [plugin_name if streaming] <module_name> <job_name> <status_code> <state> [message]
545
-PARSER_RC pluginsd_job_status(char **words, size_t num_words, PARSER *parser) {
546
- if (SERVING_PLUGINSD(parser)) {
547
- if (unlikely(num_words != 5 && num_words != 6))
548
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 4 or 5 parameters: module_name, job_name, status_code, state, [optional: message]");
549
- } else {
550
- if (unlikely(num_words != 6 && num_words != 7))
551
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 5 or 6 parameters: plugin_name, module_name, job_name, status_code, state, [optional: message]");
552
- }
553
-
554
- if (SERVING_PLUGINSD(parser)) {
555
- return pluginsd_job_status_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name);
556
- }
557
- return pluginsd_job_status_common(&words[2], num_words - 2, parser, words[1]);
558
-}
559
-
560
-PARSER_RC pluginsd_delete_job(char **words, size_t num_words, PARSER *parser) {
561
- // this can confuse a bit but there is a diference between KEYWORD_DELETE_JOB and actual delete_job function
562
- // they are of opossite direction
563
- if (num_words != 4)
564
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DELETE_JOB, "expected 2 parameters: plugin_name, module_name, job_name");
565
-
566
- const char *plugin_name = get_word(words, num_words, 1);
567
- const char *module_name = get_word(words, num_words, 2);
568
- const char *job_name = get_word(words, num_words, 3);
569
-
570
- if (SERVING_STREAMING(parser))
571
- delete_job_pname(parser->user.host->configurable_plugins, plugin_name, module_name, job_name);
572
-
573
- // forward to parent if any
574
- rrdpush_send_job_deleted(parser->user.host, plugin_name, module_name, job_name);
575
- return PARSER_RC_OK;
576
-}
577
-
578
-void pluginsd_dyncfg_cleanup(PARSER *parser) {
579
- if (parser->user.cd != NULL && parser->user.cd->configuration != NULL) {
580
- unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item);
581
- parser->user.cd->configuration = NULL;
582
- } else if (parser->user.host != NULL && SERVING_STREAMING(parser) && parser->user.host != localhost){
583
- dictionary_flush(parser->user.host->configurable_plugins);
584
- }
585
-}
collectors/plugins.d/pluginsd_dyncfg.h
+2
-8
@@ -5,13 +5,7 @@
5
6
#include "pluginsd_internals.h"
7
8
-PARSER_RC pluginsd_register_plugin(char **words, size_t num_words, PARSER *parser);
9
-PARSER_RC pluginsd_register_module(char **words, size_t num_words, PARSER *parser);
10
-PARSER_RC pluginsd_register_job(char **words, size_t num_words, PARSER *parser);
11
-PARSER_RC pluginsd_dyncfg_reset(char **words, size_t num_words, PARSER *parser);
12
-PARSER_RC pluginsd_job_status(char **words, size_t num_words, PARSER *parser);
13
-PARSER_RC pluginsd_delete_job(char **words, size_t num_words, PARSER *parser);
14
-
15
-void pluginsd_dyncfg_cleanup(PARSER *parser);
8
+PARSER_RC pluginsd_config(char **words, size_t num_words, PARSER *parser);
9
+PARSER_RC pluginsd_dyncfg_noop(char **words, size_t num_words, PARSER *parser);
10
11
#endif //NETDATA_PLUGINSD_DYNCFG_H
collectors/plugins.d/pluginsd_functions.c
+88
-143
@@ -21,47 +21,52 @@ static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void
21
if(rc != 0)
22
netdata_log_error("FUNCTION: '%s': cannot parse transaction UUID", string2str(pf->function));
23
24
- char buffer[2048 + 1];
25
- snprintfz(buffer, sizeof(buffer) - 1, "%s %s %d \"%s\"\n",
26
- pf->payload ? PLUGINSD_KEYWORD_FUNCTION_PAYLOAD : PLUGINSD_KEYWORD_FUNCTION,
27
- transaction,
28
- pf->timeout_s,
29
- string2str(pf->function));
24
+ CLEAN_BUFFER *buffer = buffer_create(1024, NULL);
25
+ if(pf->payload && buffer_strlen(pf->payload)) {
26
+ buffer_sprintf(
27
+ buffer,
28
+ PLUGINSD_KEYWORD_FUNCTION_PAYLOAD " %s %d \"%s\" \"%s\" \"%s\"\n",
29
+ transaction,
30
+ pf->timeout_s,
31
+ string2str(pf->function),
32
+ pf->source ? pf->source : "",
33
+ content_type_id2string(pf->payload->content_type)
34
+ );
35
+
36
+ buffer_fast_strcat(buffer, buffer_tostring(pf->payload), buffer_strlen(pf->payload));
37
+ buffer_strcat(buffer, "\nFUNCTION_PAYLOAD_END\n");
38
+ }
39
+ else {
40
+ buffer_sprintf(
41
+ buffer,
42
+ PLUGINSD_KEYWORD_FUNCTION " %s %d \"%s\" \"%s\"\n",
43
+ transaction,
44
+ pf->timeout_s,
45
+ string2str(pf->function),
46
+ pf->source ? pf->source : ""
47
+ );
48
+ }
49
50
// send the command to the plugin
32
- ssize_t ret = send_to_plugin(buffer, parser);
33
-
51
+ // IMPORTANT: make sure all commands are sent in 1 call, because in streaming they may interfere with others
52
+ ssize_t ret = send_to_plugin(buffer_tostring(buffer), parser);
53
pf->sent_monotonic_ut = now_monotonic_usec();
54
55
if(ret < 0) {
56
+ pf->sent_successfully = false;
57
+
58
+ pf->code = HTTP_RESP_SERVICE_UNAVAILABLE;
59
netdata_log_error("FUNCTION '%s': failed to send it to the plugin, error %zd", string2str(pf->function), ret);
38
- rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE);
60
+ rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", pf->code);
61
}
62
else {
41
- internal_error(LOG_FUNCTIONS,
42
- "FUNCTION '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)",
43
- string2str(pf->function), dictionary_acquired_item_name(item), ret,
44
- pf->sent_monotonic_ut - pf->started_monotonic_ut);
45
- }
46
-
47
- if (!pf->payload)
48
- return;
63
+ pf->sent_successfully = true;
64
50
- // send the payload to the plugin
51
- ret = send_to_plugin(pf->payload, parser);
52
-
53
- if(ret < 0) {
54
- netdata_log_error("FUNCTION_PAYLOAD '%s': failed to send function to plugin, error %zd", string2str(pf->function), ret);
55
- rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE);
56
- }
57
- else {
65
internal_error(LOG_FUNCTIONS,
59
- "FUNCTION_PAYLOAD '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)",
60
- string2str(pf->function), dictionary_acquired_item_name(item), ret,
61
- pf->sent_monotonic_ut - pf->started_monotonic_ut);
66
+ "FUNCTION '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)",
67
+ string2str(pf->function), dictionary_acquired_item_name(item), ret,
68
+ pf->sent_monotonic_ut - pf->started_monotonic_ut);
69
}
63
-
64
- send_to_plugin("\nFUNCTION_PAYLOAD_END\n", parser);
70
}
71
72
static bool inflight_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused, void *new_func, void *parser_ptr __maybe_unused) {
@@ -75,83 +80,22 @@ static bool inflight_functions_conflict_callback(const DICTIONARY_ITEM *item __m
80
return false;
81
}
82
78
-static void delete_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug, const char *fnc_sig, int code) {
79
- if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED)
80
- return;
81
-
82
- char *params_local = strdupz(fnc_sig);
83
- char *words[DYNCFG_MAX_WORDS];
84
- size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd);
85
-
86
- if (words_c != 3) {
87
- netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for delete_job");
88
- freez(params_local);
89
- return;
90
- }
91
-
92
- const char *module = words[1];
93
- const char *job = words[2];
94
-
95
- delete_job(plug, module, job);
96
-
97
- unlink_job(plug->name, module, job);
98
-
99
- rrdpush_send_job_deleted(localhost, plug->name, module, job);
100
-
101
- freez(params_local);
102
-}
103
-
104
-static void set_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug __maybe_unused, const char *fnc_sig, int code) {
105
- if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED)
106
- return;
107
-
108
- char *params_local = strdupz(fnc_sig);
109
- char *words[DYNCFG_MAX_WORDS];
110
- size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd);
111
-
112
- if (words_c != 3) {
113
- netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for set_job_config");
114
- freez(params_local);
115
- return;
116
- }
117
-
118
- const char *module_name = get_word(words, words_c, 1);
119
- const char *job_name = get_word(words, words_c, 2);
120
-
121
- if (register_job(parser->user.host->configurable_plugins, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED, 1)) {
122
- freez(params_local);
123
- return;
124
- }
125
-
126
- // only send this if it is not existing already (register_job cares for that)
127
- rrdpush_send_dyncfg_reg_job(localhost, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED);
128
-
129
- freez(params_local);
130
-}
131
-
83
static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *parser_ptr) {
84
struct inflight_function *pf = func;
134
- struct parser *parser = (struct parser *)parser_ptr;
85
+ struct parser *parser = (struct parser *)parser_ptr; (void)parser;
86
87
internal_error(LOG_FUNCTIONS,
137
- "FUNCTION '%s' result of transaction '%s' received from collector (%zu bytes, request %"PRIu64" usec, response %"PRIu64" usec)",
138
- string2str(pf->function), dictionary_acquired_item_name(item),
139
- buffer_strlen(pf->result_body_wb), pf->sent_monotonic_ut - pf->started_monotonic_ut, now_realtime_usec() - pf->sent_monotonic_ut);
140
-
141
- if (pf->virtual && SERVING_PLUGINSD(parser)) {
142
- if (pf->payload) {
143
- if (strncmp(string2str(pf->function), FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0)
144
- set_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code);
145
- dyn_conf_store_config(string2str(pf->function), pf->payload, parser->user.cd->configuration);
146
- } else if (strncmp(string2str(pf->function), FUNCTION_NAME_DELETE_JOB, strlen(FUNCTION_NAME_DELETE_JOB)) == 0) {
147
- delete_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code);
148
- }
149
- }
88
+ "FUNCTION '%s' result of transaction '%s' received from collector "
89
+ "(%zu bytes, request %"PRIu64" usec, response %"PRIu64" usec)",
90
+ string2str(pf->function), dictionary_acquired_item_name(item),
91
+ buffer_strlen(pf->result_body_wb),
92
+ pf->sent_monotonic_ut - pf->started_monotonic_ut, now_realtime_usec() - pf->sent_monotonic_ut);
93
94
pf->result.cb(pf->result_body_wb, pf->code, pf->result.data);
95
96
string_freez(pf->function);
154
- freez((void *)pf->payload);
97
+ buffer_free((void *)pf->payload);
98
+ freez((void *)pf->source);
99
}
100
101
void pluginsd_inflight_functions_init(PARSER *parser) {
@@ -202,10 +146,8 @@ static void pluginsd_function_cancel(void *data) {
146
147
internal_error(true, "PLUGINSD: sending function cancellation to plugin for transaction '%s'", transaction);
148
205
- char buffer[2048 + 1];
206
- snprintfz(buffer, sizeof(buffer) - 1, "%s %s\n",
207
- PLUGINSD_KEYWORD_FUNCTION_CANCEL,
208
- transaction);
149
+ char buffer[2048];
150
+ snprintfz(buffer, sizeof(buffer), PLUGINSD_KEYWORD_FUNCTION_CANCEL " %s\n", transaction);
151
152
// send the command to the plugin
153
ssize_t ret = send_to_plugin(buffer, t->parser);
@@ -232,10 +174,8 @@ static void pluginsd_function_progress_to_plugin(void *data) {
174
175
internal_error(true, "PLUGINSD: sending function progress to plugin for transaction '%s'", transaction);
176
235
- char buffer[2048 + 1];
236
- snprintfz(buffer, sizeof(buffer) - 1, "%s %s\n",
237
- PLUGINSD_KEYWORD_FUNCTION_PROGRESS,
238
- transaction);
177
+ char buffer[2048];
178
+ snprintfz(buffer, sizeof(buffer), PLUGINSD_KEYWORD_FUNCTION_PROGRESS " %s\n", transaction);
179
180
// send the command to the plugin
181
ssize_t ret = send_to_plugin(buffer, t->parser);
@@ -254,42 +194,36 @@ static void pluginsd_function_progress_to_plugin(void *data) {
194
195
// this is the function called from
196
// rrd_call_function_and_wait() and rrd_call_function_async()
257
-static int pluginsd_function_execute_cb(uuid_t *transaction, BUFFER *result_body_wb,
258
- usec_t *stop_monotonic_ut, const char *function,
259
- void *execute_cb_data,
260
- rrd_function_result_callback_t result_cb, void *result_cb_data,
261
- rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
262
- rrd_function_is_cancelled_cb_t is_cancelled_cb __maybe_unused,
263
- void *is_cancelled_cb_data __maybe_unused,
264
- rrd_function_register_canceller_cb_t register_canceller_cb,
265
- void *register_canceller_cb_data,
266
- rrd_function_register_progresser_cb_t register_progresser_cb,
267
- void *register_progresser_cb_data) {
268
- PARSER *parser = execute_cb_data;
197
+int pluginsd_function_execute_cb(struct rrd_function_execute *rfe, void *data) {
198
+
199
+ // IMPORTANT: this function MUST call the result_cb even on failures
200
+
201
+ PARSER *parser = data;
202
203
usec_t now_ut = now_monotonic_usec();
204
272
- int timeout_s = (*stop_monotonic_ut - now_ut + USEC_PER_SEC / 2) / USEC_PER_SEC;
205
+ int timeout_s = (int)((*rfe->stop_monotonic_ut - now_ut + USEC_PER_SEC / 2) / USEC_PER_SEC);
206
207
struct inflight_function tmp = {
208
.started_monotonic_ut = now_ut,
276
- .stop_monotonic_ut = stop_monotonic_ut,
277
- .result_body_wb = result_body_wb,
209
+ .stop_monotonic_ut = rfe->stop_monotonic_ut,
210
+ .result_body_wb = rfe->result.wb,
211
.timeout_s = timeout_s,
279
- .function = string_strdupz(function),
280
- .payload = NULL,
212
+ .function = string_strdupz(rfe->function),
213
+ .payload = buffer_dup(rfe->payload),
214
+ .source = rfe->source ? strdupz(rfe->source) : NULL,
215
.parser = parser,
216
217
.result = {
284
- .cb = result_cb,
285
- .data = result_cb_data,
218
+ .cb = rfe->result.cb,
219
+ .data = rfe->result.data,
220
},
221
.progress = {
288
- .cb = progress_cb,
289
- .data = progress_cb_data,
222
+ .cb = rfe->progress.cb,
223
+ .data = rfe->progress.data,
224
},
225
};
292
- uuid_copy(tmp.transaction, *transaction);
226
+ uuid_copy(tmp.transaction, *rfe->transaction);
227
228
char transaction_str[UUID_COMPACT_STR_LEN];
229
uuid_unparse_lower_compact(tmp.transaction, transaction_str);
@@ -298,24 +232,35 @@ static int pluginsd_function_execute_cb(uuid_t *transaction, BUFFER *result_body
232
233
// if there is any error, our dictionary callbacks will call the caller callback to notify
234
// the caller about the error - no need for error handling here.
301
- void *t = dictionary_set(parser->inflight.functions, transaction_str, &tmp, sizeof(struct inflight_function));
302
- if(register_canceller_cb)
303
- register_canceller_cb(register_canceller_cb_data, pluginsd_function_cancel, t);
235
+ struct inflight_function *t = dictionary_set(parser->inflight.functions, transaction_str, &tmp, sizeof(struct inflight_function));
236
+ if(!t->sent_successfully) {
237
+ int code = t->code;
238
+ dictionary_write_unlock(parser->inflight.functions);
239
+ dictionary_del(parser->inflight.functions, transaction_str);
240
+ pluginsd_inflight_functions_garbage_collect(parser, now_ut);
241
+ return code;
242
+ }
243
+ else {
244
+ if (rfe->register_canceller.cb)
245
+ rfe->register_canceller.cb(rfe->register_canceller.data, pluginsd_function_cancel, t);
246
305
- if(register_progresser_cb && (parser->repertoire == PARSER_INIT_PLUGINSD ||
306
- (parser->repertoire == PARSER_INIT_STREAMING && stream_has_capability(&parser->user, STREAM_CAP_PROGRESS))))
307
- register_progresser_cb(register_progresser_cb_data, pluginsd_function_progress_to_plugin, t);
247
+ if (rfe->register_progresser.cb &&
248
+ (parser->repertoire == PARSER_INIT_PLUGINSD || (parser->repertoire == PARSER_INIT_STREAMING &&
249
+ stream_has_capability(&parser->user, STREAM_CAP_PROGRESS))))
250
+ rfe->register_progresser.cb(rfe->register_progresser.data, pluginsd_function_progress_to_plugin, t);
251
309
- if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut)
310
- parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
252
+ if (!parser->inflight.smaller_monotonic_timeout_ut ||
253
+ *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut)
254
+ parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
255
312
- // garbage collect stale inflight functions
313
- if(parser->inflight.smaller_monotonic_timeout_ut < now_ut)
314
- pluginsd_inflight_functions_garbage_collect(parser, now_ut);
256
+ // garbage collect stale inflight functions
257
+ if (parser->inflight.smaller_monotonic_timeout_ut < now_ut)
258
+ pluginsd_inflight_functions_garbage_collect(parser, now_ut);
259
316
- dictionary_write_unlock(parser->inflight.functions);
260
+ dictionary_write_unlock(parser->inflight.functions);
261
318
- return HTTP_RESP_OK;
262
+ return HTTP_RESP_OK;
263
+ }
264
}
265
266
PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser) {
@@ -421,7 +366,7 @@ PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, PARSER
366
struct inflight_function *pf = inflight_function_find(parser, transaction);
367
if(pf) {
368
if(format && *format)
424
- pf->result_body_wb->content_type = functions_format_to_content_type(format);
369
+ pf->result_body_wb->content_type = content_type_string2id(format);
370
371
pf->code = code;
372
collectors/plugins.d/pluginsd_functions.h
+8
-6
@@ -11,13 +11,17 @@ struct inflight_function {
11
int code;
12
int timeout_s;
13
STRING *function;
14
+ BUFFER *payload;
15
+ const char *source;
16
+
17
BUFFER *result_body_wb;
18
+
19
usec_t *stop_monotonic_ut; // pointer to caller data
20
usec_t started_monotonic_ut;
21
usec_t sent_monotonic_ut;
18
- const char *payload;
22
PARSER *parser;
20
- bool virtual;
23
+
24
+ bool sent_successfully;
25
26
struct {
27
rrd_function_result_callback_t cb;
@@ -28,10 +32,6 @@ struct inflight_function {
32
rrd_function_progress_cb_t cb;
33
void *data;
34
} progress;
31
-
32
- struct {
33
- usec_t stop_monotonic_ut;
34
- } dyncfg;
35
};
36
37
PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser);
@@ -42,4 +42,6 @@ void pluginsd_inflight_functions_init(PARSER *parser);
42
void pluginsd_inflight_functions_cleanup(PARSER *parser);
43
void pluginsd_inflight_functions_garbage_collect(PARSER *parser, usec_t now_ut);
44
45
+int pluginsd_function_execute_cb(struct rrd_function_execute *rfe, void *data);
46
+
47
#endif //NETDATA_PLUGINSD_FUNCTIONS_H
collectors/plugins.d/pluginsd_internals.c
-1
@@ -94,7 +94,6 @@ void parser_destroy(PARSER *parser) {
94
if (unlikely(!parser))
95
return;
96
97
- pluginsd_dyncfg_cleanup(parser);
97
pluginsd_inflight_functions_cleanup(parser);
98
99
freez(parser);
collectors/plugins.d/pluginsd_parser.c
+10
-16
@@ -202,6 +202,7 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si
202
false);
203
204
rrdhost_option_set(host, RRDHOST_OPTION_VIRTUAL_HOST);
205
+ dyncfg_host_init(host);
206
207
if(host->rrdlabels) {
208
rrdlabels_migrate_to_these(host->rrdlabels, parser->user.host_define.rrdlabels);
@@ -1355,23 +1356,16 @@ PARSER_RC parser_execute(PARSER *parser, PARSER_KEYWORD *keyword, char **words,
1356
case 99:
1357
return pluginsd_exit(words, num_words, parser);
1358
1358
- case 101:
1359
- return pluginsd_register_plugin(words, num_words, parser);
1359
+ case 100:
1360
+ return pluginsd_config(words, num_words, parser);
1361
1361
- case 102:
1362
- return pluginsd_register_module(words, num_words, parser);
1363
-
1364
- case 103:
1365
- return pluginsd_register_job(words, num_words, parser);
1366
-
1367
- case 104:
1368
- return pluginsd_dyncfg_reset(words, num_words, parser);
1369
-
1370
- case 110:
1371
- return pluginsd_job_status(words, num_words, parser);
1372
-
1373
- case 111:
1374
- return pluginsd_delete_job(words, num_words, parser);
1362
+ case 901:
1363
+ case 902:
1364
+ case 903:
1365
+ case 904:
1366
+ case 905:
1367
+ case 906:
1368
+ return pluginsd_dyncfg_noop(words, num_words, parser);
1369
1370
default:
1371
break;
collectors/plugins.d/pluginsd_parser.h
+1
-1
@@ -231,7 +231,7 @@ static inline int parser_action(PARSER *parser, char *input) {
231
rc = PARSER_RC_ERROR;
232
233
if(rc == PARSER_RC_ERROR) {
234
- CLEAN_BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
234
+ CLEAN_BUFFER *wb = buffer_create(1024, NULL);
235
line_splitter_reconstruct_line(wb, &parser->line);
236
netdata_log_error("PLUGINSD: parser_action('%s') failed on line %zu: { %s } (quotes added to show parsing)",
237
command, parser->line.count, buffer_tostring(wb));
collectors/proc.plugin/proc_diskstats.c
+6
-29
@@ -1033,17 +1033,7 @@ static void add_labels_to_disk(struct disk *d, RRDSET *st) {
1033
rrdlabels_add(st->rrdlabels, "device_type", get_disk_type_string(d->type), RRDLABEL_SRC_AUTO);
1034
}
1035
1036
-static int diskstats_function_block_devices(uuid_t *transaction __maybe_unused, BUFFER *wb,
1037
- usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
1038
- void *collector_data __maybe_unused,
1039
- rrd_function_result_callback_t result_cb, void *result_cb_data,
1040
- rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
1041
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
1042
- rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
1043
- void *register_canceller_cb_data __maybe_unused,
1044
- rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
1045
- void *register_progresser_cb_data __maybe_unused) {
1046
-
1036
+static int diskstats_function_block_devices(BUFFER *wb, const char *function __maybe_unused) {
1037
buffer_flush(wb);
1038
wb->content_type = CT_APPLICATION_JSON;
1039
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
@@ -1322,16 +1312,7 @@ static int diskstats_function_block_devices(uuid_t *transaction __maybe_unused,
1312
buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + 1);
1313
buffer_json_finalize(wb);
1314
1325
- int response = HTTP_RESP_OK;
1326
- if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) {
1327
- buffer_flush(wb);
1328
- response = HTTP_RESP_CLIENT_CLOSED_REQUEST;
1329
- }
1330
-
1331
- if(result_cb)
1332
- result_cb(wb, response, result_cb_data);
1333
-
1334
- return response;
1315
+ return HTTP_RESP_OK;
1316
}
1317
1318
static void diskstats_cleanup_disks() {
@@ -1478,6 +1459,10 @@ int do_proc_diskstats(int update_every, usec_t dt) {
1459
excluded_disks = simple_pattern_create(
1460
config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "exclude disks", DEFAULT_EXCLUDED_DISKS), NULL,
1461
SIMPLE_PATTERN_EXACT, true);
1462
+
1463
+ rrd_function_add_inline(localhost, NULL, "block-devices", 10,
1464
+ RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSTATS_HELP,
1465
+ "top", HTTP_ACCESS_ANY, diskstats_function_block_devices);
1466
}
1467
1468
// --------------------------------------------------------------------------
@@ -1492,14 +1477,6 @@ int do_proc_diskstats(int update_every, usec_t dt) {
1477
ff = procfile_readall(ff);
1478
if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time
1479
1495
- static bool add_func = true;
1496
- if (add_func) {
1497
- rrd_function_add(localhost, NULL, "block-devices", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSTATS_HELP,
1498
- "top", HTTP_ACCESS_ANY, true,
1499
- diskstats_function_block_devices, NULL);
1500
- add_func = false;
1501
- }
1502
-
1480
size_t lines = procfile_lines(ff), l;
1481
1482
collected_number system_read_kb = 0, system_write_kb = 0;
collectors/proc.plugin/proc_net_dev.c
+6
-18
@@ -473,17 +473,7 @@ static void netdev_rename_this_device(struct netdev *d) {
473
474
// ----------------------------------------------------------------------------
475
476
-int netdev_function_net_interfaces(uuid_t *transaction __maybe_unused, BUFFER *wb,
477
- usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
478
- void *collector_data __maybe_unused,
479
- rrd_function_result_callback_t result_cb, void *result_cb_data,
480
- rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
481
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
482
- rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
483
- void *register_canceller_cb_data __maybe_unused,
484
- rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
485
- void *register_progresser_cb_data __maybe_unused) {
486
-
476
+int netdev_function_net_interfaces(BUFFER *wb, const char *function __maybe_unused) {
477
buffer_flush(wb);
478
wb->content_type = CT_APPLICATION_JSON;
479
buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
@@ -1761,16 +1751,14 @@ void *netdev_main(void *ptr)
1751
worker_register("NETDEV");
1752
worker_register_job_name(0, "netdev");
1753
1764
- if (getenv("KUBERNETES_SERVICE_HOST") != NULL && getenv("KUBERNETES_SERVICE_PORT") != NULL) {
1754
+ if (getenv("KUBERNETES_SERVICE_HOST") != NULL && getenv("KUBERNETES_SERVICE_PORT") != NULL)
1755
double_linked_device_collect_delay_secs = 300;
1766
- }
1756
1768
- netdata_thread_cleanup_push(netdev_main_cleanup, ptr) {
1769
- rrd_collector_started();
1770
- rrd_function_add(localhost, NULL, "network-interfaces", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_NETDEV_HELP,
1771
- "top", HTTP_ACCESS_ANY,
1772
- true, netdev_function_net_interfaces, NULL);
1757
+ rrd_function_add_inline(localhost, NULL, "network-interfaces", 10,
1758
+ RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_NETDEV_HELP,
1759
+ "top", HTTP_ACCESS_ANY, netdev_function_net_interfaces);
1760
1761
+ netdata_thread_cleanup_push(netdev_main_cleanup, ptr) {
1762
usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
1763
heartbeat_t hb;
1764
heartbeat_init(&hb);
collectors/systemd-journal.plugin/systemd-internals.h
+9
-6
@@ -83,8 +83,8 @@ struct journal_file {
83
84
#define ND_SD_JOURNAL_OPEN_FLAGS (0)
85
86
-#define JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT (5 * USEC_PER_SEC) // assume always 5 seconds latency
87
-#define JOURNAL_VS_REALTIME_DELTA_MAX_UT (2 * 60 * USEC_PER_SEC) // up to 2 minutes latency
86
+#define JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT (5 * USEC_PER_SEC) // assume a 5-seconds latency
87
+#define JOURNAL_VS_REALTIME_DELTA_MAX_UT (2 * 60 * USEC_PER_SEC) // up to 2-minutes latency
88
89
extern DICTIONARY *journal_files_registry;
90
extern DICTIONARY *used_hashes_registry;
@@ -114,21 +114,22 @@ usec_t journal_file_update_annotation_boot_id(sd_journal *j, struct journal_file
114
115
#define MAX_JOURNAL_DIRECTORIES 100
116
struct journal_directory {
117
- char *path;
117
+ STRING *path;
118
};
119
extern struct journal_directory journal_directories[MAX_JOURNAL_DIRECTORIES];
120
121
void journal_init_files_and_directories(void);
122
-void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled);
122
+void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled, BUFFER *payload, const char *source, void *data);
123
void journal_file_update_header(const char *filename, struct journal_file *jf);
124
125
void netdata_systemd_journal_message_ids_init(void);
126
-void netdata_systemd_journal_transform_message_id(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused);
126
+void netdata_systemd_journal_transform_message_id(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
127
128
void *journal_watcher_main(void *arg);
129
+void journal_watcher_restart(void);
130
131
#ifdef ENABLE_SYSTEMD_DBUS
131
-void function_systemd_units(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled);
132
+void function_systemd_units(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled, BUFFER *payload, const char *source, void *data);
133
#endif
134
135
static inline void send_newline_and_flush(void) {
@@ -157,4 +158,6 @@ static inline bool parse_journal_field(const char *data, size_t data_length, con
158
return true;
159
}
160
161
+void systemd_journal_dyncfg_init(struct functions_evloop_globals *wg);
162
+
163
#endif //NETDATA_COLLECTORS_SYSTEMD_INTERNALS_H
collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
new
+98
@@ -0,0 +1,98 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "systemd-internals.h"
4
+
5
+#define JOURNAL_DIRECTORIES_JSON_NODE "journalDirectories"
6
+
7
+static int systemd_journal_directories_dyncfg_update(BUFFER *result, BUFFER *payload) {
8
+ if(!payload || !buffer_strlen(payload))
9
+ return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "empty payload received");
10
+
11
+ CLEAN_JSON_OBJECT *jobj = json_tokener_parse(buffer_tostring(payload));
12
+ if(!jobj)
13
+ return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "cannot parse json payload");
14
+
15
+ struct json_object *journalDirectories;
16
+ json_object_object_get_ex(jobj, JOURNAL_DIRECTORIES_JSON_NODE, &journalDirectories);
17
+
18
+ size_t n_directories = json_object_array_length(journalDirectories);
19
+
20
+ size_t added = 0;
21
+ for(size_t i = 0; i < n_directories; i++) {
22
+ struct json_object *dir = json_object_array_get_idx(journalDirectories, i);
23
+ const char *s = json_object_get_string(dir);
24
+ if(s && *s) {
25
+ string_freez(journal_directories[added].path);
26
+ journal_directories[added++].path = string_strdupz(s);
27
+ }
28
+ }
29
+
30
+ if(!added)
31
+ return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "no directories in the payload");
32
+ else {
33
+ for(size_t i = added; i < MAX_JOURNAL_DIRECTORIES; i++) {
34
+ string_freez(journal_directories[i].path);
35
+ journal_directories[i].path = NULL;
36
+ }
37
+ }
38
+
39
+ return dyncfg_default_response(result, HTTP_RESP_OK, "applied");
40
+}
41
+
42
+static int systemd_journal_directories_dyncfg_get(BUFFER *wb) {
43
+ buffer_flush(wb);
44
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
45
+
46
+ buffer_json_member_add_array(wb, JOURNAL_DIRECTORIES_JSON_NODE);
47
+ for(size_t i = 0; i < MAX_JOURNAL_DIRECTORIES ;i++) {
48
+ if(!journal_directories[i].path)
49
+ break;
50
+
51
+ buffer_json_add_array_item_string(wb, string2str(journal_directories[i].path));
52
+ }
53
+ buffer_json_array_close(wb);
54
+
55
+ buffer_json_finalize(wb);
56
+ return HTTP_RESP_OK;
57
+}
58
+
59
+static int systemd_journal_directories_dyncfg_cb(const char *transaction,
60
+ const char *id,
61
+ DYNCFG_CMDS cmd,
62
+ BUFFER *payload,
63
+ usec_t *stop_monotonic_ut __maybe_unused,
64
+ bool *cancelled __maybe_unused,
65
+ BUFFER *result,
66
+ const char *source __maybe_unused,
67
+ void *data __maybe_unused) {
68
+ CLEAN_BUFFER *action = buffer_create(100, NULL);
69
+ dyncfg_cmds2buffer(cmd, action);
70
+
71
+ if(cmd == DYNCFG_CMD_GET)
72
+ return systemd_journal_directories_dyncfg_get(result);
73
+
74
+ if(cmd == DYNCFG_CMD_UPDATE)
75
+ return systemd_journal_directories_dyncfg_update(result, payload);
76
+
77
+ nd_log(NDLS_COLLECTORS, NDLP_ERR,
78
+ "DYNCFG: unhandled transaction '%s', id '%s' cmd '%s', payload: %s",
79
+ transaction, id, buffer_tostring(action), payload ? buffer_tostring(payload) : "");
80
+
81
+ return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "the command is not handled by this plugin");
82
+}
83
+
84
+// ----------------------------------------------------------------------------
85
+
86
+void systemd_journal_dyncfg_init(struct functions_evloop_globals *wg) {
87
+ functions_evloop_dyncfg_add(
88
+ wg,
89
+ "systemd-journal:monitored-directories",
90
+ "/collectors/logs/systemd-journal",
91
+ DYNCFG_STATUS_RUNNING,
92
+ DYNCFG_TYPE_SINGLE,
93
+ DYNCFG_SOURCE_TYPE_INTERNAL,
94
+ "internal",
95
+ DYNCFG_CMD_SCHEMA | DYNCFG_CMD_GET | DYNCFG_CMD_UPDATE,
96
+ systemd_journal_directories_dyncfg_cb,
97
+ NULL);
98
+}
collectors/systemd-journal.plugin/systemd-journal-files.c
+5
-5
@@ -715,7 +715,7 @@ void journal_files_registry_update(void) {
715
716
for(unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES; i++) {
717
if(!journal_directories[i].path) break;
718
- journal_directory_scan_recursively(files, dirs, journal_directories[i].path, 0);
718
+ journal_directory_scan_recursively(files, dirs, string2str(journal_directories[i].path), 0);
719
}
720
721
const char **array = mallocz(sizeof(const char *) * dictionary_entries(files));
@@ -819,15 +819,15 @@ void journal_init_files_and_directories(void) {
819
// ------------------------------------------------------------------------
820
// setup the journal directories
821
822
- journal_directories[d++].path = strdupz("/run/log/journal");
823
- journal_directories[d++].path = strdupz("/var/log/journal");
822
+ journal_directories[d++].path = string_strdupz("/run/log/journal");
823
+ journal_directories[d++].path = string_strdupz("/var/log/journal");
824
825
if(*netdata_configured_host_prefix) {
826
char path[PATH_MAX];
827
snprintfz(path, sizeof(path), "%s/var/log/journal", netdata_configured_host_prefix);
828
- journal_directories[d++].path = strdupz(path);
828
+ journal_directories[d++].path = string_strdupz(path);
829
snprintfz(path, sizeof(path), "%s/run/log/journal", netdata_configured_host_prefix);
830
- journal_directories[d++].path = strdupz(path);
830
+ journal_directories[d++].path = string_strdupz(path);
831
}
832
833
// terminate the list
collectors/systemd-journal.plugin/systemd-journal-watcher.c
+11
-3
@@ -292,8 +292,16 @@ static void process_pending(Watcher *watcher) {
292
dictionary_garbage_collect(watcher->pending);
293
}
294
295
+size_t journal_watcher_wanted_session_id = 0;
296
+
297
+void journal_watcher_restart(void) {
298
+ __atomic_add_fetch(&journal_watcher_wanted_session_id, 1, __ATOMIC_RELAXED);
299
+}
300
+
301
void *journal_watcher_main(void *arg __maybe_unused) {
302
while(1) {
303
+ size_t journal_watcher_session_id = journal_watcher_wanted_session_id;
304
+
305
Watcher watcher = {
306
.watchList = mallocz(INITIAL_WATCHES * sizeof(WatchEntry)),
307
.freeList = NULL,
@@ -312,12 +320,12 @@ void *journal_watcher_main(void *arg __maybe_unused) {
320
321
for (unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES; i++) {
322
if (!journal_directories[i].path) break;
315
- watch_directory_and_subdirectories(&watcher, inotifyFd, journal_directories[i].path);
323
+ watch_directory_and_subdirectories(&watcher, inotifyFd, string2str(journal_directories[i].path));
324
}
325
326
usec_t last_headers_update_ut = now_monotonic_usec();
327
struct buffered_reader reader;
320
- while (1) {
328
+ while (journal_watcher_session_id == __atomic_load_n(&journal_watcher_wanted_session_id, __ATOMIC_RELAXED)) {
329
buffered_reader_ret_t rc = buffered_reader_read_timeout(
330
&reader, inotifyFd, SYSTEMD_JOURNAL_EXECUTE_WATCHER_PENDING_EVERY_MS, false);
331
@@ -372,7 +380,7 @@ void *journal_watcher_main(void *arg __maybe_unused) {
380
// this will scan the directories and cleanup the registry
381
journal_files_registry_update();
382
375
- sleep_usec(5 * USEC_PER_SEC);
383
+ sleep_usec(2 * USEC_PER_SEC);
384
}
385
386
return NULL;
collectors/systemd-journal.plugin/systemd-journal.c
+2
-1
@@ -1520,7 +1520,8 @@ static void netdata_systemd_journal_function_help(const char *transaction) {
1520
buffer_free(wb);
1521
}
1522
1523
-void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled) {
1523
+void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled,
1524
+ BUFFER *payload __maybe_unused, const char *source __maybe_unused, void *data __maybe_unused) {
1525
fstat_thread_calls = 0;
1526
fstat_thread_cached_responses = 0;
1527
collectors/systemd-journal.plugin/systemd-main.c
+16
-8
@@ -11,7 +11,7 @@ static bool plugin_should_exit = false;
11
static bool journal_data_direcories_exist() {
12
struct stat st;
13
for (unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES && journal_directories[i].path; i++) {
14
- if ((stat(journal_directories[i].path, &st) == 0) && S_ISDIR(st.st_mode))
14
+ if ((stat(string2str(journal_directories[i].path), &st) == 0) && S_ISDIR(st.st_mode))
15
return true;
16
}
17
return false;
@@ -49,7 +49,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
49
char buf[] = "systemd-journal after:-8640000 before:0 direction:backward last:200 data_only:false slice:true source:all";
50
// char buf[] = "systemd-journal after:1695332964 before:1695937764 direction:backward last:100 slice:true source:all DHKucpqUoe1:PtVoyIuX.MU";
51
// char buf[] = "systemd-journal after:1694511062 before:1694514662 anchor:1694514122024403";
52
- function_systemd_journal("123", buf, &stop_monotonic_ut, &cancelled);
52
+ function_systemd_journal("123", buf, &stop_monotonic_ut, &cancelled, NULL, NULL, NULL);
53
// function_systemd_units("123", "systemd-units", 600, &cancelled);
54
exit(1);
55
}
@@ -57,7 +57,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
57
if(argc == 2 && strcmp(argv[1], "debug-units") == 0) {
58
bool cancelled = false;
59
usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
60
- function_systemd_units("123", "systemd-units", &stop_monotonic_ut, &cancelled);
60
+ function_systemd_units("123", "systemd-units", &stop_monotonic_ut, &cancelled, NULL, NULL, NULL);
61
exit(1);
62
}
63
#endif
@@ -75,14 +75,22 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
75
struct functions_evloop_globals *wg =
76
functions_evloop_init(SYSTEMD_JOURNAL_WORKER_THREADS, "SDJ", &stdout_mutex, &plugin_should_exit);
77
78
- functions_evloop_add_function(wg, SYSTEMD_JOURNAL_FUNCTION_NAME, function_systemd_journal,
79
- SYSTEMD_JOURNAL_DEFAULT_TIMEOUT);
78
+ functions_evloop_add_function(wg,
79
+ SYSTEMD_JOURNAL_FUNCTION_NAME,
80
+ function_systemd_journal,
81
+ SYSTEMD_JOURNAL_DEFAULT_TIMEOUT,
82
+ NULL);
83
84
#ifdef ENABLE_SYSTEMD_DBUS
82
- functions_evloop_add_function(wg, SYSTEMD_UNITS_FUNCTION_NAME, function_systemd_units,
83
- SYSTEMD_UNITS_DEFAULT_TIMEOUT);
85
+ functions_evloop_add_function(wg,
86
+ SYSTEMD_UNITS_FUNCTION_NAME,
87
+ function_systemd_units,
88
+ SYSTEMD_UNITS_DEFAULT_TIMEOUT,
89
+ NULL);
90
#endif
91
92
+ systemd_journal_dyncfg_init(wg);
93
+
94
// ------------------------------------------------------------------------
95
// register functions to netdata
96
@@ -106,7 +114,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
114
usec_t step_ut = 100 * USEC_PER_MS;
115
usec_t send_newline_ut = 0;
116
usec_t since_last_scan_ut = SYSTEMD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
109
- bool tty = isatty(fileno(stderr)) == 1;
117
+ bool tty = isatty(fileno(stdout)) == 1;
118
119
heartbeat_t hb;
120
heartbeat_init(&hb);
collectors/systemd-journal.plugin/systemd-units.c
+3
-1
@@ -1596,7 +1596,9 @@ void systemd_units_assign_priority(UnitInfo *base) {
1596
}
1597
}
1598
1599
-void function_systemd_units(const char *transaction, char *function, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) {
1599
+void function_systemd_units(const char *transaction, char *function,
1600
+ usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused,
1601
+ BUFFER *payload __maybe_unused, const char *source __maybe_unused, void *data __maybe_unused) {
1602
char *words[SYSTEMD_UNITS_MAX_PARAMS] = { NULL };
1603
size_t num_words = quoted_strings_splitter_pluginsd(function, words, SYSTEMD_UNITS_MAX_PARAMS);
1604
for(int i = 1; i < SYSTEMD_UNITS_MAX_PARAMS ;i++) {
daemon/common.h
+2
@@ -34,6 +34,8 @@
34
// ----------------------------------------------------------------------------
35
// netdata include files
36
37
+#include "daemon/config/dyncfg.h"
38
+
39
#include "global_statistics.h"
40
41
// the netdata database
daemon/config/dyncfg-echo.c
new
+83
@@ -0,0 +1,83 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "dyncfg-internals.h"
4
+#include "dyncfg.h"
5
+
6
+// ----------------------------------------------------------------------------
7
+// echo is when we send requests to plugins without any caller
8
+// it is used for:
9
+// 1. the first enable/disable requests we send, and also
10
+// 2. updates to stock or user configurations
11
+// 3. saved dynamic jobs we need to add to templates
12
+
13
+struct dyncfg_echo {
14
+ const DICTIONARY_ITEM *item;
15
+ DYNCFG *df;
16
+ BUFFER *wb;
17
+};
18
+
19
+void dyncfg_echo_cb(BUFFER *wb __maybe_unused, int code, void *result_cb_data) {
20
+ struct dyncfg_echo *e = result_cb_data;
21
+
22
+ buffer_free(e->wb);
23
+ dictionary_acquired_item_release(dyncfg_globals.nodes, e->item);
24
+
25
+ e->wb = NULL;
26
+ e->df = NULL;
27
+ e->item = NULL;
28
+ freez(e);
29
+}
30
+
31
+void dyncfg_echo(const DICTIONARY_ITEM *item, DYNCFG *df, const char *id __maybe_unused, DYNCFG_CMDS cmd) {
32
+ if(!(df->cmds & cmd))
33
+ return;
34
+
35
+ const char *cmd_str = dyncfg_id2cmd_one(cmd);
36
+ if(!cmd_str) {
37
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: command given does not resolve to a known command");
38
+ return;
39
+ }
40
+
41
+ struct dyncfg_echo *e = callocz(1, sizeof(struct dyncfg_echo));
42
+ e->item = dictionary_acquired_item_dup(dyncfg_globals.nodes, item);
43
+ e->wb = buffer_create(0, NULL);
44
+ e->df = df;
45
+
46
+ char buf[string_strlen(df->function) + strlen(cmd_str) + 20];
47
+ snprintfz(buf, sizeof(buf), "%s %s", string2str(df->function), cmd_str);
48
+
49
+ rrd_function_run(df->host, e->wb, 10, HTTP_ACCESS_ADMIN, buf, false, NULL,
50
+ dyncfg_echo_cb, e,
51
+ NULL, NULL,
52
+ NULL, NULL,
53
+ NULL, NULL);
54
+}
55
+
56
+static void dyncfg_echo_payload(const DICTIONARY_ITEM *item, DYNCFG *df, const char *id __maybe_unused, const char *cmd) {
57
+ if(!df->payload)
58
+ return;
59
+
60
+ struct dyncfg_echo *e = callocz(1, sizeof(struct dyncfg_echo));
61
+ e->item = dictionary_acquired_item_dup(dyncfg_globals.nodes, item);
62
+ e->wb = buffer_create(0, NULL);
63
+ e->df = df;
64
+
65
+ char buf[string_strlen(df->function) + strlen(cmd) + 20];
66
+ snprintfz(buf, sizeof(buf), "%s %s", string2str(df->function), cmd);
67
+
68
+ rrd_function_run(df->host, e->wb, 10, HTTP_ACCESS_ADMIN, buf, false, NULL,
69
+ dyncfg_echo_cb, e,
70
+ NULL, NULL,
71
+ NULL, NULL,
72
+ df->payload, NULL);
73
+}
74
+
75
+void dyncfg_echo_update(const DICTIONARY_ITEM *item, DYNCFG *df, const char *id) {
76
+ dyncfg_echo_payload(item, df, id, "update");
77
+}
78
+
79
+void dyncfg_echo_add(const DICTIONARY_ITEM *template_item, DYNCFG *template_df, const char *template_id, const char *job_name) {
80
+ char buf[strlen(job_name) + 20];
81
+ snprintfz(buf, sizeof(buf), "add %s", job_name);
82
+ dyncfg_echo_payload(template_item, template_df, template_id, buf);
83
+}
daemon/config/dyncfg-files.c
new
+223
@@ -0,0 +1,223 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "dyncfg-internals.h"
4
+#include "dyncfg.h"
5
+
6
+void dyncfg_file_delete(const char *id) {
7
+ CLEAN_CHAR_P *escaped_id = dyncfg_escape_id_for_filename(id);
8
+ char filename[FILENAME_MAX];
9
+ snprintfz(filename, sizeof(filename), "%s/%s.dyncfg", dyncfg_globals.dir, escaped_id);
10
+ unlink(filename);
11
+}
12
+
13
+void dyncfg_file_save(const char *id, DYNCFG *df) {
14
+ CLEAN_CHAR_P *escaped_id = dyncfg_escape_id_for_filename(id);
15
+ char filename[FILENAME_MAX];
16
+ snprintfz(filename, sizeof(filename), "%s/%s.dyncfg", dyncfg_globals.dir, escaped_id);
17
+
18
+ FILE *fp = fopen(filename, "w");
19
+ if(!fp) {
20
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: cannot create file '%s'", filename);
21
+ return;
22
+ }
23
+
24
+ df->modified_ut = now_realtime_usec();
25
+
26
+ fprintf(fp, "version=%zu\n", DYNCFG_VERSION);
27
+ fprintf(fp, "id=%s\n", id);
28
+
29
+ if(df->template)
30
+ fprintf(fp, "template=%s\n", string2str(df->template));
31
+
32
+ char uuid_str[UUID_COMPACT_STR_LEN];
33
+ uuid_unparse_lower_compact(df->host_uuid, uuid_str);
34
+ fprintf(fp, "host=%s\n", uuid_str);
35
+
36
+ fprintf(fp, "path=%s\n", string2str(df->path));
37
+ fprintf(fp, "type=%s\n", dyncfg_id2type(df->type));
38
+
39
+ fprintf(fp, "source_type=%s\n", dyncfg_id2source_type(df->source_type));
40
+ fprintf(fp, "source=%s\n", string2str(df->source));
41
+
42
+ fprintf(fp, "created=%"PRIu64"\n", df->created_ut);
43
+ fprintf(fp, "modified=%"PRIu64"\n", df->modified_ut);
44
+ fprintf(fp, "sync=%s\n", df->sync ? "true" : "false");
45
+ fprintf(fp, "user_disabled=%s\n", df->user_disabled ? "true" : "false");
46
+ fprintf(fp, "saves=%"PRIu32"\n", ++df->saves);
47
+
48
+ fprintf(fp, "cmds=");
49
+ dyncfg_cmds2fp(df->cmds, fp);
50
+ fprintf(fp, "\n");
51
+
52
+ if(df->payload && buffer_strlen(df->payload) > 0) {
53
+ fprintf(fp, "content_type=%s\n", content_type_id2string(df->payload->content_type));
54
+ fprintf(fp, "content_length=%zu\n", buffer_strlen(df->payload));
55
+ fprintf(fp, "---\n");
56
+ fwrite(buffer_tostring(df->payload), 1, buffer_strlen(df->payload), fp);
57
+ }
58
+
59
+ fclose(fp);
60
+}
61
+
62
+void dyncfg_file_load(const char *filename) {
63
+ FILE *fp = fopen(filename, "r");
64
+ if (!fp) {
65
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: cannot open file '%s'", filename);
66
+ return;
67
+ }
68
+
69
+ DYNCFG tmp = {
70
+ .host = NULL,
71
+ .status = DYNCFG_STATUS_ORPHAN,
72
+ };
73
+
74
+ char line[PLUGINSD_LINE_MAX];
75
+ CLEAN_CHAR_P *id = NULL;
76
+
77
+ HTTP_CONTENT_TYPE content_type = CT_NONE;
78
+ size_t content_length = 0;
79
+ bool read_payload = false;
80
+
81
+ while (fgets(line, sizeof(line), fp)) {
82
+ if(strcmp(line, "---\n") == 0) {
83
+ read_payload = true;
84
+ break;
85
+ }
86
+
87
+ char *value = strchr(line, '=');
88
+ if(!value) continue;
89
+
90
+ *value++ = '\0';
91
+
92
+ value = trim(value);
93
+ if(!value) continue;
94
+
95
+ char *key = trim(line);
96
+ if(!key) continue;
97
+
98
+ // Parse key-value pairs
99
+ if (strcmp(key, "version") == 0) {
100
+ size_t version = strtoull(value, NULL, 10);
101
+
102
+ if(version > DYNCFG_VERSION)
103
+ nd_log(NDLS_DAEMON, NDLP_NOTICE,
104
+ "DYNCFG: configuration file '%s' has version %zu, which is newer than our version %zu",
105
+ filename, version, DYNCFG_VERSION);
106
+
107
+ } else if (strcmp(key, "id") == 0) {
108
+ freez(id);
109
+ id = strdupz(value);
110
+ } else if (strcmp(key, "template") == 0) {
111
+ tmp.template = string_strdupz(value);
112
+ } else if (strcmp(key, "host") == 0) {
113
+ uuid_parse_flexi(value, tmp.host_uuid);
114
+ } else if (strcmp(key, "path") == 0) {
115
+ tmp.path = string_strdupz(value);
116
+ } else if (strcmp(key, "type") == 0) {
117
+ tmp.type = dyncfg_type2id(value);
118
+ } else if (strcmp(key, "source_type") == 0) {
119
+ tmp.source_type = dyncfg_source_type2id(value);
120
+ } else if (strcmp(key, "source") == 0) {
121
+ tmp.source = string_strdupz(value);
122
+ } else if (strcmp(key, "created") == 0) {
123
+ tmp.created_ut = strtoull(value, NULL, 10);
124
+ } else if (strcmp(key, "modified") == 0) {
125
+ tmp.modified_ut = strtoull(value, NULL, 10);
126
+ } else if (strcmp(key, "sync") == 0) {
127
+ tmp.sync = (strcmp(value, "true") == 0);
128
+ } else if (strcmp(key, "user_disabled") == 0) {
129
+ tmp.user_disabled = (strcmp(value, "true") == 0);
130
+ } else if (strcmp(key, "saves") == 0) {
131
+ tmp.saves = strtoull(value, NULL, 10);
132
+ } else if (strcmp(key, "content_type") == 0) {
133
+ content_type = content_type_string2id(value);
134
+ } else if (strcmp(key, "content_length") == 0) {
135
+ content_length = strtoull(value, NULL, 10);
136
+ } else if (strcmp(key, "cmds") == 0) {
137
+ tmp.cmds = dyncfg_cmds2id(value);
138
+ }
139
+ }
140
+
141
+ if(read_payload && content_length) {
142
+ tmp.payload = buffer_create(content_length, NULL);
143
+ tmp.payload->content_type = content_type;
144
+
145
+ buffer_need_bytes(tmp.payload, content_length);
146
+ tmp.payload->len = fread(tmp.payload->buffer, 1, content_length, fp);
147
+ }
148
+
149
+ fclose(fp);
150
+
151
+ if(!id) {
152
+ nd_log(NDLS_DAEMON, NDLP_ERR,
153
+ "DYNCFG: configuration file '%s' does not include a unique id. Ignoring it.",
154
+ filename);
155
+
156
+ dyncfg_cleanup(&tmp);
157
+ return;
158
+ }
159
+
160
+ dictionary_set(dyncfg_globals.nodes, id, &tmp, sizeof(tmp));
161
+}
162
+
163
+void dyncfg_load_all(void) {
164
+ DIR *dir = opendir(dyncfg_globals.dir);
165
+ if (!dir) {
166
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: cannot open directory '%s'", dyncfg_globals.dir);
167
+ return;
168
+ }
169
+
170
+ struct dirent *entry;
171
+ char filepath[PATH_MAX];
172
+ while ((entry = readdir(dir)) != NULL) {
173
+ if ((entry->d_type == DT_REG || entry->d_type == DT_LNK) && strendswith(entry->d_name, ".dyncfg")) {
174
+ snprintf(filepath, sizeof(filepath), "%s/%s", dyncfg_globals.dir, entry->d_name);
175
+ dyncfg_file_load(filepath);
176
+ }
177
+ }
178
+
179
+ closedir(dir);
180
+}
181
+
182
+// ----------------------------------------------------------------------------
183
+// schemas loading
184
+
185
+static bool dyncfg_read_file_to_buffer(const char *filename, BUFFER *dst) {
186
+ int fd = open(filename, O_RDONLY, 0666);
187
+ if(unlikely(fd == -1))
188
+ return false;
189
+
190
+ struct stat st = { 0 };
191
+ if(fstat(fd, &st) != 0) {
192
+ close(fd);
193
+ return false;
194
+ }
195
+
196
+ buffer_flush(dst);
197
+ buffer_need_bytes(dst, st.st_size + 1); // +1 for the terminating zero
198
+
199
+ ssize_t r = read(fd, (char*)dst->buffer, st.st_size);
200
+ if(unlikely(r == -1)) {
201
+ close(fd);
202
+ return false;
203
+ }
204
+ dst->len = r;
205
+ dst->buffer[dst->len] = '\0';
206
+
207
+ close(fd);
208
+ return true;
209
+}
210
+
211
+bool dyncfg_get_schema(const char *id, BUFFER *dst) {
212
+ char filename[FILENAME_MAX + 1];
213
+
214
+ snprintfz(filename, sizeof(filename), "%s/schema.d/%s.json", netdata_configured_user_config_dir, id);
215
+ if(dyncfg_read_file_to_buffer(filename, dst))
216
+ return true;
217
+
218
+ snprintfz(filename, sizeof(filename), "%s/schema.d/%s.json", netdata_configured_stock_config_dir, id);
219
+ if(dyncfg_read_file_to_buffer(filename, dst))
220
+ return true;
221
+
222
+ return false;
223
+}
daemon/config/dyncfg-inline.c
new
+61
@@ -0,0 +1,61 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "dyncfg.h"
4
+
5
+static DICTIONARY *dyncfg_nodes = NULL;
6
+
7
+static int dyncfg_inline_callback(struct rrd_function_execute *rfe, void *data) {
8
+ char tr[UUID_COMPACT_STR_LEN];
9
+ uuid_unparse_lower_compact(*rfe->transaction, tr);
10
+
11
+ bool cancelled = rfe->is_cancelled.cb ? rfe->is_cancelled.cb(rfe->is_cancelled.data) : false;
12
+
13
+ int code;
14
+ if(cancelled)
15
+ code = HTTP_RESP_CLIENT_CLOSED_REQUEST;
16
+ else
17
+ code = dyncfg_node_find_and_call(dyncfg_nodes, tr, rfe->function, rfe->stop_monotonic_ut, &cancelled, rfe->payload, rfe->source, rfe->result.wb);
18
+
19
+ if(code == HTTP_RESP_CLIENT_CLOSED_REQUEST || (rfe->is_cancelled.cb && rfe->is_cancelled.cb(rfe->is_cancelled.data))) {
20
+ buffer_flush(rfe->result.wb);
21
+ code = HTTP_RESP_CLIENT_CLOSED_REQUEST;
22
+ }
23
+
24
+ if(rfe->result.cb)
25
+ rfe->result.cb(rfe->result.wb, code, rfe->result.data);
26
+
27
+ return code;
28
+}
29
+
30
+bool dyncfg_add(RRDHOST *host, const char *id, const char *path, DYNCFG_STATUS status, DYNCFG_TYPE type, DYNCFG_SOURCE_TYPE source_type, const char *source, DYNCFG_CMDS cmds, dyncfg_cb_t cb, void *data) {
31
+
32
+ if(dyncfg_add_low_level(host, id, path, status, type, source_type, source, cmds,
33
+ 0, 0, true,
34
+ dyncfg_inline_callback, NULL)) {
35
+ struct dyncfg_node tmp = {
36
+ .cmds = cmds,
37
+ .type = type,
38
+ .cb = cb,
39
+ .data = data,
40
+ };
41
+ dictionary_set(dyncfg_nodes, id, &tmp, sizeof(tmp));
42
+
43
+ return true;
44
+ }
45
+
46
+ return false;
47
+}
48
+
49
+void dyncfg_del(RRDHOST *host, const char *id) {
50
+ dictionary_del(dyncfg_nodes, id);
51
+ dyncfg_del_low_level(host, id);
52
+}
53
+
54
+void dyncfg_status(RRDHOST *host, const char *id, DYNCFG_STATUS status) {
55
+ dyncfg_status_low_level(host, id, status);
56
+}
57
+
58
+void dyncfg_init(bool load_saved) {
59
+ dyncfg_nodes = dyncfg_nodes_dictionary_create();
60
+ dyncfg_init_low_level(load_saved);
61
+}
daemon/config/dyncfg-intercept.c
new
+351
@@ -0,0 +1,351 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "dyncfg-internals.h"
4
+#include "dyncfg.h"
5
+
6
+// ----------------------------------------------------------------------------
7
+// we intercept the config function calls of the plugin
8
+
9
+struct dyncfg_call {
10
+ BUFFER *payload;
11
+ char *function;
12
+ char *id;
13
+ char *add_name;
14
+ char *source;
15
+ DYNCFG_CMDS cmd;
16
+ rrd_function_result_callback_t result_cb;
17
+ void *result_cb_data;
18
+ bool from_dyncfg_echo;
19
+};
20
+
21
+DYNCFG_STATUS dyncfg_status_from_successful_response(int code) {
22
+ DYNCFG_STATUS status;
23
+ if(code == DYNCFG_RESP_RUNNING)
24
+ status = DYNCFG_STATUS_RUNNING;
25
+ else if(code == DYNCFG_RESP_ACCEPTED || code == DYNCFG_RESP_ACCEPTED_RESTART_REQUIRED)
26
+ status = DYNCFG_STATUS_ACCEPTED;
27
+
28
+ return status;
29
+}
30
+
31
+void dyncfg_function_intercept_result_cb(BUFFER *wb, int code, void *result_cb_data) {
32
+ struct dyncfg_call *dc = result_cb_data;
33
+
34
+ bool called_from_dyncfg_echo = dc->from_dyncfg_echo;
35
+
36
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item_advanced(dyncfg_globals.nodes, dc->id, -1);
37
+ if(item) {
38
+ DYNCFG *df = dictionary_acquired_item_value(item);
39
+ bool old_user_disabled = df->user_disabled;
40
+ bool save_required = false;
41
+
42
+ if (!called_from_dyncfg_echo) {
43
+ // the command was sent by a user
44
+
45
+ if (DYNCFG_RESP_SUCCESS(code)) {
46
+ if (dc->cmd == DYNCFG_CMD_ADD) {
47
+ char id[strlen(dc->id) + 1 + strlen(dc->add_name) + 1];
48
+ snprintfz(id, sizeof(id), "%s:%s", dc->id, dc->add_name);
49
+
50
+ const DICTIONARY_ITEM *new_item = dyncfg_add_internal(
51
+ df->host,
52
+ id,
53
+ string2str(df->path),
54
+ dyncfg_status_from_successful_response(code),
55
+ DYNCFG_TYPE_JOB,
56
+ DYNCFG_SOURCE_TYPE_DYNCFG,
57
+ dc->source,
58
+ (df->cmds & ~DYNCFG_CMD_ADD) | DYNCFG_CMD_GET | DYNCFG_CMD_UPDATE | DYNCFG_CMD_TEST | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE | DYNCFG_CMD_REMOVE,
59
+ 0,
60
+ 0,
61
+ df->sync,
62
+ df->execute_cb, df->execute_cb_data, false);
63
+
64
+ DYNCFG *new_df = dictionary_acquired_item_value(new_item);
65
+ SWAP(new_df->payload, dc->payload);
66
+ if(code == DYNCFG_RESP_ACCEPTED_RESTART_REQUIRED)
67
+ new_df->restart_required = true;
68
+
69
+ dyncfg_file_save(id, new_df);
70
+ dictionary_acquired_item_release(dyncfg_globals.nodes, new_item);
71
+ } else if (dc->cmd == DYNCFG_CMD_UPDATE) {
72
+ df->source_type = DYNCFG_SOURCE_TYPE_DYNCFG;
73
+ string_freez(df->source);
74
+ df->source = string_strdupz(dc->source);
75
+
76
+ df->status = dyncfg_status_from_successful_response(code);
77
+ SWAP(df->payload, dc->payload);
78
+
79
+ save_required = true;
80
+ } else if (dc->cmd == DYNCFG_CMD_ENABLE) {
81
+ df->user_disabled = false;
82
+ } else if (dc->cmd == DYNCFG_CMD_DISABLE) {
83
+ df->user_disabled = true;
84
+ } else if (dc->cmd == DYNCFG_CMD_REMOVE) {
85
+ dyncfg_file_delete(dc->id);
86
+ }
87
+
88
+ if(dc->cmd != DYNCFG_CMD_ADD && code == DYNCFG_RESP_ACCEPTED_RESTART_REQUIRED)
89
+ df->restart_required = true;
90
+ }
91
+ else
92
+ nd_log(NDLS_DAEMON, NDLP_ERR,
93
+ "DYNCFG: plugin returned code %d to user initiated call: %s", code, dc->function);
94
+ }
95
+ else {
96
+ // the command was sent by dyncfg
97
+
98
+ if(DYNCFG_RESP_SUCCESS(code)) {
99
+ if(dc->cmd == DYNCFG_CMD_ADD) {
100
+ char id[strlen(dc->id) + 1 + strlen(dc->add_name) + 1];
101
+ snprintfz(id, sizeof(id), "%s:%s", dc->id, dc->add_name);
102
+
103
+ const DICTIONARY_ITEM *new_item = dictionary_get_and_acquire_item(dyncfg_globals.nodes, id);
104
+ if(new_item) {
105
+ DYNCFG *new_df = dictionary_acquired_item_value(new_item);
106
+ new_df->status = dyncfg_status_from_successful_response(code);
107
+
108
+ if(code == DYNCFG_RESP_ACCEPTED_RESTART_REQUIRED)
109
+ new_df->restart_required = true;
110
+
111
+ dictionary_acquired_item_release(dyncfg_globals.nodes, new_item);
112
+ }
113
+ }
114
+ else if(dc->cmd == DYNCFG_CMD_UPDATE) {
115
+ df->status = dyncfg_status_from_successful_response(code);
116
+ df->plugin_rejected = false;
117
+ }
118
+ else if(dc->cmd == DYNCFG_CMD_DISABLE)
119
+ df->status = DYNCFG_STATUS_DISABLED;
120
+ else if(dc->cmd == DYNCFG_CMD_ENABLE)
121
+ df->status = dyncfg_status_from_successful_response(code);
122
+
123
+ if(dc->cmd != DYNCFG_CMD_ADD && code == DYNCFG_RESP_ACCEPTED_RESTART_REQUIRED)
124
+ df->restart_required = true;
125
+ }
126
+ else {
127
+ nd_log(NDLS_DAEMON, NDLP_ERR,
128
+ "DYNCFG: plugin returned code %d to dyncfg initiated call: %s", code, dc->function);
129
+
130
+ if(dc->cmd & (DYNCFG_CMD_UPDATE | DYNCFG_CMD_ADD))
131
+ df->plugin_rejected = true;
132
+ }
133
+ }
134
+
135
+ if (save_required || old_user_disabled != df->user_disabled)
136
+ dyncfg_file_save(dc->id, df);
137
+
138
+ dictionary_acquired_item_release(dyncfg_globals.nodes, item);
139
+ }
140
+
141
+ if(dc->result_cb)
142
+ dc->result_cb(wb, code, dc->result_cb_data);
143
+
144
+ buffer_free(dc->payload);
145
+ freez(dc->function);
146
+ freez(dc->id);
147
+ freez(dc->source);
148
+ freez(dc->add_name);
149
+ freez(dc);
150
+}
151
+
152
+// ----------------------------------------------------------------------------
153
+
154
+static void dyncfg_apply_action_on_all_template_jobs(const char *template_id, DYNCFG_CMDS c) {
155
+ STRING *template = string_strdupz(template_id);
156
+
157
+ DYNCFG *df;
158
+ dfe_start_reentrant(dyncfg_globals.nodes, df) {
159
+ if(df->template == template && df->type == DYNCFG_TYPE_JOB) {
160
+ DYNCFG_STATUS cmd_to_send_to_plugin = c;
161
+
162
+ if(c == DYNCFG_CMD_ENABLE)
163
+ cmd_to_send_to_plugin = df->user_disabled ? DYNCFG_CMD_DISABLE : DYNCFG_CMD_ENABLE;
164
+ else if(c == DYNCFG_CMD_DISABLE)
165
+ cmd_to_send_to_plugin = DYNCFG_CMD_DISABLE;
166
+
167
+ dyncfg_echo(df_dfe.item, df, df_dfe.name, cmd_to_send_to_plugin);
168
+ }
169
+ }
170
+ dfe_done(df);
171
+
172
+ string_freez(template);
173
+}
174
+
175
+// ----------------------------------------------------------------------------
176
+// the callback for all config functions
177
+
178
+int dyncfg_function_intercept_cb(struct rrd_function_execute *rfe, void *data __maybe_unused) {
179
+
180
+ // IMPORTANT: this function MUST call the result_cb even on failures
181
+
182
+ bool called_from_dyncfg_echo = rrd_function_has_this_original_result_callback(rfe->transaction, dyncfg_echo_cb);
183
+
184
+ DYNCFG_CMDS c = DYNCFG_CMD_NONE;
185
+ const DICTIONARY_ITEM *item = NULL;
186
+ const char *add_name = NULL;
187
+ size_t add_name_len = 0;
188
+ if(strncmp(rfe->function, PLUGINSD_FUNCTION_CONFIG " ", sizeof(PLUGINSD_FUNCTION_CONFIG)) == 0) {
189
+ const char *id = &rfe->function[sizeof(PLUGINSD_FUNCTION_CONFIG)];
190
+ while(isspace(*id)) id++;
191
+ const char *space = id;
192
+ while(*space && !isspace(*space)) space++;
193
+ size_t id_len = space - id;
194
+
195
+ const char *cmd = space;
196
+ while(isspace(*cmd)) cmd++;
197
+ space = cmd;
198
+ while(*space && !isspace(*space)) space++;
199
+ size_t cmd_len = space - cmd;
200
+
201
+ char cmd_copy[cmd_len + 1];
202
+ strncpyz(cmd_copy, cmd, cmd_len);
203
+ c = dyncfg_cmds2id(cmd_copy);
204
+
205
+ if(c == DYNCFG_CMD_ADD) {
206
+ add_name = space;
207
+ while(isspace(*add_name)) add_name++;
208
+ space = add_name;
209
+ while(*space && !isspace(*space)) space++;
210
+ add_name_len = space - add_name;
211
+ }
212
+
213
+ item = dictionary_get_and_acquire_item_advanced(dyncfg_globals.nodes, id, (ssize_t)id_len);
214
+ }
215
+
216
+ int rc = HTTP_RESP_INTERNAL_SERVER_ERROR;
217
+
218
+ if(!item) {
219
+ rc = HTTP_RESP_NOT_FOUND;
220
+ dyncfg_default_response(rfe->result.wb, rc, "dyncfg functions intercept: id is not found");
221
+
222
+ if(rfe->result.cb)
223
+ rfe->result.cb(rfe->result.wb, rc, rfe->result.data);
224
+
225
+ return HTTP_RESP_NOT_FOUND;
226
+ }
227
+
228
+ DYNCFG *df = dictionary_acquired_item_value(item);
229
+ const char *id = dictionary_acquired_item_name(item);
230
+ bool has_payload = rfe->payload && buffer_strlen(rfe->payload) ? true : false;
231
+ bool make_the_call_to_plugin = true;
232
+
233
+ if((c & (DYNCFG_CMD_GET | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE | DYNCFG_CMD_REMOVE | DYNCFG_CMD_RESTART)) && has_payload)
234
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: command has a payload, but it is not going to be used: %s", rfe->function);
235
+
236
+ if(c == DYNCFG_CMD_NONE) {
237
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: this command is unknown: %s", rfe->function);
238
+
239
+ rc = HTTP_RESP_BAD_REQUEST;
240
+ dyncfg_default_response(rfe->result.wb, rc,
241
+ "dyncfg functions intercept: unknown command");
242
+ make_the_call_to_plugin = false;
243
+ }
244
+ else if(!(df->cmds & c)) {
245
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: this command is not supported by the configuration node: %s", rfe->function);
246
+
247
+ rc = HTTP_RESP_BAD_REQUEST;
248
+ dyncfg_default_response(rfe->result.wb, rc,
249
+ "dyncfg functions intercept: this command is not supported by this configuration node");
250
+ make_the_call_to_plugin = false;
251
+ }
252
+ else if((c & (DYNCFG_CMD_ADD | DYNCFG_CMD_UPDATE | DYNCFG_CMD_TEST)) && !has_payload) {
253
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: command requires a payload, but no payload given: %s", rfe->function);
254
+
255
+ rc = HTTP_RESP_BAD_REQUEST;
256
+ dyncfg_default_response(rfe->result.wb, rc,
257
+ "dyncfg functions intercept: payload is required");
258
+ make_the_call_to_plugin = false;
259
+ }
260
+ else if(c == DYNCFG_CMD_SCHEMA) {
261
+ bool loaded = false;
262
+ if(df->type == DYNCFG_TYPE_JOB) {
263
+ char template[strlen(id) + 1];
264
+ memcpy(template, id, sizeof(template));
265
+ char *colon = strrchr(template, ':');
266
+ if(colon) *colon = '\0';
267
+ if(template[0])
268
+ loaded = dyncfg_get_schema(template, rfe->result.wb);
269
+ }
270
+ else
271
+ loaded = dyncfg_get_schema(id, rfe->result.wb);
272
+
273
+ if(loaded) {
274
+ rfe->result.wb->content_type = CT_APPLICATION_JSON;
275
+ rfe->result.wb->expires = now_realtime_sec();
276
+ rc = HTTP_RESP_OK;
277
+ make_the_call_to_plugin = false;
278
+ }
279
+ }
280
+ else if(c & (DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE | DYNCFG_CMD_RESTART) && df->type == DYNCFG_TYPE_TEMPLATE) {
281
+ if(!called_from_dyncfg_echo) {
282
+ bool old_user_disabled = df->user_disabled;
283
+ if (c == DYNCFG_CMD_ENABLE)
284
+ df->user_disabled = false;
285
+ else if (c == DYNCFG_CMD_DISABLE)
286
+ df->user_disabled = true;
287
+
288
+ if (df->user_disabled != old_user_disabled)
289
+ dyncfg_file_save(id, df);
290
+ }
291
+
292
+ dyncfg_apply_action_on_all_template_jobs(id, c);
293
+
294
+ rc = HTTP_RESP_OK;
295
+ dyncfg_default_response(rfe->result.wb, rc, "applied");
296
+ make_the_call_to_plugin = false;
297
+ }
298
+ else if(c == DYNCFG_CMD_ADD) {
299
+ if (df->type != DYNCFG_TYPE_TEMPLATE) {
300
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: add command can only be applied on templates, not %s: %s",
301
+ dyncfg_id2type(df->type), rfe->function);
302
+
303
+ rc = HTTP_RESP_BAD_REQUEST;
304
+ dyncfg_default_response(rfe->result.wb, rc,
305
+ "dyncfg functions intercept: add command is only allowed in templates");
306
+ make_the_call_to_plugin = false;
307
+ }
308
+ else if (!add_name || !*add_name || !add_name_len) {
309
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: add command does not specify a name: %s", rfe->function);
310
+
311
+ rc = HTTP_RESP_BAD_REQUEST;
312
+ dyncfg_default_response(rfe->result.wb, rc,
313
+ "dyncfg functions intercept: command add requires a name, which is missing");
314
+
315
+ make_the_call_to_plugin = false;
316
+ }
317
+ }
318
+ else if(c == DYNCFG_CMD_ENABLE && df->type == DYNCFG_TYPE_JOB && dyncfg_is_user_disabled(string2str(df->template))) {
319
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: cannot enable a job of a disabled template: %s", rfe->function);
320
+
321
+ rc = HTTP_RESP_BAD_REQUEST;
322
+ dyncfg_default_response(rfe->result.wb, rc,
323
+ "dyncfg functions intercept: this job belongs to disabled template");
324
+
325
+ make_the_call_to_plugin = false;
326
+ }
327
+
328
+ if(make_the_call_to_plugin) {
329
+ struct dyncfg_call *dc = callocz(1, sizeof(*dc));
330
+ dc->function = strdupz(rfe->function);
331
+ dc->id = strdupz(id);
332
+ dc->source = rfe->source ? strdupz(rfe->source) : NULL;
333
+ dc->add_name = (c == DYNCFG_CMD_ADD) ? strndupz(add_name, add_name_len) : NULL;
334
+ dc->cmd = c;
335
+ dc->result_cb = rfe->result.cb;
336
+ dc->result_cb_data = rfe->result.data;
337
+ dc->payload = buffer_dup(rfe->payload);
338
+ dc->from_dyncfg_echo = called_from_dyncfg_echo;
339
+
340
+ rfe->result.cb = dyncfg_function_intercept_result_cb;
341
+ rfe->result.data = dc;
342
+
343
+ rc = df->execute_cb(rfe, df->execute_cb_data);
344
+ }
345
+ else if(rfe->result.cb)
346
+ rfe->result.cb(rfe->result.wb, rc, rfe->result.data);
347
+
348
+ dictionary_acquired_item_release(dyncfg_globals.nodes, item);
349
+ return rc;
350
+}
351
+
daemon/config/dyncfg-internals.h
new
+65
@@ -0,0 +1,65 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_DYNCFG_INTERNALS_H
4
+#define NETDATA_DYNCFG_INTERNALS_H
5
+
6
+#include "../common.h"
7
+#include "../../database/rrd.h"
8
+#include "../../database/rrdfunctions.h"
9
+#include "../../database/rrdfunctions-internals.h"
10
+#include "../../database/rrdcollector-internals.h"
11
+
12
+typedef struct dyncfg {
13
+ RRDHOST *host;
14
+ uuid_t host_uuid;
15
+ STRING *function;
16
+ STRING *template;
17
+ STRING *path;
18
+ DYNCFG_STATUS status;
19
+ DYNCFG_TYPE type;
20
+ DYNCFG_CMDS cmds;
21
+ DYNCFG_SOURCE_TYPE source_type;
22
+ STRING *source;
23
+ usec_t created_ut;
24
+ usec_t modified_ut;
25
+ uint32_t saves;
26
+ bool sync;
27
+ bool user_disabled;
28
+ bool plugin_rejected;
29
+ bool restart_required;
30
+
31
+ BUFFER *payload;
32
+
33
+ rrd_function_execute_cb_t execute_cb;
34
+ void *execute_cb_data;
35
+
36
+ // constructor data
37
+ bool overwrite_cb;
38
+} DYNCFG;
39
+
40
+struct dyncfg_globals {
41
+ const char *dir;
42
+ DICTIONARY *nodes;
43
+};
44
+
45
+extern struct dyncfg_globals dyncfg_globals;
46
+
47
+void dyncfg_load_all(void);
48
+void dyncfg_file_load(const char *filename);
49
+void dyncfg_file_save(const char *id, DYNCFG *df);
50
+void dyncfg_file_delete(const char *id);
51
+
52
+bool dyncfg_get_schema(const char *id, BUFFER *dst);
53
+
54
+void dyncfg_echo_cb(BUFFER *wb, int code, void *result_cb_data);
55
+void dyncfg_echo(const DICTIONARY_ITEM *item, DYNCFG *df, const char *id, DYNCFG_CMDS cmd);
56
+void dyncfg_echo_update(const DICTIONARY_ITEM *item, DYNCFG *df, const char *id);
57
+void dyncfg_echo_add(const DICTIONARY_ITEM *template_item, DYNCFG *template_df, const char *template_id, const char *job_name);
58
+
59
+const DICTIONARY_ITEM *dyncfg_add_internal(RRDHOST *host, const char *id, const char *path, DYNCFG_STATUS status, DYNCFG_TYPE type, DYNCFG_SOURCE_TYPE source_type, const char *source, DYNCFG_CMDS cmds, usec_t created_ut, usec_t modified_ut, bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data, bool overwrite_cb);
60
+int dyncfg_function_intercept_cb(struct rrd_function_execute *rfe, void *data);
61
+void dyncfg_cleanup(DYNCFG *v);
62
+
63
+bool dyncfg_is_user_disabled(const char *id);
64
+
65
+#endif //NETDATA_DYNCFG_INTERNALS_H
daemon/config/dyncfg-tree.c
new
+202
@@ -0,0 +1,202 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "dyncfg-internals.h"
4
+#include "dyncfg.h"
5
+
6
+static int dyncfg_tree_compar(const void *a, const void *b) {
7
+ const DICTIONARY_ITEM *item1 = *(const DICTIONARY_ITEM **)a;
8
+ const DICTIONARY_ITEM *item2 = *(const DICTIONARY_ITEM **)b;
9
+
10
+ DYNCFG *df1 = dictionary_acquired_item_value(item1);
11
+ DYNCFG *df2 = dictionary_acquired_item_value(item2);
12
+
13
+ int rc = string_cmp(df1->path, df2->path);
14
+ if(rc == 0)
15
+ rc = strcmp(dictionary_acquired_item_name(item1), dictionary_acquired_item_name(item2));
16
+
17
+ return rc;
18
+}
19
+
20
+static void dyncfg_to_json(DYNCFG *df, const char *id, BUFFER *wb) {
21
+ buffer_json_member_add_object(wb, id);
22
+ {
23
+ buffer_json_member_add_string(wb, "type", dyncfg_id2type(df->type));
24
+ buffer_json_member_add_string(wb, "status", dyncfg_id2status(df->status));
25
+ dyncfg_cmds2json_array(df->cmds, "cmds", wb);
26
+ buffer_json_member_add_string(wb, "source_type", dyncfg_id2source_type(df->source_type));
27
+ buffer_json_member_add_string(wb, "source", string2str(df->source));
28
+ buffer_json_member_add_boolean(wb, "sync", df->sync);
29
+ buffer_json_member_add_boolean(wb, "user_disabled", df->user_disabled);
30
+ buffer_json_member_add_boolean(wb, "restart_required", df->restart_required);
31
+ buffer_json_member_add_boolean(wb, "plugin_rejected", df->restart_required);
32
+ buffer_json_member_add_object(wb, "payload");
33
+ {
34
+ if (df->payload && buffer_strlen(df->payload)) {
35
+ buffer_json_member_add_boolean(wb, "available", true);
36
+ buffer_json_member_add_string(wb, "content_type", content_type_id2string(df->payload->content_type));
37
+ buffer_json_member_add_uint64(wb, "content_length", df->payload->len);
38
+ } else
39
+ buffer_json_member_add_boolean(wb, "available", false);
40
+ }
41
+ buffer_json_object_close(wb); // payload
42
+ buffer_json_member_add_uint64(wb, "saves", df->saves);
43
+ buffer_json_member_add_uint64(wb, "created_ut", df->created_ut);
44
+ buffer_json_member_add_uint64(wb, "modified_ut", df->modified_ut);
45
+ }
46
+ buffer_json_object_close(wb);
47
+}
48
+
49
+static void dyncfg_tree_for_host(RRDHOST *host, BUFFER *wb, const char *parent, const char *id) {
50
+ size_t entries = dictionary_entries(dyncfg_globals.nodes);
51
+ size_t used = 0;
52
+ const DICTIONARY_ITEM *items[entries];
53
+ size_t restart_required = 0, plugin_rejected = 0, status_incomplete = 0, status_failed = 0;
54
+
55
+ size_t parent_len = strlen(parent);
56
+ DYNCFG *df;
57
+ dfe_start_read(dyncfg_globals.nodes, df) {
58
+ if(!df->host) {
59
+ if(uuid_memcmp(&df->host_uuid, &host->host_uuid) == 0)
60
+ df->host = host;
61
+ }
62
+
63
+ if(df->host != host || strncmp(string2str(df->path), parent, parent_len) != 0)
64
+ continue;
65
+
66
+ if(!rrd_function_available(host, string2str(df->function)))
67
+ df->status = DYNCFG_STATUS_ORPHAN;
68
+
69
+ items[used++] = dictionary_acquired_item_dup(dyncfg_globals.nodes, df_dfe.item);
70
+ }
71
+ dfe_done(df);
72
+
73
+ qsort(items, used, sizeof(const DICTIONARY_ITEM *), dyncfg_tree_compar);
74
+
75
+ buffer_flush(wb);
76
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
77
+
78
+ buffer_json_member_add_uint64(wb, "version", 1);
79
+
80
+ buffer_json_member_add_object(wb, "tree");
81
+ {
82
+ STRING *last_path = NULL;
83
+ for (size_t i = 0; i < used; i++) {
84
+ df = dictionary_acquired_item_value(items[i]);
85
+ if (df->path != last_path) {
86
+ last_path = df->path;
87
+
88
+ if (i)
89
+ buffer_json_object_close(wb);
90
+
91
+ buffer_json_member_add_object(wb, string2str(last_path));
92
+ }
93
+
94
+ dyncfg_to_json(df, dictionary_acquired_item_name(items[i]), wb);
95
+
96
+ if(df->status != DYNCFG_STATUS_ORPHAN) {
97
+ if (df->restart_required)
98
+ restart_required++;
99
+
100
+ if (df->plugin_rejected)
101
+ plugin_rejected++;
102
+
103
+ if (df->status == DYNCFG_STATUS_FAILED)
104
+ status_failed++;
105
+
106
+ if (df->status == DYNCFG_STATUS_INCOMPLETE)
107
+ status_incomplete++;
108
+ }
109
+ }
110
+
111
+ if (used)
112
+ buffer_json_object_close(wb);
113
+ }
114
+ buffer_json_object_close(wb); // tree
115
+
116
+ buffer_json_member_add_object(wb, "attention");
117
+ {
118
+ buffer_json_member_add_boolean(wb, "degraded", restart_required + plugin_rejected + status_failed + status_incomplete > 0);
119
+ buffer_json_member_add_uint64(wb, "restart_required", restart_required);
120
+ buffer_json_member_add_uint64(wb, "plugin_rejected", plugin_rejected);
121
+ buffer_json_member_add_uint64(wb, "status_failed", status_failed);
122
+ buffer_json_member_add_uint64(wb, "status_incomplete", status_incomplete);
123
+ }
124
+ buffer_json_object_close(wb); // attention
125
+
126
+ buffer_json_agents_v2(wb, NULL, 0, false, false);
127
+
128
+ buffer_json_finalize(wb);
129
+
130
+ for(size_t i = 0; i < used ;i++)
131
+ dictionary_acquired_item_release(dyncfg_globals.nodes, items[i]);
132
+}
133
+
134
+static int dyncfg_config_execute_cb(struct rrd_function_execute *rfe, void *data) {
135
+ RRDHOST *host = data;
136
+ int code;
137
+
138
+ char buf[strlen(rfe->function) + 1];
139
+ memcpy(buf, rfe->function, sizeof(buf));
140
+
141
+ char *words[MAX_FUNCTION_PARAMETERS]; // an array of pointers for the words in this line
142
+ size_t num_words = quoted_strings_splitter_pluginsd(buf, words, MAX_FUNCTION_PARAMETERS);
143
+
144
+ const char *config = get_word(words, num_words, 0);
145
+ const char *action = get_word(words, num_words, 1);
146
+ const char *path = get_word(words, num_words, 2);
147
+ const char *id = get_word(words, num_words, 3);
148
+
149
+ if(!config || !*config || strcmp(config, PLUGINSD_FUNCTION_CONFIG) != 0) {
150
+ char *msg = "invalid function call, expected: config";
151
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG TREE: function call '%s': %s", rfe->function, msg);
152
+ code = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
153
+ goto cleanup;
154
+ }
155
+
156
+ if(!action || !*action) {
157
+ char *msg = "invalid function call, expected: config tree";
158
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG TREE: function call '%s': %s", rfe->function, msg);
159
+ code = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
160
+ goto cleanup;
161
+ }
162
+
163
+ if(strcmp(action, "tree") == 0) {
164
+ if(!path || !*path)
165
+ path = "/";
166
+
167
+ if(!id || !*id)
168
+ id = NULL;
169
+ else if(!dyncfg_is_valid_id(id)) {
170
+ char *msg = "invalid id given";
171
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG TREE: function call '%s': %s", rfe->function, msg);
172
+ code = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
173
+ goto cleanup;
174
+ }
175
+
176
+ code = HTTP_RESP_OK;
177
+ dyncfg_tree_for_host(host, rfe->result.wb, path, id);
178
+ }
179
+ else {
180
+ code = HTTP_RESP_NOT_FOUND;
181
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: unknown config id '%s' in call: '%s'. This can happen if the plugin that registered the dynamic configuration is not running now.", action, rfe->function);
182
+ rrd_call_function_error(rfe->result.wb, "unknown config id given", code);
183
+ }
184
+
185
+cleanup:
186
+ if(rfe->result.cb)
187
+ rfe->result.cb(rfe->result.wb, code, rfe->result.data);
188
+
189
+ return code;
190
+}
191
+
192
+// ----------------------------------------------------------------------------
193
+// this adds a 'config' function to all leaf nodes (localhost and virtual nodes)
194
+// which is used to serve the tree and act as a catch-all for all config calls
195
+// for which there is no id overloaded.
196
+
197
+void dyncfg_host_init(RRDHOST *host) {
198
+ rrd_function_add(host, NULL, PLUGINSD_FUNCTION_CONFIG, 120,
199
+ 1000, "Dynamic configuration", "config",
200
+ HTTP_ACCESS_ADMIN,
201
+ true, dyncfg_config_execute_cb, host);
202
+}
daemon/config/dyncfg-unittest.c
new
+792
@@ -0,0 +1,792 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "dyncfg-internals.h"
4
+#include "dyncfg.h"
5
+
6
+// ----------------------------------------------------------------------------
7
+// unit test
8
+
9
+#define LINE_FILE_STR TOSTRING(__LINE__) "@" __FILE__
10
+
11
+struct dyncfg_unittest {
12
+ bool enabled;
13
+ size_t errors;
14
+
15
+ DICTIONARY *nodes;
16
+
17
+ SPINLOCK spinlock;
18
+ struct dyncfg_unittest_action *queue;
19
+} dyncfg_unittest_data = { 0 };
20
+
21
+typedef struct {
22
+ bool enabled;
23
+ bool removed;
24
+ struct {
25
+ double dbl;
26
+ bool bln;
27
+ } value;
28
+} TEST_CFG;
29
+
30
+typedef struct {
31
+ const char *id;
32
+ const char *source;
33
+ bool sync;
34
+ DYNCFG_TYPE type;
35
+ DYNCFG_CMDS cmds;
36
+ DYNCFG_SOURCE_TYPE source_type;
37
+
38
+ TEST_CFG current;
39
+ TEST_CFG expected;
40
+
41
+ bool received;
42
+ bool finished;
43
+
44
+ size_t last_saves;
45
+ bool needs_save;
46
+} TEST;
47
+
48
+struct dyncfg_unittest_action {
49
+ TEST *t;
50
+ BUFFER *result;
51
+ BUFFER *payload;
52
+ DYNCFG_CMDS cmd;
53
+ const char *add_name;
54
+ const char *source;
55
+
56
+ rrd_function_result_callback_t result_cb;
57
+ void *result_cb_data;
58
+
59
+ struct dyncfg_unittest_action *prev, *next;
60
+};
61
+
62
+static void dyncfg_unittest_register_error(const char *id, const char *msg) {
63
+ if(msg)
64
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG UNITTEST: error on id '%s': %s", id ? id : "", msg);
65
+
66
+ __atomic_add_fetch(&dyncfg_unittest_data.errors, 1, __ATOMIC_RELAXED);
67
+}
68
+
69
+static int dyncfg_unittest_execute_cb(struct rrd_function_execute *rfe, void *data);
70
+
71
+bool dyncfg_unittest_parse_payload(BUFFER *payload, TEST *t, DYNCFG_CMDS cmd, const char *add_name, const char *source) {
72
+ CLEAN_JSON_OBJECT *jobj = json_tokener_parse(buffer_tostring(payload));
73
+ if(!jobj) {
74
+ dyncfg_unittest_register_error(t->id, "cannot parse json payload");
75
+ return false;
76
+ }
77
+
78
+ struct json_object *json_double;
79
+ struct json_object *json_boolean;
80
+
81
+ json_object_object_get_ex(jobj, "double", &json_double);
82
+ double value_double = json_object_get_double(json_double);
83
+
84
+ json_object_object_get_ex(jobj, "boolean", &json_boolean);
85
+ int value_boolean = json_object_get_boolean(json_boolean);
86
+
87
+ if(cmd == DYNCFG_CMD_UPDATE) {
88
+ t->current.value.dbl = value_double;
89
+ t->current.value.bln = value_boolean;
90
+ }
91
+ else if(cmd == DYNCFG_CMD_ADD) {
92
+ char buf[strlen(t->id) + strlen(add_name) + 20];
93
+ snprintfz(buf, sizeof(buf), "%s:%s", t->id, add_name);
94
+ TEST tmp = {
95
+ .id = strdupz(buf),
96
+ .source = strdupz(source),
97
+ .cmds = (t->cmds & ~DYNCFG_CMD_ADD) | DYNCFG_CMD_GET | DYNCFG_CMD_REMOVE | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE | DYNCFG_CMD_TEST,
98
+ .sync = t->sync,
99
+ .type = DYNCFG_TYPE_JOB,
100
+ .source_type = DYNCFG_SOURCE_TYPE_DYNCFG,
101
+ .received = true,
102
+ .finished = true,
103
+ .current =
104
+ {.enabled = true,
105
+ .removed = false,
106
+ .value =
107
+ {
108
+ .dbl = value_double,
109
+ .bln = value_boolean,
110
+ }},
111
+ .expected = {
112
+ .enabled = true,
113
+ .removed = false,
114
+ .value = {
115
+ .dbl = 3.14,
116
+ .bln = true,
117
+ }
118
+ },
119
+ .needs_save = true,
120
+ };
121
+ const DICTIONARY_ITEM *item = dictionary_set_and_acquire_item(dyncfg_unittest_data.nodes, buf, &tmp, sizeof(tmp));
122
+ TEST *t2 = dictionary_acquired_item_value(item);
123
+ dictionary_acquired_item_release(dyncfg_unittest_data.nodes, item);
124
+
125
+ dyncfg_add_low_level(localhost, t2->id, "/unittests",
126
+ DYNCFG_STATUS_RUNNING, t2->type, t2->source_type, t2->source,
127
+ t2->cmds, 0, 0, t2->sync,
128
+ dyncfg_unittest_execute_cb, t2);
129
+ }
130
+ else {
131
+ dyncfg_unittest_register_error(t->id, "invalid command received to parse payload");
132
+ return false;
133
+ }
134
+
135
+ return true;
136
+}
137
+
138
+static int dyncfg_unittest_action(struct dyncfg_unittest_action *a) {
139
+ TEST *t = a->t;
140
+
141
+ int rc = HTTP_RESP_OK;
142
+
143
+ if(a->cmd == DYNCFG_CMD_ENABLE)
144
+ t->current.enabled = true;
145
+ else if(a->cmd == DYNCFG_CMD_DISABLE)
146
+ t->current.enabled = false;
147
+ else if(a->cmd == DYNCFG_CMD_ADD || a->cmd == DYNCFG_CMD_UPDATE)
148
+ rc = dyncfg_unittest_parse_payload(a->payload, a->t, a->cmd, a->add_name, a->source) ? HTTP_RESP_OK : HTTP_RESP_BAD_REQUEST;
149
+ else if(a->cmd == DYNCFG_CMD_REMOVE)
150
+ t->current.removed = true;
151
+ else
152
+ rc = HTTP_RESP_BAD_REQUEST;
153
+
154
+ dyncfg_default_response(a->result, rc, NULL);
155
+
156
+ a->result_cb(a->result, rc, a->result_cb_data);
157
+
158
+ buffer_free(a->payload);
159
+ freez((void *)a->add_name);
160
+ freez(a);
161
+
162
+ __atomic_store_n(&t->finished, true, __ATOMIC_RELAXED);
163
+
164
+ return rc;
165
+}
166
+
167
+static void *dyncfg_unittest_thread_action(void *ptr) {
168
+ while(1) {
169
+ struct dyncfg_unittest_action *a = NULL;
170
+ spinlock_lock(&dyncfg_unittest_data.spinlock);
171
+ a = dyncfg_unittest_data.queue;
172
+ if(a)
173
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dyncfg_unittest_data.queue, a, prev, next);
174
+ spinlock_unlock(&dyncfg_unittest_data.spinlock);
175
+
176
+ if(a)
177
+ dyncfg_unittest_action(a);
178
+ else
179
+ sleep_usec(10 * USEC_PER_MS);
180
+ }
181
+}
182
+
183
+static int dyncfg_unittest_execute_cb(struct rrd_function_execute *rfe, void *data) {
184
+
185
+ int rc;
186
+ bool run_the_callback = true;
187
+ TEST *t = data;
188
+
189
+ t->received = true;
190
+
191
+ char buf[strlen(rfe->function) + 1];
192
+ memcpy(buf, rfe->function, sizeof(buf));
193
+
194
+ char *words[MAX_FUNCTION_PARAMETERS]; // an array of pointers for the words in this line
195
+ size_t num_words = quoted_strings_splitter_pluginsd(buf, words, MAX_FUNCTION_PARAMETERS);
196
+
197
+ const char *config = get_word(words, num_words, 0);
198
+ const char *id = get_word(words, num_words, 1);
199
+ const char *action = get_word(words, num_words, 2);
200
+ const char *add_name = get_word(words, num_words, 3);
201
+
202
+ if(!config || !*config || strcmp(config, PLUGINSD_FUNCTION_CONFIG) != 0) {
203
+ char *msg = "did not receive a config call";
204
+ dyncfg_unittest_register_error(id, msg);
205
+ rc = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
206
+ goto cleanup;
207
+ }
208
+
209
+ if(!id || !*id) {
210
+ char *msg = "did not receive an id";
211
+ dyncfg_unittest_register_error(id, msg);
212
+ rc = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
213
+ goto cleanup;
214
+ }
215
+
216
+ if(t->type != DYNCFG_TYPE_TEMPLATE && strcmp(t->id, id) != 0) {
217
+ char *msg = "id received is not the expected";
218
+ dyncfg_unittest_register_error(id, msg);
219
+ rc = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
220
+ goto cleanup;
221
+ }
222
+
223
+ if(!action || !*action) {
224
+ char *msg = "did not receive an action";
225
+ dyncfg_unittest_register_error(id, msg);
226
+ rc = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
227
+ goto cleanup;
228
+ }
229
+
230
+ DYNCFG_CMDS cmd = dyncfg_cmds2id(action);
231
+ if(cmd == DYNCFG_CMD_NONE) {
232
+ char *msg = "action received is not known";
233
+ dyncfg_unittest_register_error(id, msg);
234
+ rc = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
235
+ goto cleanup;
236
+ }
237
+
238
+ if(!(t->cmds & cmd)) {
239
+ char *msg = "received a command that is not supported";
240
+ dyncfg_unittest_register_error(id, msg);
241
+ rc = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
242
+ goto cleanup;
243
+ }
244
+
245
+ if(t->current.removed && cmd != DYNCFG_CMD_ADD) {
246
+ char *msg = "received a command for a removed entry";
247
+ dyncfg_unittest_register_error(id, msg);
248
+ rc = dyncfg_default_response(rfe->result.wb, HTTP_RESP_BAD_REQUEST, msg);
249
+ goto cleanup;
250
+ }
251
+
252
+ struct dyncfg_unittest_action *a = callocz(1, sizeof(*a));
253
+ a->t = t;
254
+ a->add_name = add_name ? strdupz(add_name) : NULL;
255
+ a->source = rfe->source,
256
+ a->result = rfe->result.wb;
257
+ a->payload = buffer_dup(rfe->payload);
258
+ a->cmd = cmd;
259
+ a->result_cb = rfe->result.cb;
260
+ a->result_cb_data = rfe->result.data;
261
+
262
+ run_the_callback = false;
263
+
264
+ if(t->sync)
265
+ rc = dyncfg_unittest_action(a);
266
+ else {
267
+ spinlock_lock(&dyncfg_unittest_data.spinlock);
268
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(dyncfg_unittest_data.queue, a, prev, next);
269
+ spinlock_unlock(&dyncfg_unittest_data.spinlock);
270
+ rc = HTTP_RESP_OK;
271
+ }
272
+
273
+cleanup:
274
+ if(run_the_callback) {
275
+ __atomic_store_n(&t->finished, true, __ATOMIC_RELAXED);
276
+
277
+ if (rfe->result.cb)
278
+ rfe->result.cb(rfe->result.wb, rc, rfe->result.data);
279
+ }
280
+
281
+ return rc;
282
+}
283
+
284
+static bool dyncfg_unittest_check(TEST *t, const char *cmd, bool received) {
285
+ size_t errors = 0;
286
+
287
+ fprintf(stderr, "CHECK '%s' after cmd '%s'...", t->id, cmd);
288
+
289
+ if(t->received != received) {
290
+ fprintf(stderr, "\n - received flag found '%s', expected '%s'",
291
+ t->received?"true":"false",
292
+ received?"true":"false");
293
+ errors++;
294
+ goto cleanup;
295
+ }
296
+
297
+ if(!received)
298
+ goto cleanup;
299
+
300
+ usec_t give_up_ut = now_monotonic_usec() + 2 * USEC_PER_SEC;
301
+ while(!__atomic_load_n(&t->finished, __ATOMIC_RELAXED)) {
302
+ static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
303
+ nanosleep(&ns, NULL);
304
+
305
+ if(now_monotonic_usec() > give_up_ut) {
306
+ fprintf(stderr, "\n - gave up waiting for the plugin to process this!");
307
+ errors++;
308
+ goto cleanup;
309
+ }
310
+ }
311
+
312
+ if(t->type != DYNCFG_TYPE_TEMPLATE && t->current.enabled != t->expected.enabled) {
313
+ fprintf(stderr, "\n - enabled flag found '%s', expected '%s'",
314
+ t->current.enabled?"true":"false",
315
+ t->expected.enabled?"true":"false");
316
+ errors++;
317
+ }
318
+ if(t->current.removed != t->expected.removed) {
319
+ fprintf(stderr, "\n - removed flag found '%s', expected '%s'",
320
+ t->current.removed?"true":"false",
321
+ t->expected.removed?"true":"false");
322
+ errors++;
323
+ }
324
+ if(t->current.value.bln != t->expected.value.bln) {
325
+ fprintf(stderr, "\n - boolean value found '%s', expected '%s'",
326
+ t->current.value.bln?"true":"false",
327
+ t->expected.value.bln?"true":"false");
328
+ errors++;
329
+ }
330
+ if(t->current.value.dbl != t->expected.value.dbl) {
331
+ fprintf(stderr, "\n - double value found '%f', expected '%f'",
332
+ t->current.value.dbl, t->expected.value.dbl);
333
+ errors++;
334
+ }
335
+
336
+ DYNCFG *df = dictionary_get(dyncfg_globals.nodes, t->id);
337
+ if(!df) {
338
+ fprintf(stderr, "\n - not found in DYNCFG nodes dictionary!");
339
+ errors++;
340
+ }
341
+ else if(df->cmds != t->cmds) {
342
+ fprintf(stderr, "\n - has different cmds in DYNCFG nodes dictionary; found: ");
343
+ dyncfg_cmds2fp(df->cmds, stderr);
344
+ fprintf(stderr, ", expected: ");
345
+ dyncfg_cmds2fp(t->cmds, stderr);
346
+ fprintf(stderr, "\n");
347
+ errors++;
348
+ }
349
+ else if(df->type == DYNCFG_TYPE_JOB && df->source_type == DYNCFG_SOURCE_TYPE_DYNCFG && !df->saves) {
350
+ fprintf(stderr, "\n - DYNCFG job has no saves!");
351
+ errors++;
352
+ }
353
+ else if(df->type == DYNCFG_TYPE_JOB && df->source_type == DYNCFG_SOURCE_TYPE_DYNCFG && (!df->payload || !buffer_strlen(df->payload))) {
354
+ fprintf(stderr, "\n - DYNCFG job has no payload!");
355
+ errors++;
356
+ }
357
+ else if(df->user_disabled && !df->saves) {
358
+ fprintf(stderr, "\n - DYNCFG disabled config has no saves!");
359
+ errors++;
360
+ }
361
+ else if(t->source && string_strcmp(df->source, t->source) != 0) {
362
+ fprintf(stderr, "\n - source does not match!");
363
+ errors++;
364
+ }
365
+ else if(df->source && !t->source) {
366
+ fprintf(stderr, "\n - there is a source but it shouldn't be any!");
367
+ errors++;
368
+ }
369
+ else if(t->needs_save && df->saves <= t->last_saves) {
370
+ fprintf(stderr, "\n - should be saved, but it is not saved!");
371
+ errors++;
372
+ }
373
+ else if(!t->needs_save && df->saves > t->last_saves) {
374
+ fprintf(stderr, "\n - should be not be saved, but it saved!");
375
+ errors++;
376
+ }
377
+
378
+cleanup:
379
+ if(errors) {
380
+ fprintf(stderr, "\n >>> FAILED\n\n");
381
+ dyncfg_unittest_register_error(NULL, NULL);
382
+ return false;
383
+ }
384
+
385
+ fprintf(stderr, " OK\n");
386
+ return true;
387
+}
388
+
389
+static void dyncfg_unittest_reset(void) {
390
+ TEST *t;
391
+ dfe_start_read(dyncfg_unittest_data.nodes, t) {
392
+ t->received = t->finished = false;
393
+ t->needs_save = false;
394
+
395
+ DYNCFG *df = dictionary_get(dyncfg_globals.nodes, t->id);
396
+ if(!df) {
397
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG UNITTEST: cannot find id '%s'", t->id);
398
+ dyncfg_unittest_register_error(NULL, NULL);
399
+ }
400
+ else
401
+ t->last_saves = df->saves;
402
+ }
403
+ dfe_done(t);
404
+}
405
+
406
+void should_be_saved(TEST *t, DYNCFG_CMDS c) {
407
+ DYNCFG *df;
408
+
409
+ if(t->type == DYNCFG_TYPE_TEMPLATE) {
410
+ df = dictionary_get(dyncfg_globals.nodes, t->id);
411
+ t->current.enabled = !df->user_disabled;
412
+ }
413
+
414
+ t->needs_save =
415
+ c == DYNCFG_CMD_UPDATE ||
416
+ (t->current.enabled && c == DYNCFG_CMD_DISABLE) ||
417
+ (!t->current.enabled && c == DYNCFG_CMD_ENABLE);
418
+}
419
+
420
+static int dyncfg_unittest_run(const char *cmd, BUFFER *wb, const char *payload, const char *source) {
421
+ dyncfg_unittest_reset();
422
+
423
+ char buf[strlen(cmd) + 1];
424
+ memcpy(buf, cmd, sizeof(buf));
425
+
426
+ char *words[MAX_FUNCTION_PARAMETERS]; // an array of pointers for the words in this line
427
+ size_t num_words = quoted_strings_splitter_pluginsd(buf, words, MAX_FUNCTION_PARAMETERS);
428
+
429
+ // const char *config = get_word(words, num_words, 0);
430
+ const char *id = get_word(words, num_words, 1);
431
+ char *action = get_word(words, num_words, 2);
432
+ const char *add_name = get_word(words, num_words, 3);
433
+
434
+ DYNCFG_CMDS c = dyncfg_cmds2id(action);
435
+
436
+ TEST *t = dictionary_get(dyncfg_unittest_data.nodes, id);
437
+ if(!t) {
438
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG UNITTEST: cannot find id '%s' from cmd: %s", id, cmd);
439
+ dyncfg_unittest_register_error(NULL, NULL);
440
+ return HTTP_RESP_NOT_FOUND;
441
+ }
442
+
443
+ if(t->type == DYNCFG_TYPE_TEMPLATE)
444
+ t->received = t->finished = true;
445
+
446
+ if(c == DYNCFG_CMD_DISABLE)
447
+ t->expected.enabled = false;
448
+ if(c == DYNCFG_CMD_ENABLE)
449
+ t->expected.enabled = true;
450
+ if(c == DYNCFG_CMD_UPDATE)
451
+ memset(&t->current.value, 0, sizeof(t->current.value));
452
+
453
+ buffer_flush(wb);
454
+
455
+ CLEAN_BUFFER *pld = NULL;
456
+
457
+ if(payload) {
458
+ pld = buffer_create(1024, NULL);
459
+ buffer_strcat(pld, payload);
460
+ }
461
+
462
+ should_be_saved(t, c);
463
+
464
+ int rc = rrd_function_run(localhost, wb, 10, HTTP_ACCESS_ADMIN, cmd,
465
+ true, NULL,
466
+ NULL, NULL,
467
+ NULL, NULL,
468
+ NULL, NULL,
469
+ pld, source);
470
+ if(!DYNCFG_RESP_SUCCESS(rc)) {
471
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG UNITTEST: failed to run: %s; returned code %d", cmd, rc);
472
+ dyncfg_unittest_register_error(NULL, NULL);
473
+ }
474
+
475
+ dyncfg_unittest_check(t, cmd, true);
476
+
477
+ if(rc == HTTP_RESP_OK && t->type == DYNCFG_TYPE_TEMPLATE) {
478
+ if(c == DYNCFG_CMD_ADD) {
479
+ char buf2[strlen(id) + strlen(add_name) + 2];
480
+ snprintfz(buf2, sizeof(buf2), "%s:%s", id, add_name);
481
+ TEST *tt = dictionary_get(dyncfg_unittest_data.nodes, buf2);
482
+ if (!tt) {
483
+ nd_log(NDLS_DAEMON, NDLP_ERR,
484
+ "DYNCFG UNITTEST: failed to find newly added id '%s' of command: %s",
485
+ id, cmd);
486
+ dyncfg_unittest_register_error(NULL, NULL);
487
+ }
488
+ dyncfg_unittest_check(tt, cmd, true);
489
+ }
490
+ else {
491
+ STRING *template = string_strdupz(t->id);
492
+ DYNCFG *df;
493
+ dfe_start_read(dyncfg_globals.nodes, df) {
494
+ if(df->type == DYNCFG_TYPE_JOB && df->template == template) {
495
+ TEST *tt = dictionary_get(dyncfg_unittest_data.nodes, df_dfe.name);
496
+ if (!tt) {
497
+ nd_log(NDLS_DAEMON, NDLP_ERR,
498
+ "DYNCFG UNITTEST: failed to find id '%s' while running command: %s", df_dfe.name, cmd);
499
+ dyncfg_unittest_register_error(NULL, NULL);
500
+ }
501
+ else {
502
+ if(c == DYNCFG_CMD_DISABLE)
503
+ tt->expected.enabled = false;
504
+ if(c == DYNCFG_CMD_ENABLE)
505
+ tt->expected.enabled = true;
506
+ dyncfg_unittest_check(tt, cmd, true);
507
+ }
508
+ }
509
+ }
510
+ dfe_done(df);
511
+ string_freez(template);
512
+ }
513
+ }
514
+
515
+ return rc;
516
+}
517
+
518
+static void dyncfg_unittest_cleanup_files(void) {
519
+ char path[PATH_MAX];
520
+ snprintfz(path, sizeof(path), "%s/%s", netdata_configured_varlib_dir, "config");
521
+
522
+ DIR *dir = opendir(path);
523
+ if (!dir) {
524
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG UNITTEST: cannot open directory '%s'", path);
525
+ return;
526
+ }
527
+
528
+ struct dirent *entry;
529
+ char filename[PATH_MAX];
530
+ while ((entry = readdir(dir)) != NULL) {
531
+ if ((entry->d_type == DT_REG || entry->d_type == DT_LNK) && strstartswith(entry->d_name, "unittest:") && strendswith(entry->d_name, ".dyncfg")) {
532
+ snprintf(filename, sizeof(filename), "%s/%s", path, entry->d_name);
533
+ nd_log(NDLS_DAEMON, NDLP_INFO, "DYNCFG UNITTEST: deleting file '%s'", filename);
534
+ unlink(filename);
535
+ }
536
+ }
537
+
538
+ closedir(dir);
539
+}
540
+
541
+static TEST *dyncfg_unittest_add(TEST t) {
542
+ dyncfg_unittest_reset();
543
+
544
+ TEST *ret = dictionary_set(dyncfg_unittest_data.nodes, t.id, &t, sizeof(t));
545
+
546
+ if(!dyncfg_add_low_level(localhost, t.id, "/unittests", DYNCFG_STATUS_RUNNING, t.type,
547
+ t.source_type, t.source,
548
+ t.cmds, 0, 0, t.sync, dyncfg_unittest_execute_cb, ret)) {
549
+ dyncfg_unittest_register_error(t.id, "addition of job failed");
550
+ }
551
+
552
+ dyncfg_unittest_check(ret, "plugin create", t.type != DYNCFG_TYPE_TEMPLATE);
553
+
554
+ return ret;
555
+}
556
+
557
+void dyncfg_unittest_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
558
+ TEST *v = value;
559
+ freez((void *)v->id);
560
+ freez((void *)v->source);
561
+}
562
+
563
+int dyncfg_unittest(void) {
564
+ dyncfg_unittest_data.nodes = dictionary_create(DICT_OPTION_NONE);
565
+ dictionary_register_delete_callback(dyncfg_unittest_data.nodes, dyncfg_unittest_delete_cb, NULL);
566
+
567
+ dyncfg_unittest_cleanup_files();
568
+ rrd_functions_inflight_init();
569
+ dyncfg_init(false);
570
+
571
+ // ------------------------------------------------------------------------
572
+ // create the thread for testing async communication
573
+
574
+ netdata_thread_t thread;
575
+ netdata_thread_create(&thread, "unittest", NETDATA_THREAD_OPTION_JOINABLE,
576
+ dyncfg_unittest_thread_action, NULL);
577
+
578
+ // ------------------------------------------------------------------------
579
+ // single
580
+
581
+ TEST *single1 = dyncfg_unittest_add((TEST){
582
+ .id = strdupz("unittest:sync:single1"),
583
+ .source = strdupz(LINE_FILE_STR),
584
+ .type = DYNCFG_TYPE_SINGLE,
585
+ .cmds = DYNCFG_CMD_GET | DYNCFG_CMD_SCHEMA | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
586
+ .source_type = DYNCFG_SOURCE_TYPE_INTERNAL,
587
+ .sync = true,
588
+ .current = {
589
+ .enabled = true,
590
+ },
591
+ .expected = {
592
+ .enabled = true,
593
+ }
594
+ }); (void)single1;
595
+
596
+ TEST *single2 = dyncfg_unittest_add((TEST){
597
+ .id = strdupz("unittest:async:single2"),
598
+ .source = strdupz(LINE_FILE_STR),
599
+ .type = DYNCFG_TYPE_SINGLE,
600
+ .cmds = DYNCFG_CMD_GET | DYNCFG_CMD_SCHEMA | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
601
+ .source_type = DYNCFG_SOURCE_TYPE_INTERNAL,
602
+ .sync = false,
603
+ .current = {
604
+ .enabled = true,
605
+ },
606
+ .expected = {
607
+ .enabled = true,
608
+ }
609
+ }); (void)single2;
610
+
611
+ // ------------------------------------------------------------------------
612
+ // template
613
+
614
+ TEST *template1 = dyncfg_unittest_add((TEST){
615
+ .id = strdupz("unittest:sync:template1"),
616
+ .source = strdupz(LINE_FILE_STR),
617
+ .type = DYNCFG_TYPE_TEMPLATE,
618
+ .cmds = DYNCFG_CMD_SCHEMA | DYNCFG_CMD_ADD | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
619
+ .source_type = DYNCFG_SOURCE_TYPE_INTERNAL,
620
+ .sync = true,
621
+ }); (void)template1;
622
+
623
+ TEST *template2 = dyncfg_unittest_add((TEST){
624
+ .id = strdupz("unittest:async:template2"),
625
+ .source = strdupz(LINE_FILE_STR),
626
+ .type = DYNCFG_TYPE_TEMPLATE,
627
+ .cmds = DYNCFG_CMD_SCHEMA | DYNCFG_CMD_ADD | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
628
+ .source_type = DYNCFG_SOURCE_TYPE_INTERNAL,
629
+ .sync = false,
630
+ }); (void)template2;
631
+
632
+ // ------------------------------------------------------------------------
633
+ // job
634
+
635
+ TEST *user1 = dyncfg_unittest_add((TEST){
636
+ .id = strdupz("unittest:sync:template1:user1"),
637
+ .source = strdupz(LINE_FILE_STR),
638
+ .type = DYNCFG_TYPE_JOB,
639
+ .cmds = DYNCFG_CMD_SCHEMA | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
640
+ .source_type = DYNCFG_SOURCE_TYPE_USER,
641
+ .sync = true,
642
+ .current = {
643
+ .enabled = true,
644
+ },
645
+ .expected = {
646
+ .enabled = true,
647
+ }
648
+ }); (void)user1;
649
+
650
+ TEST *user2 = dyncfg_unittest_add((TEST){
651
+ .id = strdupz("unittest:async:template2:user2"),
652
+ .source = strdupz(LINE_FILE_STR),
653
+ .type = DYNCFG_TYPE_JOB,
654
+ .cmds = DYNCFG_CMD_SCHEMA | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
655
+ .source_type = DYNCFG_SOURCE_TYPE_USER,
656
+ .sync = false,
657
+ .expected = {
658
+ .enabled = true,
659
+ }
660
+ }); (void)user2;
661
+
662
+ // ------------------------------------------------------------------------
663
+
664
+ int rc; (void)rc;
665
+ BUFFER *wb = buffer_create(0, NULL);
666
+
667
+ // ------------------------------------------------------------------------
668
+ // dynamic job
669
+
670
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1 add dyn1", wb, "{\"double\":3.14,\"boolean\":true}", LINE_FILE_STR);
671
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1 add dyn2", wb, "{\"double\":3.14,\"boolean\":true}", LINE_FILE_STR);
672
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2 add dyn3", wb, "{\"double\":3.14,\"boolean\":true}", LINE_FILE_STR);
673
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2 add dyn4", wb, "{\"double\":3.14,\"boolean\":true}", LINE_FILE_STR);
674
+
675
+ // ------------------------------------------------------------------------
676
+ // saving of user_disabled
677
+
678
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:single1 disable", wb, NULL, LINE_FILE_STR);
679
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:single2 disable", wb, NULL, LINE_FILE_STR);
680
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1:user1 disable", wb, NULL, LINE_FILE_STR);
681
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2:user2 disable", wb, NULL, LINE_FILE_STR);
682
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1:dyn1 disable", wb, NULL, LINE_FILE_STR);
683
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1:dyn2 disable", wb, NULL, LINE_FILE_STR);
684
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2:dyn3 disable", wb, NULL, LINE_FILE_STR);
685
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2:dyn4 disable", wb, NULL, LINE_FILE_STR);
686
+
687
+ // ------------------------------------------------------------------------
688
+ // enabling
689
+
690
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:single1 enable", wb, NULL, LINE_FILE_STR);
691
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:single2 enable", wb, NULL, LINE_FILE_STR);
692
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1:user1 enable", wb, NULL, LINE_FILE_STR);
693
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2:user2 enable", wb, NULL, LINE_FILE_STR);
694
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1:dyn1 enable", wb, NULL, LINE_FILE_STR);
695
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1:dyn2 enable", wb, NULL, LINE_FILE_STR);
696
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2:dyn3 enable", wb, NULL, LINE_FILE_STR);
697
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2:dyn4 enable", wb, NULL, LINE_FILE_STR);
698
+
699
+ // ------------------------------------------------------------------------
700
+ // disabling template
701
+
702
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1 disable", wb, NULL, LINE_FILE_STR);
703
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2 disable", wb, NULL, LINE_FILE_STR);
704
+
705
+ // ------------------------------------------------------------------------
706
+ // enabling template
707
+
708
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1 enable", wb, NULL, LINE_FILE_STR);
709
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2 enable", wb, NULL, LINE_FILE_STR);
710
+
711
+ // ------------------------------------------------------------------------
712
+ // adding job on disabled template
713
+
714
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1 disable", wb, NULL, LINE_FILE_STR);
715
+ dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2 disable", wb, NULL, LINE_FILE_STR);
716
+
717
+ TEST *user3 = dyncfg_unittest_add((TEST){
718
+ .id = strdupz("unittest:sync:template1:user3"),
719
+ .source = strdupz(LINE_FILE_STR),
720
+ .type = DYNCFG_TYPE_JOB,
721
+ .cmds = DYNCFG_CMD_SCHEMA | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
722
+ .source_type = DYNCFG_SOURCE_TYPE_USER,
723
+ .sync = true,
724
+ .expected = {
725
+ .enabled = false,
726
+ }
727
+ }); (void)user3;
728
+
729
+ TEST *user4 = dyncfg_unittest_add((TEST){
730
+ .id = strdupz("unittest:async:template2:user4"),
731
+ .source = strdupz(LINE_FILE_STR),
732
+ .type = DYNCFG_TYPE_JOB,
733
+ .cmds = DYNCFG_CMD_SCHEMA | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
734
+ .source_type = DYNCFG_SOURCE_TYPE_USER,
735
+ .sync = false,
736
+ .expected = {
737
+ .enabled = false,
738
+ }
739
+ }); (void)user4;
740
+
741
+ TEST *user5 = dyncfg_unittest_add((TEST){
742
+ .id = strdupz("unittest:sync:template1:user5"),
743
+ .source = strdupz(LINE_FILE_STR),
744
+ .type = DYNCFG_TYPE_JOB,
745
+ .cmds = DYNCFG_CMD_SCHEMA | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
746
+ .source_type = DYNCFG_SOURCE_TYPE_USER,
747
+ .sync = true,
748
+ .expected = {
749
+ .enabled = false,
750
+ }
751
+ }); (void)user5;
752
+
753
+ TEST *user6 = dyncfg_unittest_add((TEST){
754
+ .id = strdupz("unittest:async:template2:user6"),
755
+ .source = strdupz(LINE_FILE_STR),
756
+ .type = DYNCFG_TYPE_JOB,
757
+ .cmds = DYNCFG_CMD_SCHEMA | DYNCFG_CMD_UPDATE | DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE,
758
+ .source_type = DYNCFG_SOURCE_TYPE_USER,
759
+ .sync = false,
760
+ .expected = {
761
+ .enabled = false,
762
+ }
763
+ }); (void)user6;
764
+
765
+// dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1:user5 disable", wb, NULL, LINE_FILE_STR);
766
+// dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2:user6 disable", wb, NULL, LINE_FILE_STR);
767
+
768
+// // ------------------------------------------------------------------------
769
+// // enable template with disabled jobs
770
+//
771
+// user3->expected.enabled = true;
772
+// user5->expected.enabled = false;
773
+// dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:sync:template1 enable", wb, NULL, LINE_FILE_STR);
774
+//
775
+// user4->expected.enabled = true;
776
+// user6->expected.enabled = false;
777
+// dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " unittest:async:template2 enable", wb, NULL, LINE_FILE_STR);
778
+
779
+
780
+// // ------------------------------------------------------------------------
781
+//
782
+// rc = dyncfg_unittest_run(PLUGINSD_FUNCTION_CONFIG " tree", wb, NULL);
783
+// if(rc == HTTP_RESP_OK)
784
+// fprintf(stderr, "%s\n", buffer_tostring(wb));
785
+
786
+ void *ptr;
787
+ netdata_thread_cancel(thread);
788
+ netdata_thread_join(thread, &ptr);
789
+ dyncfg_unittest_cleanup_files();
790
+ dictionary_destroy(dyncfg_unittest_data.nodes);
791
+ return __atomic_load_n(&dyncfg_unittest_data.errors, __ATOMIC_RELAXED) > 0 ? 1 : 0;
792
+}
daemon/config/dyncfg.c
new
+405
@@ -0,0 +1,405 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "dyncfg-internals.h"
4
+#include "dyncfg.h"
5
+
6
+struct dyncfg_globals dyncfg_globals = { 0 };
7
+
8
+void dyncfg_cleanup(DYNCFG *v) {
9
+ buffer_free(v->payload);
10
+ v->payload = NULL;
11
+
12
+ string_freez(v->path);
13
+ v->path = NULL;
14
+
15
+ string_freez(v->source);
16
+ v->source = NULL;
17
+
18
+ string_freez(v->function);
19
+ v->function = NULL;
20
+
21
+ string_freez(v->template);
22
+ v->template = NULL;
23
+}
24
+
25
+static void dyncfg_normalize(DYNCFG *df) {
26
+ usec_t now_ut = now_realtime_usec();
27
+
28
+ if(!df->created_ut)
29
+ df->created_ut = now_ut;
30
+
31
+ if(!df->modified_ut)
32
+ df->modified_ut = now_ut;
33
+}
34
+
35
+static void dyncfg_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
36
+ DYNCFG *df = value;
37
+ dyncfg_cleanup(df);
38
+}
39
+
40
+static void dyncfg_insert_cb(const DICTIONARY_ITEM *item, void *value, void *data __maybe_unused) {
41
+ DYNCFG *df = value;
42
+ dyncfg_normalize(df);
43
+
44
+ const char *id = dictionary_acquired_item_name(item);
45
+ char buf[strlen(id) + 20];
46
+ snprintfz(buf, sizeof(buf), PLUGINSD_FUNCTION_CONFIG " %s", id);
47
+ df->function = string_strdupz(buf);
48
+
49
+ if(df->type == DYNCFG_TYPE_JOB && !df->template) {
50
+ const char *last_colon = strrchr(id, ':');
51
+ if(last_colon)
52
+ df->template = string_strndupz(id, last_colon - id);
53
+ else
54
+ nd_log(NDLS_DAEMON, NDLP_WARNING,
55
+ "DYNCFG: id '%s' is a job, but does not contain a colon to find the template", id);
56
+ }
57
+}
58
+
59
+static void dyncfg_react_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
60
+ DYNCFG *df = value; (void)df;
61
+ ;
62
+}
63
+
64
+static bool dyncfg_conflict_cb(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data __maybe_unused) {
65
+ DYNCFG *v = old_value;
66
+ DYNCFG *nv = new_value;
67
+
68
+ size_t changes = 0;
69
+
70
+ dyncfg_normalize(nv);
71
+
72
+ if(v->host != nv->host) {
73
+ SWAP(v->host, nv->host);
74
+ changes++;
75
+ }
76
+
77
+ if(v->path != nv->path) {
78
+ SWAP(v->path, nv->path);
79
+ changes++;
80
+ }
81
+
82
+ if(v->status != nv->status) {
83
+ SWAP(v->status, nv->status);
84
+ changes++;
85
+ }
86
+
87
+ if(v->type != nv->type) {
88
+ SWAP(v->type, nv->type);
89
+ changes++;
90
+ }
91
+
92
+ if(v->source_type != nv->source_type) {
93
+ SWAP(v->source_type, nv->source_type);
94
+ changes++;
95
+ }
96
+
97
+ if(v->cmds != nv->cmds) {
98
+ SWAP(v->cmds, nv->cmds);
99
+ changes++;
100
+ }
101
+
102
+ if(v->source != nv->source) {
103
+ SWAP(v->source, nv->source);
104
+ changes++;
105
+ }
106
+
107
+ if(nv->created_ut < v->created_ut) {
108
+ SWAP(v->created_ut, nv->created_ut);
109
+ changes++;
110
+ }
111
+
112
+ if(nv->modified_ut > v->modified_ut) {
113
+ SWAP(v->modified_ut, nv->modified_ut);
114
+ changes++;
115
+ }
116
+
117
+ if(v->sync != nv->sync) {
118
+ SWAP(v->sync, nv->sync);
119
+ changes++;
120
+ }
121
+
122
+ if(nv->payload) {
123
+ SWAP(v->payload, nv->payload);
124
+ changes++;
125
+ }
126
+
127
+ if(!v->execute_cb || (nv->overwrite_cb && nv->execute_cb && (v->execute_cb != nv->execute_cb || v->execute_cb_data != nv->execute_cb_data))) {
128
+ v->execute_cb = nv->execute_cb;
129
+ v->execute_cb_data = nv->execute_cb_data;
130
+ changes++;
131
+ }
132
+
133
+ dyncfg_cleanup(nv);
134
+
135
+ return changes > 0;
136
+}
137
+
138
+// ----------------------------------------------------------------------------
139
+
140
+void dyncfg_init_low_level(bool load_saved) {
141
+ if(!dyncfg_globals.nodes) {
142
+ dyncfg_globals.nodes = dictionary_create_advanced(DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, sizeof(DYNCFG));
143
+ dictionary_register_insert_callback(dyncfg_globals.nodes, dyncfg_insert_cb, NULL);
144
+ dictionary_register_react_callback(dyncfg_globals.nodes, dyncfg_react_cb, NULL);
145
+ dictionary_register_conflict_callback(dyncfg_globals.nodes, dyncfg_conflict_cb, NULL);
146
+ dictionary_register_delete_callback(dyncfg_globals.nodes, dyncfg_delete_cb, NULL);
147
+
148
+ char path[PATH_MAX];
149
+ snprintfz(path, sizeof(path), "%s/%s", netdata_configured_varlib_dir, "config");
150
+
151
+ if(mkdir(path, 0755) == -1) {
152
+ if(errno != EEXIST)
153
+ nd_log(NDLS_DAEMON, NDLP_CRIT, "DYNCFG: failed to create dynamic configuration directory '%s'", path);
154
+ }
155
+
156
+ dyncfg_globals.dir = strdupz(path);
157
+
158
+ if(load_saved)
159
+ dyncfg_load_all();
160
+ }
161
+}
162
+
163
+// ----------------------------------------------------------------------------
164
+
165
+const DICTIONARY_ITEM *dyncfg_add_internal(RRDHOST *host, const char *id, const char *path, DYNCFG_STATUS status, DYNCFG_TYPE type, DYNCFG_SOURCE_TYPE source_type, const char *source, DYNCFG_CMDS cmds, usec_t created_ut, usec_t modified_ut, bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data, bool overwrite_cb) {
166
+ DYNCFG tmp = {
167
+ .host = host,
168
+ .path = string_strdupz(path),
169
+ .status = status,
170
+ .type = type,
171
+ .cmds = cmds,
172
+ .source_type = source_type,
173
+ .source = string_strdupz(source),
174
+ .created_ut = created_ut,
175
+ .modified_ut = modified_ut,
176
+ .sync = sync,
177
+ .user_disabled = false,
178
+ .restart_required = false,
179
+ .payload = NULL,
180
+ .execute_cb = execute_cb,
181
+ .execute_cb_data = execute_cb_data,
182
+ .overwrite_cb = overwrite_cb,
183
+ };
184
+ uuid_copy(tmp.host_uuid, host->host_uuid);
185
+
186
+ return dictionary_set_and_acquire_item_advanced(dyncfg_globals.nodes, id, -1, &tmp, sizeof(tmp), NULL);
187
+}
188
+
189
+static void dyncfg_send_updates(const char *id) {
190
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item_advanced(dyncfg_globals.nodes, id, -1);
191
+ if(!item) {
192
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: asked to update plugin for configuration '%s', but it is not found.", id);
193
+ return;
194
+ }
195
+
196
+ DYNCFG *df = dictionary_acquired_item_value(item);
197
+
198
+ if(df->type == DYNCFG_TYPE_SINGLE || df->type == DYNCFG_TYPE_JOB) {
199
+ if (df->cmds & DYNCFG_CMD_UPDATE)
200
+ dyncfg_echo_update(item, df, id);
201
+ }
202
+ else if(df->type == DYNCFG_TYPE_TEMPLATE && (df->cmds & DYNCFG_CMD_ADD)) {
203
+ STRING *template = string_strdupz(id);
204
+
205
+ size_t len = strlen(id);
206
+ DYNCFG *tf;
207
+ dfe_start_reentrant(dyncfg_globals.nodes, tf) {
208
+ const char *t_id = tf_dfe.name;
209
+ if(tf->type == DYNCFG_TYPE_JOB && tf->template == template && strncmp(t_id, id, len) == 0 && t_id[len] == ':' && t_id[len + 1]) {
210
+ dyncfg_echo_add(item, df, id, &t_id[len + 1]);
211
+ }
212
+ }
213
+ dfe_done(tf);
214
+
215
+ string_freez(template);
216
+ }
217
+
218
+ dictionary_acquired_item_release(dyncfg_globals.nodes, item);
219
+}
220
+
221
+bool dyncfg_is_user_disabled(const char *id) {
222
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(dyncfg_globals.nodes, id);
223
+ if(!item)
224
+ return false;
225
+
226
+ DYNCFG *df = dictionary_acquired_item_value(item);
227
+ bool ret = df->user_disabled;
228
+ dictionary_acquired_item_release(dyncfg_globals.nodes, item);
229
+ return ret;
230
+}
231
+
232
+bool dyncfg_job_has_registered_template(const char *id) {
233
+ char buf[strlen(id) + 1];
234
+ memcpy(buf, id, sizeof(buf));
235
+ char *colon = strrchr(buf, ':');
236
+ if(!colon)
237
+ return false;
238
+
239
+ *colon = '\0';
240
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(dyncfg_globals.nodes, buf);
241
+ if(!item)
242
+ return false;
243
+
244
+ DYNCFG *df = dictionary_acquired_item_value(item);
245
+ bool ret = df->type == DYNCFG_TYPE_TEMPLATE;
246
+
247
+ dictionary_acquired_item_release(dyncfg_globals.nodes, item);
248
+ return ret;
249
+}
250
+
251
+bool dyncfg_add_low_level(RRDHOST *host, const char *id, const char *path, DYNCFG_STATUS status, DYNCFG_TYPE type, DYNCFG_SOURCE_TYPE source_type, const char *source, DYNCFG_CMDS cmds, usec_t created_ut, usec_t modified_ut, bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data) {
252
+ if(!dyncfg_is_valid_id(id)) {
253
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: id '%s' is invalid. Ignoring dynamic configuration for it.", id);
254
+ return false;
255
+ }
256
+
257
+ if(type == DYNCFG_TYPE_JOB && !dyncfg_job_has_registered_template(id)) {
258
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: job id '%s' does not have a registered template. Ignoring dynamic configuration for it.", id);
259
+ return false;
260
+ }
261
+
262
+ DYNCFG_CMDS old_cmds = cmds;
263
+
264
+ // all configurations support schema
265
+ cmds |= DYNCFG_CMD_SCHEMA;
266
+
267
+ // if there is either enable or disable, both are supported
268
+ if(cmds & (DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE))
269
+ cmds |= DYNCFG_CMD_ENABLE | DYNCFG_CMD_DISABLE;
270
+
271
+ // add
272
+ if(type == DYNCFG_TYPE_TEMPLATE) {
273
+ // templates must always support "add"
274
+ cmds |= DYNCFG_CMD_ADD;
275
+ }
276
+ else {
277
+ // only templates can have "add"
278
+ cmds &= ~DYNCFG_CMD_ADD;
279
+ }
280
+
281
+ // remove
282
+ if(source_type == DYNCFG_SOURCE_TYPE_DYNCFG && type == DYNCFG_TYPE_JOB) {
283
+ // remove is only available for dyncfg jobs
284
+ cmds |= DYNCFG_CMD_REMOVE;
285
+ }
286
+ else {
287
+ // remove is only available for dyncfg jobs
288
+ cmds &= ~DYNCFG_CMD_REMOVE;
289
+ }
290
+
291
+ // data
292
+ if(type == DYNCFG_TYPE_TEMPLATE) {
293
+ // templates do not have data
294
+ cmds &= ~(DYNCFG_CMD_GET | DYNCFG_CMD_UPDATE | DYNCFG_CMD_TEST);
295
+ }
296
+
297
+ if(cmds != old_cmds) {
298
+ CLEAN_BUFFER *t = buffer_create(1024, NULL);
299
+ buffer_sprintf(t, "DYNCFG: id '%s' was declared with cmds: ", id);
300
+ dyncfg_cmds2buffer(old_cmds, t);
301
+ buffer_strcat(t, ", but they have sanitized to: ");
302
+ dyncfg_cmds2buffer(cmds, t);
303
+ nd_log(NDLS_DAEMON, NDLP_NOTICE, "%s", buffer_tostring(t));
304
+ }
305
+
306
+ const DICTIONARY_ITEM *item = dyncfg_add_internal(host, id, path, status, type, source_type, source, cmds, created_ut, modified_ut, sync, execute_cb, execute_cb_data, true);
307
+ DYNCFG *df = dictionary_acquired_item_value(item);
308
+
309
+// if(df->source_type == DYNCFG_SOURCE_TYPE_DYNCFG && !df->saves)
310
+// nd_log(NDLS_DAEMON, NDLP_WARNING, "DYNCFG: configuration '%s' is created with source type dyncfg, but we don't have a saved configuration for it", id);
311
+
312
+ rrd_collector_started();
313
+ rrd_function_add(
314
+ host,
315
+ NULL,
316
+ string2str(df->function),
317
+ 120,
318
+ 1000,
319
+ "Dynamic configuration",
320
+ "config",
321
+ HTTP_ACCESS_ADMIN,
322
+ sync,
323
+ dyncfg_function_intercept_cb,
324
+ NULL);
325
+
326
+ DYNCFG_STATUS status_to_send_to_plugin = df->user_disabled ? DYNCFG_CMD_DISABLE : DYNCFG_CMD_ENABLE;
327
+ if(status_to_send_to_plugin == DYNCFG_CMD_ENABLE && dyncfg_is_user_disabled(string2str(df->template)))
328
+ status_to_send_to_plugin = DYNCFG_CMD_DISABLE;
329
+
330
+ dyncfg_echo(item, df, id, status_to_send_to_plugin);
331
+ dyncfg_send_updates(id);
332
+ dictionary_acquired_item_release(dyncfg_globals.nodes, item);
333
+
334
+ return true;
335
+}
336
+
337
+void dyncfg_del_low_level(RRDHOST *host, const char *id) {
338
+ if(!dyncfg_is_valid_id(id)) {
339
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: id '%s' is invalid. Ignoring dynamic configuration for it.", id);
340
+ return;
341
+ }
342
+
343
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(dyncfg_globals.nodes, id);
344
+ if(item) {
345
+ DYNCFG *df = dictionary_acquired_item_value(item);
346
+ rrd_function_del(host, NULL, string2str(df->function));
347
+
348
+ bool garbage_collect = false;
349
+ if(df->saves == 0) {
350
+ dictionary_del(dyncfg_globals.nodes, id);
351
+ garbage_collect = true;
352
+ }
353
+
354
+ dictionary_acquired_item_release(dyncfg_globals.nodes, item);
355
+
356
+ if(garbage_collect)
357
+ dictionary_garbage_collect(dyncfg_globals.nodes);
358
+ }
359
+}
360
+
361
+void dyncfg_status_low_level(RRDHOST *host __maybe_unused, const char *id, DYNCFG_STATUS status) {
362
+ if(!dyncfg_is_valid_id(id)) {
363
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: id '%s' is invalid. Ignoring dynamic configuration for it.", id);
364
+ return;
365
+ }
366
+
367
+ if(status == DYNCFG_STATUS_NONE) {
368
+ nd_log(NDLS_DAEMON, NDLP_ERR, "DYNCFG: status provided to id '%s' is invalid. Ignoring it.", id);
369
+ return;
370
+ }
371
+
372
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(dyncfg_globals.nodes, id);
373
+ if(item) {
374
+ DYNCFG *df = dictionary_acquired_item_value(item);
375
+ df->status = status;
376
+ dictionary_acquired_item_release(dyncfg_globals.nodes, item);
377
+ }
378
+}
379
+
380
+// ----------------------------------------------------------------------------
381
+
382
+void dyncfg_add_streaming(BUFFER *wb) {
383
+ // when sending config functions to parents, we send only 1 function called 'config';
384
+ // the parent will send the command to the child, and the child will validate it;
385
+ // this way the parent does not need to receive removals of config functions;
386
+
387
+ buffer_sprintf(wb
388
+ , PLUGINSD_KEYWORD_FUNCTION " GLOBAL " PLUGINSD_FUNCTION_CONFIG " %d \"%s\" \"%s\" \"%s\" %d\n"
389
+ , 120
390
+ , "Dynamic configuration"
391
+ , "config"
392
+ , http_id2access(HTTP_ACCESS_ADMIN)
393
+ , 1000
394
+ );
395
+}
396
+
397
+bool dyncfg_available_for_rrdhost(RRDHOST *host) {
398
+ if(host == localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST))
399
+ return true;
400
+
401
+ return rrd_function_available(host, PLUGINSD_FUNCTION_CONFIG);
402
+}
403
+
404
+// ----------------------------------------------------------------------------
405
+
daemon/config/dyncfg.h
new
+31
@@ -0,0 +1,31 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_DYNCFG_H
4
+#define NETDATA_DYNCFG_H
5
+
6
+#include "../common.h"
7
+#include "../../database/rrd.h"
8
+#include "../../database/rrdfunctions.h"
9
+
10
+void dyncfg_add_streaming(BUFFER *wb);
11
+bool dyncfg_available_for_rrdhost(RRDHOST *host);
12
+void dyncfg_host_init(RRDHOST *host);
13
+
14
+// low-level API used by plugins.d and high-level API
15
+bool dyncfg_add_low_level(RRDHOST *host, const char *id, const char *path, DYNCFG_STATUS status, DYNCFG_TYPE type,
16
+ DYNCFG_SOURCE_TYPE source_type, const char *source, DYNCFG_CMDS cmds,
17
+ usec_t created_ut, usec_t modified_ut, bool sync,
18
+ rrd_function_execute_cb_t execute_cb, void *execute_cb_data);
19
+void dyncfg_del_low_level(RRDHOST *host, const char *id);
20
+void dyncfg_status_low_level(RRDHOST *host, const char *id, DYNCFG_STATUS status);
21
+void dyncfg_init_low_level(bool load_saved);
22
+
23
+// high-level API for internal modules
24
+bool dyncfg_add(RRDHOST *host, const char *id, const char *path, DYNCFG_STATUS status, DYNCFG_TYPE type,
25
+ DYNCFG_SOURCE_TYPE source_type, const char *source, DYNCFG_CMDS cmds, dyncfg_cb_t cb, void *data);
26
+void dyncfg_del(RRDHOST *host, const char *id);
27
+void dyncfg_status(RRDHOST *host, const char *id, DYNCFG_STATUS status);
28
+
29
+void dyncfg_init(bool load_saved);
30
+
31
+#endif //NETDATA_DYNCFG_H
daemon/main.c
+43
-40
@@ -1404,6 +1404,24 @@ void bearer_tokens_init(void);
1404
int unittest_rrdpush_compressions(void);
1405
int uuid_unittest(void);
1406
int progress_unittest(void);
1407
+int dyncfg_unittest(void);
1408
+
1409
+int unittest_prepare_rrd(char **user) {
1410
+ post_conf_load(user);
1411
+ get_netdata_configured_variables();
1412
+ default_rrd_update_every = 1;
1413
+ default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
1414
+ default_health_enabled = 0;
1415
+ storage_tiers = 1;
1416
+ registry_init();
1417
+ if(rrd_init("unittest", NULL, true)) {
1418
+ fprintf(stderr, "rrd_init failed for unittest\n");
1419
+ return 1;
1420
+ }
1421
+ default_rrdpush_enabled = 0;
1422
+
1423
+ return 0;
1424
+}
1425
1426
int main(int argc, char **argv) {
1427
// initialize the system clocks
@@ -1523,49 +1541,28 @@ int main(int argc, char **argv) {
1541
if(strcmp(optarg, "unittest") == 0) {
1542
unittest_running = true;
1543
1526
- if (pluginsd_parser_unittest())
1527
- return 1;
1544
+ if (pluginsd_parser_unittest()) return 1;
1545
+ if (unit_test_static_threads()) return 1;
1546
+ if (unit_test_buffer()) return 1;
1547
+ if (unit_test_str2ld()) return 1;
1548
+ if (buffer_unittest()) return 1;
1549
+ if (unit_test_bitmaps()) return 1;
1550
1529
- if (unit_test_static_threads())
1530
- return 1;
1531
- if (unit_test_buffer())
1532
- return 1;
1533
- if (unit_test_str2ld())
1534
- return 1;
1535
- if (buffer_unittest())
1536
- return 1;
1537
- if (unit_test_bitmaps())
1538
- return 1;
1551
// No call to load the config file on this code-path
1540
- post_conf_load(&user);
1541
- get_netdata_configured_variables();
1542
- default_rrd_update_every = 1;
1543
- default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
1544
- default_health_enabled = 0;
1545
- storage_tiers = 1;
1546
- registry_init();
1547
- if(rrd_init("unittest", NULL, true)) {
1548
- fprintf(stderr, "rrd_init failed for unittest\n");
1549
- return 1;
1550
- }
1551
- default_rrdpush_enabled = 0;
1552
- if(run_all_mockup_tests()) return 1;
1553
- if(unit_test_storage()) return 1;
1552
+ if (unittest_prepare_rrd(&user)) return 1;
1553
+ if (run_all_mockup_tests()) return 1;
1554
+ if (unit_test_storage()) return 1;
1555
#ifdef ENABLE_DBENGINE
1555
- if(test_dbengine()) return 1;
1556
+ if (test_dbengine()) return 1;
1557
#endif
1557
- if(test_sqlite()) return 1;
1558
- if(string_unittest(10000)) return 1;
1559
- if (dictionary_unittest(10000))
1560
- return 1;
1561
- if(aral_unittest(10000))
1562
- return 1;
1563
- if (rrdlabels_unittest())
1564
- return 1;
1565
- if (ctx_unittest())
1566
- return 1;
1567
- if (uuid_unittest())
1568
- return 1;
1558
+ if (test_sqlite()) return 1;
1559
+ if (string_unittest(10000)) return 1;
1560
+ if (dictionary_unittest(10000)) return 1;
1561
+ if (aral_unittest(10000)) return 1;
1562
+ if (rrdlabels_unittest()) return 1;
1563
+ if (ctx_unittest()) return 1;
1564
+ if (uuid_unittest()) return 1;
1565
+ if (dyncfg_unittest()) return 1;
1566
fprintf(stderr, "\n\nALL TESTS PASSED\n\n");
1567
return 0;
1568
}
@@ -1633,6 +1630,12 @@ int main(int argc, char **argv) {
1630
unittest_running = true;
1631
return progress_unittest();
1632
}
1633
+ else if(strcmp(optarg, "dyncfgtest") == 0) {
1634
+ unittest_running = true;
1635
+ if(unittest_prepare_rrd(&user))
1636
+ return 1;
1637
+ return dyncfg_unittest();
1638
+ }
1639
else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
1640
optarg += strlen(createdataset_string);
1641
unsigned history_seconds = strtoul(optarg, NULL, 0);
@@ -2110,7 +2113,7 @@ int main(int argc, char **argv) {
2113
2114
setenv("HOME", netdata_configured_home_dir, 1);
2115
2113
- dyn_conf_init();
2116
+ dyncfg_init(true);
2117
2118
netdata_log_info("netdata started on pid %d.", getpid());
2119
daemon/static_threads.c
-9
@@ -195,15 +195,6 @@ const struct netdata_static_thread static_threads_common[] = {
195
.init_routine = NULL,
196
.start_routine = profile_main
197
},
198
- {
199
- .name = "DYNCFG",
200
- .config_section = NULL,
201
- .config_name = NULL,
202
- .enabled = 1,
203
- .thread = NULL,
204
- .init_routine = NULL,
205
- .start_routine = dyncfg_main
206
- },
198
199
// terminator
200
{
database/contexts/api_v2.c
+12
-1
@@ -729,6 +729,15 @@ static void agent_capabilities_to_json(BUFFER *wb, RRDHOST *host, const char *ke
729
freez(capas);
730
}
731
732
+static inline void host_dyncfg_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) {
733
+ buffer_json_member_add_object(wb, key);
734
+ {
735
+ buffer_json_member_add_string(wb, "status", rrdhost_dyncfg_status_to_string(s->dyncfg.status));
736
+ }
737
+ buffer_json_object_close(wb); // health
738
+
739
+}
740
+
741
static inline void rrdhost_health_to_json_v2(BUFFER *wb, const char *key, RRDHOST_STATUS *s) {
742
buffer_json_member_add_object(wb, key);
743
{
@@ -841,6 +850,8 @@ static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdc
850
851
host_functions2json(host, wb); // functions
852
agent_capabilities_to_json(wb, host, "capabilities");
853
+
854
+ host_dyncfg_to_json_v2(wb, "dyncfg", &s);
855
}
856
buffer_json_object_close(wb); // this instance
857
buffer_json_array_close(wb); // instances
@@ -917,7 +928,7 @@ static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool qu
928
.node_ids = &ctl->nodes.ni,
929
.help = NULL,
930
.tags = NULL,
920
- .access = HTTP_ACCESS_MEMBERS,
931
+ .access = HTTP_ACCESS_MEMBER,
932
.priority = RRDFUNCTIONS_PRIORITY_DEFAULT,
933
};
934
host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help, &t.tags, &t.access, &t.priority);
database/rrd.h
-2
@@ -1358,8 +1358,6 @@ struct rrdhost {
1358
netdata_mutex_t aclk_state_lock;
1359
aclk_rrdhost_state aclk_state;
1360
1361
- DICTIONARY *configurable_plugins; // configurable plugins for this host
1362
-
1361
struct rrdhost *next;
1362
struct rrdhost *prev;
1363
};
database/rrdcalc.c
+5
-5
@@ -190,11 +190,11 @@ const RRDCALC_ACQUIRED *rrdcalc_from_rrdset_get(RRDSET *st, const char *alert_na
190
char key[RRDCALC_MAX_KEY_SIZE + 1];
191
size_t key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_id(st), alert_name);
192
193
- const RRDCALC_ACQUIRED *rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)(key_len + 1));
193
+ const RRDCALC_ACQUIRED *rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)key_len);
194
195
if(!rca) {
196
key_len = rrdcalc_key(key, RRDCALC_MAX_KEY_SIZE, rrdset_name(st), alert_name);
197
- rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)(key_len + 1));
197
+ rca = (const RRDCALC_ACQUIRED *)dictionary_get_and_acquire_item_advanced(st->rrdhost->rrdcalc_root_index, key, (ssize_t)key_len);
198
}
199
200
return rca;
@@ -727,7 +727,7 @@ void rrdcalc_add_from_rrdcalctemplate(RRDHOST *host, RRDCALCTEMPLATE *rt, RRDSET
727
.existing_from_template = false,
728
};
729
730
- dictionary_set_advanced(host->rrdcalc_root_index, key, (ssize_t)(key_len + 1), NULL, sizeof(RRDCALC), &tmp);
730
+ dictionary_set_advanced(host->rrdcalc_root_index, key, (ssize_t)key_len, NULL, sizeof(RRDCALC), &tmp);
731
if(tmp.react_action != RRDCALC_REACT_NEW && tmp.existing_from_template == false)
732
netdata_log_error("RRDCALC: from template '%s' on chart '%s' with key '%s', failed to be added to host '%s'. It is manually configured.",
733
string2str(rt->name), rrdset_id(st), key, rrdhost_hostname(host));
@@ -761,7 +761,7 @@ int rrdcalc_add_from_config(RRDHOST *host, RRDCALC *rc) {
761
};
762
763
int ret = 1;
764
- RRDCALC *t = dictionary_set_advanced(host->rrdcalc_root_index, key, (ssize_t)(key_len + 1), rc, sizeof(RRDCALC), &tmp);
764
+ RRDCALC *t = dictionary_set_advanced(host->rrdcalc_root_index, key, (ssize_t)key_len, rc, sizeof(RRDCALC), &tmp);
765
if(tmp.react_action == RRDCALC_REACT_NEW) {
766
// we copied rc into the dictionary, so we have to free the container here
767
freez(rc);
@@ -795,7 +795,7 @@ static void rrdcalc_unlink_and_delete(RRDHOST *host, RRDCALC *rc, bool having_ll
795
if(rc->rrdset)
796
rrdcalc_unlink_from_rrdset(rc, having_ll_wrlock);
797
798
- dictionary_del_advanced(host->rrdcalc_root_index, string2str(rc->key), (ssize_t)string_strlen(rc->key) + 1);
798
+ dictionary_del_advanced(host->rrdcalc_root_index, string2str(rc->key), (ssize_t)string_strlen(rc->key));
799
}
800
801
database/rrdcalctemplate.c
+1
-1
@@ -231,7 +231,7 @@ void rrdcalctemplate_add_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
231
size_t key_len = snprintfz(key, RRDCALCTEMPLATE_MAX_KEY_SIZE, "%s", rrdcalctemplate_name(rt));
232
233
bool added = false;
234
- dictionary_set_advanced(host->rrdcalctemplate_root_index, key, (ssize_t)(key_len + 1), rt, sizeof(*rt), &added);
234
+ dictionary_set_advanced(host->rrdcalctemplate_root_index, key, (ssize_t)key_len, rt, sizeof(*rt), &added);
235
236
if(added)
237
freez(rt);
database/rrdcollector-internals.h
new
+17
@@ -0,0 +1,17 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_RRDCOLLECTOR_INTERNALS_H
4
+#define NETDATA_RRDCOLLECTOR_INTERNALS_H
5
+
6
+#include "rrd.h"
7
+
8
+struct rrd_collector;
9
+struct rrd_collector *rrd_collector_acquire_current_thread(void);
10
+void rrd_collector_release(struct rrd_collector *rdc);
11
+extern __thread struct rrd_collector *thread_rrd_collector;
12
+bool rrd_collector_running(struct rrd_collector *rdc);
13
+pid_t rrd_collector_tid(struct rrd_collector *rdc);
14
+bool rrd_collector_dispatcher_acquire(struct rrd_collector *rdc);
15
+void rrd_collector_dispatcher_release(struct rrd_collector *rdc);
16
+
17
+#endif //NETDATA_RRDCOLLECTOR_INTERNALS_H
database/rrdcollector.c
+1
-1
@@ -1,7 +1,7 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
-#define NETDATA_RRDCOLLECTOR_INTERNALS
3
#include "rrdcollector.h"
4
+#include "rrdcollector-internals.h"
5
6
// Each function points to this collector structure
7
// so that when the collector exits, all of them will
database/rrdcollector.h
-16
@@ -5,22 +5,6 @@
5
6
#include "rrd.h"
7
8
-#ifdef NETDATA_RRDCOLLECTOR_INTERNALS
9
-
10
-// ----------------------------------------------------------------------------
11
-// private API
12
-
13
-struct rrd_collector;
14
-struct rrd_collector *rrd_collector_acquire_current_thread(void);
15
-void rrd_collector_release(struct rrd_collector *rdc);
16
-extern __thread struct rrd_collector *thread_rrd_collector;
17
-bool rrd_collector_running(struct rrd_collector *rdc);
18
-pid_t rrd_collector_tid(struct rrd_collector *rdc);
19
-bool rrd_collector_dispatcher_acquire(struct rrd_collector *rdc);
20
-void rrd_collector_dispatcher_release(struct rrd_collector *rdc);
21
-
22
-#endif // NETDATA_RRDCOLLECTOR_INTERNALS
23
-
8
// ----------------------------------------------------------------------------
9
// public API
10
database/rrddimvar.c
+1
-1
@@ -243,7 +243,7 @@ void rrddimvar_add_and_leave_released(RRDDIM *rd, RRDVAR_TYPE type, const char *
243
.value = value,
244
.rrddim = rd
245
};
246
- dictionary_set_advanced(rd->rrdset->rrddimvar_root_index, key, (ssize_t)(key_len + 1), NULL, sizeof(RRDDIMVAR), &tmp);
246
+ dictionary_set_advanced(rd->rrdset->rrddimvar_root_index, key, (ssize_t)key_len, NULL, sizeof(RRDDIMVAR), &tmp);
247
}
248
249
void rrddimvar_rename_all(RRDDIM *rd) {
database/rrdfunctions-exporters.c
new
+164
@@ -0,0 +1,164 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#define NETDATA_RRD_INTERNALS
4
+
5
+#include "rrdfunctions-internals.h"
6
+#include "rrdfunctions-exporters.h"
7
+
8
+void rrd_chart_functions_expose_rrdpush(RRDSET *st, BUFFER *wb) {
9
+ if(!st->functions_view)
10
+ return;
11
+
12
+ struct rrd_host_function *t;
13
+ dfe_start_read(st->functions_view, t) {
14
+ if(t->options & RRD_FUNCTION_DYNCFG) continue;
15
+
16
+ buffer_sprintf(wb
17
+ , PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\" \"%s\" \"%s\" %d\n"
18
+ , t_dfe.name
19
+ , t->timeout
20
+ , string2str(t->help)
21
+ , string2str(t->tags)
22
+ , http_id2access(t->access)
23
+ ,
24
+ t->priority
25
+ );
26
+ }
27
+ dfe_done(t);
28
+}
29
+
30
+void rrd_global_functions_expose_rrdpush(RRDHOST *host, BUFFER *wb, bool dyncfg) {
31
+ rrdhost_flag_clear(host, RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED);
32
+
33
+ size_t configs = 0;
34
+
35
+ struct rrd_host_function *tmp;
36
+ dfe_start_read(host->functions, tmp) {
37
+ if(tmp->options & RRD_FUNCTION_LOCAL) continue;
38
+ if(tmp->options & RRD_FUNCTION_DYNCFG) {
39
+ // we should not send dyncfg to this parent
40
+ configs++;
41
+ continue;
42
+ }
43
+
44
+ buffer_sprintf(wb
45
+ , PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"%s\" \"%s\" %d\n"
46
+ , tmp_dfe.name
47
+ , tmp->timeout
48
+ , string2str(tmp->help)
49
+ , string2str(tmp->tags)
50
+ , http_id2access(tmp->access)
51
+ , tmp->priority
52
+ );
53
+ }
54
+ dfe_done(tmp);
55
+
56
+ if(dyncfg && configs)
57
+ dyncfg_add_streaming(wb);
58
+}
59
+
60
+static void functions2json(DICTIONARY *functions, BUFFER *wb) {
61
+ struct rrd_host_function *t;
62
+ dfe_start_read(functions, t) {
63
+ if (!rrd_collector_running(t->collector)) continue;
64
+ if(t->options & RRD_FUNCTION_DYNCFG) continue;
65
+
66
+ buffer_json_member_add_object(wb, t_dfe.name);
67
+ {
68
+ buffer_json_member_add_string_or_empty(wb, "help", string2str(t->help));
69
+ buffer_json_member_add_int64(wb, "timeout", (int64_t) t->timeout);
70
+
71
+ char options[65];
72
+ snprintfz(
73
+ options, 64
74
+ , "%s%s"
75
+ , (t->options & RRD_FUNCTION_LOCAL) ? "LOCAL " : ""
76
+ , (t->options & RRD_FUNCTION_GLOBAL) ? "GLOBAL" : ""
77
+ );
78
+
79
+ buffer_json_member_add_string_or_empty(wb, "options", options);
80
+ buffer_json_member_add_string_or_empty(wb, "tags", string2str(t->tags));
81
+ buffer_json_member_add_string(wb, "access", http_id2access(t->access));
82
+ buffer_json_member_add_uint64(wb, "priority", t->priority);
83
+ }
84
+ buffer_json_object_close(wb);
85
+ }
86
+ dfe_done(t);
87
+}
88
+
89
+void chart_functions2json(RRDSET *st, BUFFER *wb) {
90
+ if(!st || !st->functions_view) return;
91
+
92
+ functions2json(st->functions_view, wb);
93
+}
94
+
95
+void host_functions2json(RRDHOST *host, BUFFER *wb) {
96
+ if(!host || !host->functions) return;
97
+
98
+ buffer_json_member_add_object(wb, "functions");
99
+
100
+ struct rrd_host_function *t;
101
+ dfe_start_read(host->functions, t) {
102
+ if(!rrd_collector_running(t->collector)) continue;
103
+ if(t->options & RRD_FUNCTION_DYNCFG) continue;
104
+
105
+ buffer_json_member_add_object(wb, t_dfe.name);
106
+ {
107
+ buffer_json_member_add_string(wb, "help", string2str(t->help));
108
+ buffer_json_member_add_int64(wb, "timeout", t->timeout);
109
+ buffer_json_member_add_array(wb, "options");
110
+ {
111
+ if (t->options & RRD_FUNCTION_GLOBAL)
112
+ buffer_json_add_array_item_string(wb, "GLOBAL");
113
+ if (t->options & RRD_FUNCTION_LOCAL)
114
+ buffer_json_add_array_item_string(wb, "LOCAL");
115
+ }
116
+ buffer_json_array_close(wb);
117
+ buffer_json_member_add_string(wb, "tags", string2str(t->tags));
118
+ buffer_json_member_add_string(wb, "access", http_id2access(t->access));
119
+ buffer_json_member_add_uint64(wb, "priority", t->priority);
120
+ }
121
+ buffer_json_object_close(wb);
122
+ }
123
+ dfe_done(t);
124
+
125
+ buffer_json_object_close(wb);
126
+}
127
+
128
+void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst, void *value, size_t value_size) {
129
+ if(!rrdset_functions_view || !dst) return;
130
+
131
+ struct rrd_host_function *t;
132
+ dfe_start_read(rrdset_functions_view, t) {
133
+ if(!rrd_collector_running(t->collector)) continue;
134
+ if(t->options & RRD_FUNCTION_DYNCFG) continue;
135
+
136
+ dictionary_set(dst, t_dfe.name, value, value_size);
137
+ }
138
+ dfe_done(t);
139
+}
140
+
141
+void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help, STRING **tags, HTTP_ACCESS *access, int *priority) {
142
+ if(!host || !host->functions || !dictionary_entries(host->functions) || !dst) return;
143
+
144
+ struct rrd_host_function *t;
145
+ dfe_start_read(host->functions, t) {
146
+ if(!rrd_collector_running(t->collector)) continue;
147
+ if(t->options & RRD_FUNCTION_DYNCFG) continue;
148
+
149
+ if(help)
150
+ *help = t->help;
151
+
152
+ if(tags)
153
+ *tags = t->tags;
154
+
155
+ if(access)
156
+ *access = t->access;
157
+
158
+ if(priority)
159
+ *priority = t->priority;
160
+
161
+ dictionary_set(dst, t_dfe.name, value, value_size);
162
+ }
163
+ dfe_done(t);
164
+}
database/rrdfunctions-exporters.h
new
+16
@@ -0,0 +1,16 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_RRDFUNCTIONS_EXPORTERS_H
4
+#define NETDATA_RRDFUNCTIONS_EXPORTERS_H
5
+
6
+#include "rrd.h"
7
+
8
+void rrd_chart_functions_expose_rrdpush(RRDSET *st, BUFFER *wb);
9
+void rrd_global_functions_expose_rrdpush(RRDHOST *host, BUFFER *wb, bool dyncfg);
10
+
11
+void chart_functions2json(RRDSET *st, BUFFER *wb);
12
+void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst, void *value, size_t value_size);
13
+void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help, STRING **tags, HTTP_ACCESS *access, int *priority);
14
+void host_functions2json(RRDHOST *host, BUFFER *wb);
15
+
16
+#endif //NETDATA_RRDFUNCTIONS_EXPORTERS_H
database/rrdfunctions-inflight.c
new
+641
@@ -0,0 +1,641 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#define NETDATA_RRD_INTERNALS
4
+
5
+#include "rrdcollector-internals.h"
6
+#include "rrdfunctions-internals.h"
7
+#include "rrdfunctions-inflight.h"
8
+
9
+struct rrd_function_inflight {
10
+ bool used;
11
+
12
+ RRDHOST *host;
13
+ uuid_t transaction_uuid;
14
+ const char *transaction;
15
+ const char *cmd;
16
+ const char *sanitized_cmd;
17
+ const char *source;
18
+ size_t sanitized_cmd_length;
19
+ int timeout;
20
+ bool cancelled;
21
+ usec_t stop_monotonic_ut;
22
+
23
+ BUFFER *payload;
24
+
25
+ const DICTIONARY_ITEM *host_function_acquired;
26
+
27
+ // the collector
28
+ // we acquire this structure at the beginning,
29
+ // and we release it at the end
30
+ struct rrd_host_function *rdcf;
31
+
32
+ struct {
33
+ BUFFER *wb;
34
+
35
+ // in async mode,
36
+ // the function to call to send the result back
37
+ rrd_function_result_callback_t cb;
38
+ void *data;
39
+ } result;
40
+
41
+ struct {
42
+ // to be called in sync mode
43
+ // while the function is running
44
+ // to check if the function has been canceled
45
+ rrd_function_is_cancelled_cb_t cb;
46
+ void *data;
47
+ } is_cancelled;
48
+
49
+ struct {
50
+ // to be registered by the function itself
51
+ // used to signal the function to cancel
52
+ rrd_function_cancel_cb_t cb;
53
+ void *data;
54
+ } canceller;
55
+
56
+ struct {
57
+ // callback to receive progress reports from function
58
+ rrd_function_progress_cb_t cb;
59
+ void *data;
60
+ } progress;
61
+
62
+ struct {
63
+ // to be registered by the function itself
64
+ // used to send progress requests to function
65
+ rrd_function_progresser_cb_t cb;
66
+ void *data;
67
+ } progresser;
68
+};
69
+
70
+static DICTIONARY *rrd_functions_inflight_requests = NULL;
71
+
72
+static void rrd_function_cancel_inflight(struct rrd_function_inflight *r);
73
+
74
+// ----------------------------------------------------------------------------
75
+
76
+static void rrd_functions_inflight_cleanup(struct rrd_function_inflight *r) {
77
+ buffer_free(r->payload);
78
+ freez((void *)r->transaction);
79
+ freez((void *)r->cmd);
80
+ freez((void *)r->sanitized_cmd);
81
+ freez((void *)r->source);
82
+
83
+ r->payload = NULL;
84
+ r->transaction = NULL;
85
+ r->cmd = NULL;
86
+ r->sanitized_cmd = NULL;
87
+}
88
+
89
+static void rrd_functions_inflight_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
90
+ struct rrd_function_inflight *r = value;
91
+
92
+ // internal_error(true, "FUNCTIONS: transaction '%s' finished", r->transaction);
93
+
94
+ rrd_functions_inflight_cleanup(r);
95
+ dictionary_acquired_item_release(r->host->functions, r->host_function_acquired);
96
+}
97
+
98
+void rrd_functions_inflight_init(void) {
99
+ if(rrd_functions_inflight_requests)
100
+ return;
101
+
102
+ rrd_functions_inflight_requests = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct rrd_function_inflight));
103
+
104
+ dictionary_register_delete_callback(rrd_functions_inflight_requests, rrd_functions_inflight_delete_cb, NULL);
105
+}
106
+
107
+void rrd_functions_inflight_destroy(void) {
108
+ if(!rrd_functions_inflight_requests)
109
+ return;
110
+
111
+ dictionary_destroy(rrd_functions_inflight_requests);
112
+ rrd_functions_inflight_requests = NULL;
113
+}
114
+
115
+static void rrd_inflight_async_function_register_canceller_cb(void *register_canceller_cb_data, rrd_function_cancel_cb_t canceller_cb, void *canceller_cb_data) {
116
+ struct rrd_function_inflight *r = register_canceller_cb_data;
117
+ r->canceller.cb = canceller_cb;
118
+ r->canceller.data = canceller_cb_data;
119
+}
120
+
121
+static void rrd_inflight_async_function_register_progresser_cb(void *register_progresser_cb_data, rrd_function_progresser_cb_t progresser_cb, void *progresser_cb_data) {
122
+ struct rrd_function_inflight *r = register_progresser_cb_data;
123
+ r->progresser.cb = progresser_cb;
124
+ r->progresser.data = progresser_cb_data;
125
+}
126
+
127
+// ----------------------------------------------------------------------------
128
+// waiting for async function completion
129
+
130
+struct rrd_function_call_wait {
131
+ RRDHOST *host;
132
+ const DICTIONARY_ITEM *host_function_acquired;
133
+ char *transaction;
134
+
135
+ bool free_with_signal;
136
+ bool data_are_ready;
137
+ netdata_mutex_t mutex;
138
+ pthread_cond_t cond;
139
+ int code;
140
+};
141
+
142
+static void rrd_inflight_function_cleanup(RRDHOST *host __maybe_unused, const char *transaction) {
143
+ dictionary_del(rrd_functions_inflight_requests, transaction);
144
+ dictionary_garbage_collect(rrd_functions_inflight_requests);
145
+}
146
+
147
+static void rrd_function_call_wait_free(struct rrd_function_call_wait *tmp) {
148
+ rrd_inflight_function_cleanup(tmp->host, tmp->transaction);
149
+ freez(tmp->transaction);
150
+
151
+ pthread_cond_destroy(&tmp->cond);
152
+ netdata_mutex_destroy(&tmp->mutex);
153
+ freez(tmp);
154
+}
155
+
156
+static void rrd_async_function_signal_when_ready(BUFFER *temp_wb __maybe_unused, int code, void *callback_data) {
157
+ struct rrd_function_call_wait *tmp = callback_data;
158
+ bool we_should_free = false;
159
+
160
+ netdata_mutex_lock(&tmp->mutex);
161
+
162
+ // since we got the mutex,
163
+ // the waiting thread is either in pthread_cond_timedwait()
164
+ // or gave up and left.
165
+
166
+ tmp->code = code;
167
+ tmp->data_are_ready = true;
168
+
169
+ if(tmp->free_with_signal)
170
+ we_should_free = true;
171
+
172
+ pthread_cond_signal(&tmp->cond);
173
+
174
+ netdata_mutex_unlock(&tmp->mutex);
175
+
176
+ if(we_should_free) {
177
+ buffer_free(temp_wb);
178
+ rrd_function_call_wait_free(tmp);
179
+ }
180
+}
181
+
182
+static void rrd_inflight_async_function_nowait_finished(BUFFER *wb, int code, void *data) {
183
+ struct rrd_function_inflight *r = data;
184
+
185
+ if(r->result.cb)
186
+ r->result.cb(wb, code, r->result.data);
187
+
188
+ rrd_inflight_function_cleanup(r->host, r->transaction);
189
+}
190
+
191
+static bool rrd_inflight_async_function_is_cancelled(void *data) {
192
+ struct rrd_function_inflight *r = data;
193
+ return __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED);
194
+}
195
+
196
+static inline int rrd_call_function_async_and_dont_wait(struct rrd_function_inflight *r) {
197
+ struct rrd_function_execute rfe = {
198
+ .transaction = &r->transaction_uuid,
199
+ .function = r->sanitized_cmd,
200
+ .payload = r->payload,
201
+ .source = r->source,
202
+ .stop_monotonic_ut = &r->stop_monotonic_ut,
203
+ .result = {
204
+ .wb = r->result.wb,
205
+ .cb = rrd_inflight_async_function_nowait_finished,
206
+ .data = r,
207
+ },
208
+ .progress = {
209
+ .cb = r->progress.cb,
210
+ .data = r->progress.data,
211
+ },
212
+ .is_cancelled = {
213
+ .cb = rrd_inflight_async_function_is_cancelled,
214
+ .data = r,
215
+ },
216
+ .register_canceller = {
217
+ .cb = rrd_inflight_async_function_register_canceller_cb,
218
+ .data = r,
219
+ },
220
+ .register_progresser = {
221
+ .cb = rrd_inflight_async_function_register_progresser_cb,
222
+ .data = r,
223
+ },
224
+ };
225
+ int code = r->rdcf->execute_cb(&rfe, r->rdcf->execute_cb_data);
226
+
227
+ return code;
228
+}
229
+
230
+static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) {
231
+ struct rrd_function_call_wait *tmp = mallocz(sizeof(struct rrd_function_call_wait));
232
+ tmp->free_with_signal = false;
233
+ tmp->data_are_ready = false;
234
+ tmp->host = r->host;
235
+ tmp->host_function_acquired = r->host_function_acquired;
236
+ tmp->transaction = strdupz(r->transaction);
237
+ netdata_mutex_init(&tmp->mutex);
238
+ pthread_cond_init(&tmp->cond, NULL);
239
+
240
+ // we need a temporary BUFFER, because we may time out and the caller supplied one may vanish,
241
+ // so we create a new one we guarantee will survive until the collector finishes...
242
+
243
+ bool we_should_free = false;
244
+ BUFFER *temp_wb = buffer_create(1024, &netdata_buffers_statistics.buffers_functions); // we need it because we may give up on it
245
+ temp_wb->content_type = r->result.wb->content_type;
246
+
247
+ struct rrd_function_execute rfe = {
248
+ .transaction = &r->transaction_uuid,
249
+ .function = r->sanitized_cmd,
250
+ .payload = r->payload,
251
+ .source = r->source,
252
+ .stop_monotonic_ut = &r->stop_monotonic_ut,
253
+ .result = {
254
+ .wb = temp_wb,
255
+
256
+ // we overwrite the result callbacks,
257
+ // so that we can clean up the allocations made
258
+ .cb = rrd_async_function_signal_when_ready,
259
+ .data = tmp,
260
+ },
261
+ .progress = {
262
+ .cb = r->progress.cb,
263
+ .data = r->progress.data,
264
+ },
265
+ .is_cancelled = {
266
+ .cb = rrd_inflight_async_function_is_cancelled,
267
+ .data = r,
268
+ },
269
+ .register_canceller = {
270
+ .cb = rrd_inflight_async_function_register_canceller_cb,
271
+ .data = r,
272
+ },
273
+ .register_progresser = {
274
+ .cb = rrd_inflight_async_function_register_progresser_cb,
275
+ .data = r,
276
+ },
277
+ };
278
+ int code = r->rdcf->execute_cb(&rfe, r->rdcf->execute_cb_data);
279
+
280
+ // this has to happen after we execute the callback
281
+ // because if an async call is responded in sync mode, there will be a deadlock.
282
+ netdata_mutex_lock(&tmp->mutex);
283
+
284
+ if (code == HTTP_RESP_OK || tmp->data_are_ready) {
285
+ bool cancelled = false;
286
+ int rc = 0;
287
+ while (rc == 0 && !cancelled && !tmp->data_are_ready) {
288
+ usec_t now_mono_ut = now_monotonic_usec();
289
+ usec_t stop_mono_ut = __atomic_load_n(&r->stop_monotonic_ut, __ATOMIC_RELAXED) + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
290
+ if(now_mono_ut > stop_mono_ut) {
291
+ rc = ETIMEDOUT;
292
+ break;
293
+ }
294
+
295
+ // wait for 10ms, and loop again...
296
+ struct timespec tp;
297
+ clock_gettime(CLOCK_REALTIME, &tp);
298
+ tp.tv_nsec += 10 * NSEC_PER_MSEC;
299
+ if(tp.tv_nsec > (long)(1 * NSEC_PER_SEC)) {
300
+ tp.tv_sec++;
301
+ tp.tv_nsec -= 1 * NSEC_PER_SEC;
302
+ }
303
+
304
+ // the mutex is unlocked within pthread_cond_timedwait()
305
+ rc = pthread_cond_timedwait(&tmp->cond, &tmp->mutex, &tp);
306
+ // the mutex is again ours
307
+
308
+ if(rc == ETIMEDOUT) {
309
+ // 10ms have passed
310
+
311
+ rc = 0;
312
+ if (!tmp->data_are_ready && r->is_cancelled.cb &&
313
+ r->is_cancelled.cb(r->is_cancelled.data)) {
314
+ // internal_error(true, "FUNCTIONS: transaction '%s' is cancelled while waiting for response",
315
+ // r->transaction);
316
+ cancelled = true;
317
+ rrd_function_cancel_inflight(r);
318
+ break;
319
+ }
320
+ }
321
+ }
322
+
323
+ if (tmp->data_are_ready) {
324
+ // we have a response
325
+
326
+ buffer_contents_replace(r->result.wb, buffer_tostring(temp_wb), buffer_strlen(temp_wb));
327
+ r->result.wb->content_type = temp_wb->content_type;
328
+ r->result.wb->expires = temp_wb->expires;
329
+
330
+ if(r->result.wb->expires)
331
+ buffer_cacheable(r->result.wb);
332
+ else
333
+ buffer_no_cacheable(r->result.wb);
334
+
335
+ code = tmp->code;
336
+
337
+ tmp->free_with_signal = false;
338
+ we_should_free = true;
339
+ }
340
+ else if (rc == ETIMEDOUT || cancelled) {
341
+ // timeout
342
+ // we will go away and let the callback free the structure
343
+
344
+ if(cancelled)
345
+ code = rrd_call_function_error(r->result.wb,
346
+ "Request cancelled",
347
+ HTTP_RESP_CLIENT_CLOSED_REQUEST);
348
+ else
349
+ code = rrd_call_function_error(r->result.wb,
350
+ "Timeout while waiting for a response from the collector.",
351
+ HTTP_RESP_GATEWAY_TIMEOUT);
352
+
353
+ tmp->free_with_signal = true;
354
+ we_should_free = false;
355
+ }
356
+ else {
357
+ code = rrd_call_function_error(
358
+ r->result.wb, "Internal error while communicating with the collector",
359
+ HTTP_RESP_INTERNAL_SERVER_ERROR);
360
+
361
+ tmp->free_with_signal = true;
362
+ we_should_free = false;
363
+ }
364
+ }
365
+ else {
366
+ // the response is not ok, and we don't have the data
367
+ tmp->free_with_signal = true;
368
+ we_should_free = false;
369
+ }
370
+
371
+ netdata_mutex_unlock(&tmp->mutex);
372
+
373
+ if (we_should_free) {
374
+ rrd_function_call_wait_free(tmp);
375
+ buffer_free(temp_wb);
376
+ }
377
+
378
+ return code;
379
+}
380
+
381
+static inline int rrd_call_function_async(struct rrd_function_inflight *r, bool wait) {
382
+ if(wait)
383
+ return rrd_call_function_async_and_wait(r);
384
+ else
385
+ return rrd_call_function_async_and_dont_wait(r);
386
+}
387
+
388
+
389
+// ----------------------------------------------------------------------------
390
+
391
+int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, HTTP_ACCESS access, const char *cmd,
392
+ bool wait, const char *transaction,
393
+ rrd_function_result_callback_t result_cb, void *result_cb_data,
394
+ rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
395
+ rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
396
+ BUFFER *payload, const char *source) {
397
+
398
+ int code;
399
+ char sanitized_cmd[PLUGINSD_LINE_MAX + 1];
400
+ const DICTIONARY_ITEM *host_function_acquired = NULL;
401
+
402
+ char sanitized_source[(source ? strlen(source) : 0) + 1];
403
+ rrd_functions_sanitize(sanitized_source, source ? source : "", sizeof(sanitized_source));
404
+
405
+ // ------------------------------------------------------------------------
406
+ // find the function
407
+
408
+ size_t sanitized_cmd_length = rrd_functions_sanitize(sanitized_cmd, cmd, sizeof(sanitized_cmd));
409
+
410
+ code = rrd_functions_find_by_name(host, result_wb, sanitized_cmd, sanitized_cmd_length, &host_function_acquired);
411
+ if(code != HTTP_RESP_OK) {
412
+ rrd_call_function_error(result_wb, "not found", code);
413
+
414
+ if(result_cb)
415
+ result_cb(result_wb, code, result_cb_data);
416
+
417
+ return code;
418
+ }
419
+
420
+ struct rrd_host_function *rdcf = dictionary_acquired_item_value(host_function_acquired);
421
+
422
+ if(!web_client_has_enough_access_level(access, rdcf->access)) {
423
+
424
+ if(!aclk_connected)
425
+ rrd_call_function_error(result_wb, "This Netdata must be connected to Netdata Cloud to access this function.", HTTP_RESP_PRECOND_FAIL);
426
+ else if(access >= HTTP_ACCESS_ANY)
427
+ rrd_call_function_error(result_wb, "You need to login to the Netdata Cloud space this agent is claimed to, to access this function.", HTTP_RESP_PRECOND_FAIL);
428
+ else /* if(access < HTTP_ACCESS_ANY && rdcf->access < access) */
429
+ rrd_call_function_error(result_wb, "To access this function you need to be an admin in this Netdata Cloud space.", HTTP_RESP_PRECOND_FAIL);
430
+
431
+ dictionary_acquired_item_release(host->functions, host_function_acquired);
432
+
433
+ if(result_cb)
434
+ result_cb(result_wb, HTTP_RESP_PRECOND_FAIL, result_cb_data);
435
+
436
+ return HTTP_RESP_PRECOND_FAIL;
437
+ }
438
+
439
+ if(timeout_s <= 0)
440
+ timeout_s = rdcf->timeout;
441
+
442
+ // ------------------------------------------------------------------------
443
+ // validate and parse the transaction, or generate a new transaction id
444
+
445
+ char uuid_str[UUID_COMPACT_STR_LEN];
446
+ uuid_t uuid;
447
+
448
+ if(!transaction || !*transaction || uuid_parse_flexi(transaction, uuid) != 0)
449
+ uuid_generate_random(uuid);
450
+
451
+ uuid_unparse_lower_compact(uuid, uuid_str);
452
+ transaction = uuid_str;
453
+
454
+ // ------------------------------------------------------------------------
455
+ // the function can only be executed in async mode
456
+ // put the function into the inflight requests
457
+
458
+ struct rrd_function_inflight t = {
459
+ .used = false,
460
+ .host = host,
461
+ .cmd = strdupz(cmd),
462
+ .sanitized_cmd = strdupz(sanitized_cmd),
463
+ .sanitized_cmd_length = sanitized_cmd_length,
464
+ .transaction = strdupz(transaction),
465
+ .source = strdupz(sanitized_source),
466
+ .payload = buffer_dup(payload),
467
+ .timeout = timeout_s,
468
+ .cancelled = false,
469
+ .stop_monotonic_ut = now_monotonic_usec() + timeout_s * USEC_PER_SEC,
470
+ .host_function_acquired = host_function_acquired,
471
+ .rdcf = rdcf,
472
+ .result = {
473
+ .wb = result_wb,
474
+ .cb = result_cb,
475
+ .data = result_cb_data,
476
+ },
477
+ .is_cancelled = {
478
+ .cb = is_cancelled_cb,
479
+ .data = is_cancelled_cb_data,
480
+ },
481
+ .progress = {
482
+ .cb = progress_cb,
483
+ .data = progress_cb_data,
484
+ },
485
+ };
486
+ uuid_copy(t.transaction_uuid, uuid);
487
+
488
+ struct rrd_function_inflight *r = dictionary_set(rrd_functions_inflight_requests, transaction, &t, sizeof(t));
489
+ if(r->used) {
490
+ nd_log(NDLS_DAEMON, NDLP_NOTICE,
491
+ "FUNCTIONS: duplicate transaction '%s', function: '%s'",
492
+ t.transaction, t.cmd);
493
+
494
+ code = rrd_call_function_error(result_wb, "duplicate transaction", HTTP_RESP_BAD_REQUEST);
495
+
496
+ rrd_functions_inflight_cleanup(&t);
497
+ dictionary_acquired_item_release(r->host->functions, t.host_function_acquired);
498
+
499
+ if(result_cb)
500
+ result_cb(result_wb, code, result_cb_data);
501
+
502
+ return code;
503
+ }
504
+ r->used = true;
505
+ // internal_error(true, "FUNCTIONS: transaction '%s' started", r->transaction);
506
+
507
+ if(r->rdcf->sync) {
508
+ // the caller has to wait
509
+
510
+ struct rrd_function_execute rfe = {
511
+ .transaction = &r->transaction_uuid,
512
+ .function = r->sanitized_cmd,
513
+ .payload = r->payload,
514
+ .source = r->source,
515
+ .stop_monotonic_ut = &r->stop_monotonic_ut,
516
+ .result = {
517
+ .wb = r->result.wb,
518
+
519
+ // we overwrite the result callbacks,
520
+ // so that we can clean up the allocations made
521
+ .cb = r->result.cb,
522
+ .data = r->result.data,
523
+ },
524
+ .progress = {
525
+ .cb = r->progress.cb,
526
+ .data = r->progress.data,
527
+ },
528
+ .is_cancelled = {
529
+ .cb = r->is_cancelled.cb,
530
+ .data = r->is_cancelled.data,
531
+ },
532
+ .register_canceller = {
533
+ .cb = NULL,
534
+ .data = NULL,
535
+ },
536
+ .register_progresser = {
537
+ .cb = NULL,
538
+ .data = NULL,
539
+ },
540
+ };
541
+ code = r->rdcf->execute_cb(&rfe, r->rdcf->execute_cb_data);
542
+
543
+ rrd_inflight_function_cleanup(host, r->transaction);
544
+ return code;
545
+ }
546
+
547
+ return rrd_call_function_async(r, wait);
548
+}
549
+
550
+bool rrd_function_has_this_original_result_callback(uuid_t *transaction, rrd_function_result_callback_t cb) {
551
+ bool ret = false;
552
+ char str[UUID_COMPACT_STR_LEN];
553
+ uuid_unparse_lower_compact(*transaction, str);
554
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, str);
555
+ if(item) {
556
+ struct rrd_function_inflight *r = dictionary_acquired_item_value(item);
557
+ if(r->result.cb == cb)
558
+ ret = true;
559
+
560
+ dictionary_acquired_item_release(rrd_functions_inflight_requests, item);
561
+ }
562
+ return ret;
563
+}
564
+
565
+static void rrd_function_cancel_inflight(struct rrd_function_inflight *r) {
566
+ if(!r)
567
+ return;
568
+
569
+ bool cancelled = __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED);
570
+ if(cancelled) {
571
+ nd_log(NDLS_DAEMON, NDLP_DEBUG,
572
+ "FUNCTIONS: received a CANCEL request for transaction '%s', but it is already cancelled.",
573
+ r->transaction);
574
+ return;
575
+ }
576
+
577
+ __atomic_store_n(&r->cancelled, true, __ATOMIC_RELAXED);
578
+
579
+ if(!rrd_collector_dispatcher_acquire(r->rdcf->collector)) {
580
+ nd_log(NDLS_DAEMON, NDLP_DEBUG,
581
+ "FUNCTIONS: received a CANCEL request for transaction '%s', but the collector is not running.",
582
+ r->transaction);
583
+ return;
584
+ }
585
+
586
+ if(r->canceller.cb)
587
+ r->canceller.cb(r->canceller.data);
588
+
589
+ rrd_collector_dispatcher_release(r->rdcf->collector);
590
+}
591
+
592
+void rrd_function_cancel(const char *transaction) {
593
+ // internal_error(true, "FUNCTIONS: request to cancel transaction '%s'", transaction);
594
+
595
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction);
596
+ if(!item) {
597
+ nd_log(NDLS_DAEMON, NDLP_DEBUG,
598
+ "FUNCTIONS: received a CANCEL request for transaction '%s', but the transaction is not running.",
599
+ transaction);
600
+ return;
601
+ }
602
+
603
+ struct rrd_function_inflight *r = dictionary_acquired_item_value(item);
604
+ rrd_function_cancel_inflight(r);
605
+ dictionary_acquired_item_release(rrd_functions_inflight_requests, item);
606
+}
607
+
608
+void rrd_function_progress(const char *transaction) {
609
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction);
610
+ if(!item) {
611
+ nd_log(NDLS_DAEMON, NDLP_DEBUG,
612
+ "FUNCTIONS: received a PROGRESS request for transaction '%s', but the transaction is not running.",
613
+ transaction);
614
+ return;
615
+ }
616
+
617
+ struct rrd_function_inflight *r = dictionary_acquired_item_value(item);
618
+
619
+ if(!rrd_collector_dispatcher_acquire(r->rdcf->collector)) {
620
+ nd_log(NDLS_DAEMON, NDLP_DEBUG,
621
+ "FUNCTIONS: received a PROGRESS request for transaction '%s', but the collector is not running.",
622
+ transaction);
623
+ goto cleanup;
624
+ }
625
+
626
+ functions_stop_monotonic_update_on_progress(&r->stop_monotonic_ut);
627
+
628
+ if(r->progresser.cb)
629
+ r->progresser.cb(r->progresser.data);
630
+
631
+ rrd_collector_dispatcher_release(r->rdcf->collector);
632
+
633
+cleanup:
634
+ dictionary_acquired_item_release(rrd_functions_inflight_requests, item);
635
+}
636
+
637
+void rrd_function_call_progresser(uuid_t *transaction) {
638
+ char str[UUID_COMPACT_STR_LEN];
639
+ uuid_unparse_lower_compact(*transaction, str);
640
+ rrd_function_progress(str);
641
+}
database/rrdfunctions-inflight.h
new
+16
@@ -0,0 +1,16 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_RRDFUNCTIONS_INFLIGHT_H
4
+#define NETDATA_RRDFUNCTIONS_INFLIGHT_H
5
+
6
+#include "rrd.h"
7
+
8
+void rrd_functions_inflight_init(void);
9
+
10
+// cancel a running function, to be run from anywhere
11
+void rrd_function_cancel(const char *transaction);
12
+
13
+void rrd_function_progress(const char *transaction);
14
+void rrd_function_call_progresser(uuid_t *transaction);
15
+
16
+#endif //NETDATA_RRDFUNCTIONS_INFLIGHT_H
database/rrdfunctions-inline.c
new
+42
@@ -0,0 +1,42 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "rrdfunctions-inline.h"
4
+
5
+struct rrd_function_inline {
6
+ rrd_function_execute_inline_cb_t cb;
7
+};
8
+
9
+static int rrd_function_run_inline(struct rrd_function_execute *rfe, void *data) {
10
+
11
+ // IMPORTANT: this function MUST call the result_cb even on failures
12
+
13
+ struct rrd_function_inline *fi = data;
14
+
15
+ int code;
16
+
17
+ if(rfe->is_cancelled.cb && rfe->is_cancelled.cb(rfe->is_cancelled.data))
18
+ code = HTTP_RESP_CLIENT_CLOSED_REQUEST;
19
+ else
20
+ code = fi->cb(rfe->result.wb, rfe->function);
21
+
22
+ if(code == HTTP_RESP_CLIENT_CLOSED_REQUEST || (rfe->is_cancelled.cb && rfe->is_cancelled.cb(rfe->is_cancelled.data))) {
23
+ buffer_flush(rfe->result.wb);
24
+ code = HTTP_RESP_CLIENT_CLOSED_REQUEST;
25
+ }
26
+
27
+ if(rfe->result.cb)
28
+ rfe->result.cb(rfe->result.wb, code, rfe->result.data);
29
+
30
+ return code;
31
+}
32
+
33
+void rrd_function_add_inline(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority, const char *help, const char *tags,
34
+ HTTP_ACCESS access, rrd_function_execute_inline_cb_t execute_cb) {
35
+
36
+ rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
37
+
38
+ struct rrd_function_inline *fi = callocz(1, sizeof(struct rrd_function_inline));
39
+ fi->cb = execute_cb;
40
+
41
+ rrd_function_add(host, st, name, timeout, priority, help, tags, access, true, rrd_function_run_inline, fi);
42
+}
database/rrdfunctions-inline.h
new
+14
@@ -0,0 +1,14 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_RRDFUNCTIONS_INLINE_H
4
+#define NETDATA_RRDFUNCTIONS_INLINE_H
5
+
6
+#include "rrd.h"
7
+
8
+typedef int (*rrd_function_execute_inline_cb_t)(BUFFER *wb, const char *function);
9
+
10
+void rrd_function_add_inline(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority,
11
+ const char *help, const char *tags,
12
+ HTTP_ACCESS access, rrd_function_execute_inline_cb_t execute_cb);
13
+
14
+#endif //NETDATA_RRDFUNCTIONS_INLINE_H
database/rrdfunctions-internals.h
new
+36
@@ -0,0 +1,36 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_RRDFUNCTIONS_INTERNALS_H
4
+#define NETDATA_RRDFUNCTIONS_INTERNALS_H
5
+
6
+#include "rrd.h"
7
+
8
+#include "rrdcollector-internals.h"
9
+
10
+typedef enum __attribute__((packed)) {
11
+ RRD_FUNCTION_LOCAL = (1 << 0),
12
+ RRD_FUNCTION_GLOBAL = (1 << 1),
13
+ RRD_FUNCTION_DYNCFG = (1 << 2),
14
+
15
+ // this is 8-bit
16
+} RRD_FUNCTION_OPTIONS;
17
+
18
+struct rrd_host_function {
19
+ bool sync; // when true, the function is called synchronously
20
+ RRD_FUNCTION_OPTIONS options; // RRD_FUNCTION_OPTIONS
21
+ HTTP_ACCESS access;
22
+ STRING *help;
23
+ STRING *tags;
24
+ int timeout; // the default timeout of the function
25
+ int priority;
26
+
27
+ rrd_function_execute_cb_t execute_cb;
28
+ void *execute_cb_data;
29
+
30
+ struct rrd_collector *collector;
31
+};
32
+
33
+size_t rrd_functions_sanitize(char *dst, const char *src, size_t dst_len);
34
+int rrd_functions_find_by_name(RRDHOST *host, BUFFER *wb, const char *name, size_t key_length, const DICTIONARY_ITEM **item);
35
+
36
+#endif //NETDATA_RRDFUNCTIONS_INTERNALS_H
database/rrdfunctions-progress.c
new
+8
@@ -0,0 +1,8 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "rrdfunctions-progress.h"
4
+
5
+int rrdhost_function_progress(BUFFER *wb, const char *function __maybe_unused) {
6
+ return progress_function_result(wb, rrdhost_hostname(localhost));
7
+}
8
+
database/rrdfunctions-progress.h
new
+10
@@ -0,0 +1,10 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_RRDFUNCTIONS_PROGRESS_H
4
+#define NETDATA_RRDFUNCTIONS_PROGRESS_H
5
+
6
+#include "rrd.h"
7
+
8
+int rrdhost_function_progress(BUFFER *wb, const char *function __maybe_unused);
9
+
10
+#endif //NETDATA_RRDFUNCTIONS_PROGRESS_H
database/rrdfunctions-streaming.c
new
+626
@@ -0,0 +1,626 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "rrdfunctions-streaming.h"
4
+
5
+int rrdhost_function_streaming(BUFFER *wb, const char *function __maybe_unused) {
6
+
7
+ time_t now = now_realtime_sec();
8
+
9
+ buffer_flush(wb);
10
+ wb->content_type = CT_APPLICATION_JSON;
11
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
12
+
13
+ buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
14
+ buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
15
+ buffer_json_member_add_string(wb, "type", "table");
16
+ buffer_json_member_add_time_t(wb, "update_every", 1);
17
+ buffer_json_member_add_string(wb, "help", RRDFUNCTIONS_STREAMING_HELP);
18
+ buffer_json_member_add_array(wb, "data");
19
+
20
+ size_t max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_MAX] = { 0 };
21
+ size_t max_db_metrics = 0, max_db_instances = 0, max_db_contexts = 0;
22
+ size_t max_collection_replication_instances = 0, max_streaming_replication_instances = 0;
23
+ size_t max_ml_anomalous = 0, max_ml_normal = 0, max_ml_trained = 0, max_ml_pending = 0, max_ml_silenced = 0;
24
+ {
25
+ RRDHOST *host;
26
+ dfe_start_read(rrdhost_root_index, host) {
27
+ RRDHOST_STATUS s;
28
+ rrdhost_status(host, now, &s);
29
+ buffer_json_add_array_item_array(wb);
30
+
31
+ if(s.db.metrics > max_db_metrics)
32
+ max_db_metrics = s.db.metrics;
33
+
34
+ if(s.db.instances > max_db_instances)
35
+ max_db_instances = s.db.instances;
36
+
37
+ if(s.db.contexts > max_db_contexts)
38
+ max_db_contexts = s.db.contexts;
39
+
40
+ if(s.ingest.replication.instances > max_collection_replication_instances)
41
+ max_collection_replication_instances = s.ingest.replication.instances;
42
+
43
+ if(s.stream.replication.instances > max_streaming_replication_instances)
44
+ max_streaming_replication_instances = s.stream.replication.instances;
45
+
46
+ for(int i = 0; i < STREAM_TRAFFIC_TYPE_MAX ;i++) {
47
+ if (s.stream.sent_bytes_on_this_connection_per_type[i] >
48
+ max_sent_bytes_on_this_connection_per_type[i])
49
+ max_sent_bytes_on_this_connection_per_type[i] =
50
+ s.stream.sent_bytes_on_this_connection_per_type[i];
51
+ }
52
+
53
+ // retention
54
+ buffer_json_add_array_item_string(wb, rrdhost_hostname(s.host)); // Node
55
+ buffer_json_add_array_item_uint64(wb, s.db.first_time_s * MSEC_PER_SEC); // dbFrom
56
+ buffer_json_add_array_item_uint64(wb, s.db.last_time_s * MSEC_PER_SEC); // dbTo
57
+
58
+ if(s.db.first_time_s && s.db.last_time_s && s.db.last_time_s > s.db.first_time_s)
59
+ buffer_json_add_array_item_uint64(wb, s.db.last_time_s - s.db.first_time_s); // dbDuration
60
+ else
61
+ buffer_json_add_array_item_string(wb, NULL); // dbDuration
62
+
63
+ buffer_json_add_array_item_uint64(wb, s.db.metrics); // dbMetrics
64
+ buffer_json_add_array_item_uint64(wb, s.db.instances); // dbInstances
65
+ buffer_json_add_array_item_uint64(wb, s.db.contexts); // dbContexts
66
+
67
+ // statuses
68
+ buffer_json_add_array_item_string(wb, rrdhost_ingest_status_to_string(s.ingest.status)); // InStatus
69
+ buffer_json_add_array_item_string(wb, rrdhost_streaming_status_to_string(s.stream.status)); // OutStatus
70
+ buffer_json_add_array_item_string(wb, rrdhost_ml_status_to_string(s.ml.status)); // MLStatus
71
+
72
+ // collection
73
+ if(s.ingest.since) {
74
+ buffer_json_add_array_item_uint64(wb, s.ingest.since * MSEC_PER_SEC); // InSince
75
+ buffer_json_add_array_item_time_t(wb, s.now - s.ingest.since); // InAge
76
+ }
77
+ else {
78
+ buffer_json_add_array_item_string(wb, NULL); // InSince
79
+ buffer_json_add_array_item_string(wb, NULL); // InAge
80
+ }
81
+ buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(s.ingest.reason)); // InReason
82
+ buffer_json_add_array_item_uint64(wb, s.ingest.hops); // InHops
83
+ buffer_json_add_array_item_double(wb, s.ingest.replication.completion); // InReplCompletion
84
+ buffer_json_add_array_item_uint64(wb, s.ingest.replication.instances); // InReplInstances
85
+ buffer_json_add_array_item_string(wb, s.ingest.peers.local.ip); // InLocalIP
86
+ buffer_json_add_array_item_uint64(wb, s.ingest.peers.local.port); // InLocalPort
87
+ buffer_json_add_array_item_string(wb, s.ingest.peers.peer.ip); // InRemoteIP
88
+ buffer_json_add_array_item_uint64(wb, s.ingest.peers.peer.port); // InRemotePort
89
+ buffer_json_add_array_item_string(wb, s.ingest.ssl ? "SSL" : "PLAIN"); // InSSL
90
+ stream_capabilities_to_json_array(wb, s.ingest.capabilities, NULL); // InCapabilities
91
+
92
+ // streaming
93
+ if(s.stream.since) {
94
+ buffer_json_add_array_item_uint64(wb, s.stream.since * MSEC_PER_SEC); // OutSince
95
+ buffer_json_add_array_item_time_t(wb, s.now - s.stream.since); // OutAge
96
+ }
97
+ else {
98
+ buffer_json_add_array_item_string(wb, NULL); // OutSince
99
+ buffer_json_add_array_item_string(wb, NULL); // OutAge
100
+ }
101
+ buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(s.stream.reason)); // OutReason
102
+ buffer_json_add_array_item_uint64(wb, s.stream.hops); // OutHops
103
+ buffer_json_add_array_item_double(wb, s.stream.replication.completion); // OutReplCompletion
104
+ buffer_json_add_array_item_uint64(wb, s.stream.replication.instances); // OutReplInstances
105
+ buffer_json_add_array_item_string(wb, s.stream.peers.local.ip); // OutLocalIP
106
+ buffer_json_add_array_item_uint64(wb, s.stream.peers.local.port); // OutLocalPort
107
+ buffer_json_add_array_item_string(wb, s.stream.peers.peer.ip); // OutRemoteIP
108
+ buffer_json_add_array_item_uint64(wb, s.stream.peers.peer.port); // OutRemotePort
109
+ buffer_json_add_array_item_string(wb, s.stream.ssl ? "SSL" : "PLAIN"); // OutSSL
110
+ buffer_json_add_array_item_string(wb, s.stream.compression ? "COMPRESSED" : "UNCOMPRESSED"); // OutCompression
111
+ stream_capabilities_to_json_array(wb, s.stream.capabilities, NULL); // OutCapabilities
112
+ buffer_json_add_array_item_uint64(wb, s.stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA]);
113
+ buffer_json_add_array_item_uint64(wb, s.stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA]);
114
+ buffer_json_add_array_item_uint64(wb, s.stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION]);
115
+ buffer_json_add_array_item_uint64(wb, s.stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS]);
116
+
117
+ buffer_json_add_array_item_array(wb); // OutAttemptHandshake
118
+ time_t last_attempt = 0;
119
+ for(struct rrdpush_destinations *d = host->destinations; d ; d = d->next) {
120
+ if(d->since > last_attempt)
121
+ last_attempt = d->since;
122
+
123
+ buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(d->reason));
124
+ }
125
+ buffer_json_array_close(wb); // // OutAttemptHandshake
126
+
127
+ if(!last_attempt) {
128
+ buffer_json_add_array_item_string(wb, NULL); // OutAttemptSince
129
+ buffer_json_add_array_item_string(wb, NULL); // OutAttemptAge
130
+ }
131
+ else {
132
+ buffer_json_add_array_item_uint64(wb, last_attempt * 1000); // OutAttemptSince
133
+ buffer_json_add_array_item_time_t(wb, s.now - last_attempt); // OutAttemptAge
134
+ }
135
+
136
+ // ML
137
+ if(s.ml.status == RRDHOST_ML_STATUS_RUNNING) {
138
+ buffer_json_add_array_item_uint64(wb, s.ml.metrics.anomalous); // MlAnomalous
139
+ buffer_json_add_array_item_uint64(wb, s.ml.metrics.normal); // MlNormal
140
+ buffer_json_add_array_item_uint64(wb, s.ml.metrics.trained); // MlTrained
141
+ buffer_json_add_array_item_uint64(wb, s.ml.metrics.pending); // MlPending
142
+ buffer_json_add_array_item_uint64(wb, s.ml.metrics.silenced); // MlSilenced
143
+
144
+ if(s.ml.metrics.anomalous > max_ml_anomalous)
145
+ max_ml_anomalous = s.ml.metrics.anomalous;
146
+
147
+ if(s.ml.metrics.normal > max_ml_normal)
148
+ max_ml_normal = s.ml.metrics.normal;
149
+
150
+ if(s.ml.metrics.trained > max_ml_trained)
151
+ max_ml_trained = s.ml.metrics.trained;
152
+
153
+ if(s.ml.metrics.pending > max_ml_pending)
154
+ max_ml_pending = s.ml.metrics.pending;
155
+
156
+ if(s.ml.metrics.silenced > max_ml_silenced)
157
+ max_ml_silenced = s.ml.metrics.silenced;
158
+
159
+ }
160
+ else {
161
+ buffer_json_add_array_item_string(wb, NULL); // MlAnomalous
162
+ buffer_json_add_array_item_string(wb, NULL); // MlNormal
163
+ buffer_json_add_array_item_string(wb, NULL); // MlTrained
164
+ buffer_json_add_array_item_string(wb, NULL); // MlPending
165
+ buffer_json_add_array_item_string(wb, NULL); // MlSilenced
166
+ }
167
+
168
+ // close
169
+ buffer_json_array_close(wb);
170
+ }
171
+ dfe_done(host);
172
+ }
173
+ buffer_json_array_close(wb); // data
174
+ buffer_json_member_add_object(wb, "columns");
175
+ {
176
+ size_t field_id = 0;
177
+
178
+ // Node
179
+ buffer_rrdf_table_add_field(wb, field_id++, "Node", "Node's Hostname",
180
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
181
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
182
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
183
+ RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY | RRDF_FIELD_OPTS_STICKY,
184
+ NULL);
185
+
186
+ buffer_rrdf_table_add_field(wb, field_id++, "dbFrom", "DB Data Retention From",
187
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
188
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
189
+ RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
190
+ RRDF_FIELD_OPTS_NONE, NULL);
191
+
192
+ buffer_rrdf_table_add_field(wb, field_id++, "dbTo", "DB Data Retention To",
193
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
194
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
195
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
196
+ RRDF_FIELD_OPTS_NONE, NULL);
197
+
198
+ buffer_rrdf_table_add_field(wb, field_id++, "dbDuration", "DB Data Retention Duration",
199
+ RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
200
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
201
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
202
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
203
+
204
+ buffer_rrdf_table_add_field(wb, field_id++, "dbMetrics", "Time-series Metrics in the DB",
205
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
206
+ 0, NULL, (double)max_db_metrics, RRDF_FIELD_SORT_DESCENDING, NULL,
207
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
208
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
209
+
210
+ buffer_rrdf_table_add_field(wb, field_id++, "dbInstances", "Instances in the DB",
211
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
212
+ 0, NULL, (double)max_db_instances, RRDF_FIELD_SORT_DESCENDING, NULL,
213
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
214
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
215
+
216
+ buffer_rrdf_table_add_field(wb, field_id++, "dbContexts", "Contexts in the DB",
217
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
218
+ 0, NULL, (double)max_db_contexts, RRDF_FIELD_SORT_DESCENDING, NULL,
219
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
220
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
221
+
222
+ // --- statuses ---
223
+
224
+ buffer_rrdf_table_add_field(wb, field_id++, "InStatus", "Data Collection Online Status",
225
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
226
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
227
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
228
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
229
+
230
+
231
+ buffer_rrdf_table_add_field(wb, field_id++, "OutStatus", "Streaming Online Status",
232
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
233
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
234
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
235
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
236
+
237
+ buffer_rrdf_table_add_field(wb, field_id++, "MlStatus", "ML Status",
238
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
239
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
240
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
241
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
242
+
243
+ // --- collection ---
244
+
245
+ buffer_rrdf_table_add_field(wb, field_id++, "InSince", "Last Data Collection Status Change",
246
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
247
+ 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
248
+ RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
249
+ RRDF_FIELD_OPTS_NONE, NULL);
250
+
251
+ buffer_rrdf_table_add_field(wb, field_id++, "InAge", "Last Data Collection Online Status Change Age",
252
+ RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
253
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
254
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
255
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
256
+
257
+ buffer_rrdf_table_add_field(wb, field_id++, "InReason", "Data Collection Online Status Reason",
258
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
259
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
260
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
261
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
262
+
263
+ buffer_rrdf_table_add_field(wb, field_id++, "InHops", "Data Collection Distance Hops from Origin Node",
264
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
265
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
266
+ RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
267
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
268
+
269
+ buffer_rrdf_table_add_field(wb, field_id++, "InReplCompletion", "Inbound Replication Completion",
270
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
271
+ 1, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
272
+ RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
273
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
274
+
275
+ buffer_rrdf_table_add_field(wb, field_id++, "InReplInstances", "Inbound Replicating Instances",
276
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
277
+ 0, "instances", (double)max_collection_replication_instances, RRDF_FIELD_SORT_DESCENDING,
278
+ NULL,
279
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
280
+ RRDF_FIELD_OPTS_NONE, NULL);
281
+
282
+ buffer_rrdf_table_add_field(wb, field_id++, "InLocalIP", "Inbound Local IP",
283
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
284
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
285
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
286
+ RRDF_FIELD_OPTS_NONE, NULL);
287
+
288
+ buffer_rrdf_table_add_field(wb, field_id++, "InLocalPort", "Inbound Local Port",
289
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
290
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
291
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE,
292
+ RRDF_FIELD_OPTS_NONE, NULL);
293
+
294
+ buffer_rrdf_table_add_field(wb, field_id++, "InRemoteIP", "Inbound Remote IP",
295
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
296
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
297
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
298
+ RRDF_FIELD_OPTS_NONE, NULL);
299
+
300
+ buffer_rrdf_table_add_field(wb, field_id++, "InRemotePort", "Inbound Remote Port",
301
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
302
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
303
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE,
304
+ RRDF_FIELD_OPTS_NONE, NULL);
305
+
306
+ buffer_rrdf_table_add_field(wb, field_id++, "InSSL", "Inbound SSL Connection",
307
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
308
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
309
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
310
+ RRDF_FIELD_OPTS_NONE, NULL);
311
+
312
+ buffer_rrdf_table_add_field(wb, field_id++, "InCapabilities", "Inbound Connection Capabilities",
313
+ RRDF_FIELD_TYPE_ARRAY, RRDF_FIELD_VISUAL_PILL, RRDF_FIELD_TRANSFORM_NONE,
314
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
315
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
316
+ RRDF_FIELD_OPTS_NONE, NULL);
317
+
318
+ // --- streaming ---
319
+
320
+ buffer_rrdf_table_add_field(wb, field_id++, "OutSince", "Last Streaming Status Change",
321
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
322
+ 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
323
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
324
+ RRDF_FIELD_OPTS_NONE, NULL);
325
+
326
+ buffer_rrdf_table_add_field(wb, field_id++, "OutAge", "Last Streaming Status Change Age",
327
+ RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
328
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
329
+ RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
330
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
331
+
332
+ buffer_rrdf_table_add_field(wb, field_id++, "OutReason", "Streaming Status Reason",
333
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
334
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
335
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
336
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
337
+
338
+ buffer_rrdf_table_add_field(wb, field_id++, "OutHops", "Streaming Distance Hops from Origin Node",
339
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
340
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
341
+ RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
342
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
343
+
344
+ buffer_rrdf_table_add_field(wb, field_id++, "OutReplCompletion", "Outbound Replication Completion",
345
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
346
+ 1, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
347
+ RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
348
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
349
+
350
+ buffer_rrdf_table_add_field(wb, field_id++, "OutReplInstances", "Outbound Replicating Instances",
351
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
352
+ 0, "instances", (double)max_streaming_replication_instances, RRDF_FIELD_SORT_DESCENDING,
353
+ NULL,
354
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
355
+ RRDF_FIELD_OPTS_NONE, NULL);
356
+
357
+ buffer_rrdf_table_add_field(wb, field_id++, "OutLocalIP", "Outbound Local IP",
358
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
359
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
360
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
361
+ RRDF_FIELD_OPTS_NONE, NULL);
362
+
363
+ buffer_rrdf_table_add_field(wb, field_id++, "OutLocalPort", "Outbound Local Port",
364
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
365
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
366
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE,
367
+ RRDF_FIELD_OPTS_NONE, NULL);
368
+
369
+ buffer_rrdf_table_add_field(wb, field_id++, "OutRemoteIP", "Outbound Remote IP",
370
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
371
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
372
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
373
+ RRDF_FIELD_OPTS_NONE, NULL);
374
+
375
+ buffer_rrdf_table_add_field(wb, field_id++, "OutRemotePort", "Outbound Remote Port",
376
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
377
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
378
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE,
379
+ RRDF_FIELD_OPTS_NONE, NULL);
380
+
381
+ buffer_rrdf_table_add_field(wb, field_id++, "OutSSL", "Outbound SSL Connection",
382
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
383
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
384
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
385
+ RRDF_FIELD_OPTS_NONE, NULL);
386
+
387
+ buffer_rrdf_table_add_field(wb, field_id++, "OutCompression", "Outbound Compressed Connection",
388
+ RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
389
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
390
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
391
+ RRDF_FIELD_OPTS_NONE, NULL);
392
+
393
+ buffer_rrdf_table_add_field(wb, field_id++, "OutCapabilities", "Outbound Connection Capabilities",
394
+ RRDF_FIELD_TYPE_ARRAY, RRDF_FIELD_VISUAL_PILL, RRDF_FIELD_TRANSFORM_NONE,
395
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
396
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
397
+ RRDF_FIELD_OPTS_NONE, NULL);
398
+
399
+ buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficData", "Outbound Metric Data Traffic",
400
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
401
+ 0, "bytes", (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA],
402
+ RRDF_FIELD_SORT_DESCENDING, NULL,
403
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
404
+ RRDF_FIELD_OPTS_NONE, NULL);
405
+
406
+ buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficMetadata", "Outbound Metric Metadata Traffic",
407
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
408
+ 0, "bytes",
409
+ (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA],
410
+ RRDF_FIELD_SORT_DESCENDING, NULL,
411
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
412
+ RRDF_FIELD_OPTS_NONE, NULL);
413
+
414
+ buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficReplication", "Outbound Metric Replication Traffic",
415
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
416
+ 0, "bytes",
417
+ (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION],
418
+ RRDF_FIELD_SORT_DESCENDING, NULL,
419
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
420
+ RRDF_FIELD_OPTS_NONE, NULL);
421
+
422
+ buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficFunctions", "Outbound Metric Functions Traffic",
423
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
424
+ 0, "bytes",
425
+ (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS],
426
+ RRDF_FIELD_SORT_DESCENDING, NULL,
427
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
428
+ RRDF_FIELD_OPTS_NONE, NULL);
429
+
430
+ buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptHandshake",
431
+ "Outbound Connection Attempt Handshake Status",
432
+ RRDF_FIELD_TYPE_ARRAY, RRDF_FIELD_VISUAL_PILL, RRDF_FIELD_TRANSFORM_NONE,
433
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
434
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
435
+ RRDF_FIELD_OPTS_NONE, NULL);
436
+
437
+ buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptSince",
438
+ "Last Outbound Connection Attempt Status Change Time",
439
+ RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
440
+ 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
441
+ RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
442
+ RRDF_FIELD_OPTS_NONE, NULL);
443
+
444
+ buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptAge",
445
+ "Last Outbound Connection Attempt Status Change Age",
446
+ RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
447
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
448
+ RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
449
+ RRDF_FIELD_OPTS_VISIBLE, NULL);
450
+
451
+ // --- ML ---
452
+
453
+ buffer_rrdf_table_add_field(wb, field_id++, "MlAnomalous", "Number of Anomalous Metrics",
454
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
455
+ 0, "metrics",
456
+ (double)max_ml_anomalous,
457
+ RRDF_FIELD_SORT_DESCENDING, NULL,
458
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
459
+ RRDF_FIELD_OPTS_NONE, NULL);
460
+
461
+ buffer_rrdf_table_add_field(wb, field_id++, "MlNormal", "Number of Not Anomalous Metrics",
462
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
463
+ 0, "metrics",
464
+ (double)max_ml_normal,
465
+ RRDF_FIELD_SORT_DESCENDING, NULL,
466
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
467
+ RRDF_FIELD_OPTS_NONE, NULL);
468
+
469
+ buffer_rrdf_table_add_field(wb, field_id++, "MlTrained", "Number of Trained Metrics",
470
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
471
+ 0, "metrics",
472
+ (double)max_ml_trained,
473
+ RRDF_FIELD_SORT_DESCENDING, NULL,
474
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
475
+ RRDF_FIELD_OPTS_NONE, NULL);
476
+
477
+ buffer_rrdf_table_add_field(wb, field_id++, "MlPending", "Number of Pending Metrics",
478
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
479
+ 0, "metrics",
480
+ (double)max_ml_pending,
481
+ RRDF_FIELD_SORT_DESCENDING, NULL,
482
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
483
+ RRDF_FIELD_OPTS_NONE, NULL);
484
+
485
+ buffer_rrdf_table_add_field(wb, field_id++, "MlSilenced", "Number of Silenced Metrics",
486
+ RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
487
+ 0, "metrics",
488
+ (double)max_ml_silenced,
489
+ RRDF_FIELD_SORT_DESCENDING, NULL,
490
+ RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
491
+ RRDF_FIELD_OPTS_NONE, NULL);
492
+ }
493
+ buffer_json_object_close(wb); // columns
494
+ buffer_json_member_add_string(wb, "default_sort_column", "Node");
495
+ buffer_json_member_add_object(wb, "charts");
496
+ {
497
+ // Data Collection Age chart
498
+ buffer_json_member_add_object(wb, "InAge");
499
+ {
500
+ buffer_json_member_add_string(wb, "name", "Data Collection Age");
501
+ buffer_json_member_add_string(wb, "type", "stacked-bar");
502
+ buffer_json_member_add_array(wb, "columns");
503
+ {
504
+ buffer_json_add_array_item_string(wb, "InAge");
505
+ }
506
+ buffer_json_array_close(wb);
507
+ }
508
+ buffer_json_object_close(wb);
509
+
510
+ // Streaming Age chart
511
+ buffer_json_member_add_object(wb, "OutAge");
512
+ {
513
+ buffer_json_member_add_string(wb, "name", "Streaming Age");
514
+ buffer_json_member_add_string(wb, "type", "stacked-bar");
515
+ buffer_json_member_add_array(wb, "columns");
516
+ {
517
+ buffer_json_add_array_item_string(wb, "OutAge");
518
+ }
519
+ buffer_json_array_close(wb);
520
+ }
521
+ buffer_json_object_close(wb);
522
+
523
+ // DB Duration
524
+ buffer_json_member_add_object(wb, "dbDuration");
525
+ {
526
+ buffer_json_member_add_string(wb, "name", "Retention Duration");
527
+ buffer_json_member_add_string(wb, "type", "stacked-bar");
528
+ buffer_json_member_add_array(wb, "columns");
529
+ {
530
+ buffer_json_add_array_item_string(wb, "dbDuration");
531
+ }
532
+ buffer_json_array_close(wb);
533
+ }
534
+ buffer_json_object_close(wb);
535
+ }
536
+ buffer_json_object_close(wb); // charts
537
+
538
+ buffer_json_member_add_array(wb, "default_charts");
539
+ {
540
+ buffer_json_add_array_item_array(wb);
541
+ buffer_json_add_array_item_string(wb, "InAge");
542
+ buffer_json_add_array_item_string(wb, "Node");
543
+ buffer_json_array_close(wb);
544
+
545
+ buffer_json_add_array_item_array(wb);
546
+ buffer_json_add_array_item_string(wb, "OutAge");
547
+ buffer_json_add_array_item_string(wb, "Node");
548
+ buffer_json_array_close(wb);
549
+ }
550
+ buffer_json_array_close(wb);
551
+
552
+ buffer_json_member_add_object(wb, "group_by");
553
+ {
554
+ buffer_json_member_add_object(wb, "Node");
555
+ {
556
+ buffer_json_member_add_string(wb, "name", "Node");
557
+ buffer_json_member_add_array(wb, "columns");
558
+ {
559
+ buffer_json_add_array_item_string(wb, "Node");
560
+ }
561
+ buffer_json_array_close(wb);
562
+ }
563
+ buffer_json_object_close(wb);
564
+
565
+ buffer_json_member_add_object(wb, "InStatus");
566
+ {
567
+ buffer_json_member_add_string(wb, "name", "Nodes by Collection Status");
568
+ buffer_json_member_add_array(wb, "columns");
569
+ {
570
+ buffer_json_add_array_item_string(wb, "InStatus");
571
+ }
572
+ buffer_json_array_close(wb);
573
+ }
574
+ buffer_json_object_close(wb);
575
+
576
+ buffer_json_member_add_object(wb, "OutStatus");
577
+ {
578
+ buffer_json_member_add_string(wb, "name", "Nodes by Streaming Status");
579
+ buffer_json_member_add_array(wb, "columns");
580
+ {
581
+ buffer_json_add_array_item_string(wb, "OutStatus");
582
+ }
583
+ buffer_json_array_close(wb);
584
+ }
585
+ buffer_json_object_close(wb);
586
+
587
+ buffer_json_member_add_object(wb, "MlStatus");
588
+ {
589
+ buffer_json_member_add_string(wb, "name", "Nodes by ML Status");
590
+ buffer_json_member_add_array(wb, "columns");
591
+ {
592
+ buffer_json_add_array_item_string(wb, "MlStatus");
593
+ }
594
+ buffer_json_array_close(wb);
595
+ }
596
+ buffer_json_object_close(wb);
597
+
598
+ buffer_json_member_add_object(wb, "InRemoteIP");
599
+ {
600
+ buffer_json_member_add_string(wb, "name", "Nodes by Inbound IP");
601
+ buffer_json_member_add_array(wb, "columns");
602
+ {
603
+ buffer_json_add_array_item_string(wb, "InRemoteIP");
604
+ }
605
+ buffer_json_array_close(wb);
606
+ }
607
+ buffer_json_object_close(wb);
608
+
609
+ buffer_json_member_add_object(wb, "OutRemoteIP");
610
+ {
611
+ buffer_json_member_add_string(wb, "name", "Nodes by Outbound IP");
612
+ buffer_json_member_add_array(wb, "columns");
613
+ {
614
+ buffer_json_add_array_item_string(wb, "OutRemoteIP");
615
+ }
616
+ buffer_json_array_close(wb);
617
+ }
618
+ buffer_json_object_close(wb);
619
+ }
620
+ buffer_json_object_close(wb); // group_by
621
+
622
+ buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + 1);
623
+ buffer_json_finalize(wb);
624
+
625
+ return HTTP_RESP_OK;
626
+}
database/rrdfunctions-streaming.h
new
+12
@@ -0,0 +1,12 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_RRDFUNCTIONS_STREAMING_H
4
+#define NETDATA_RRDFUNCTIONS_STREAMING_H
5
+
6
+#include "rrd.h"
7
+
8
+#define RRDFUNCTIONS_STREAMING_HELP "Streaming status for parents and children."
9
+
10
+int rrdhost_function_streaming(BUFFER *wb, const char *function);
11
+
12
+#endif //NETDATA_RRDFUNCTIONS_STREAMING_H
database/rrdfunctions.c
+50
-1400
@@ -1,8 +1,9 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
#define NETDATA_RRD_INTERNALS
3
-#define NETDATA_RRDCOLLECTOR_INTERNALS
4
5
#include "rrd.h"
6
+#include "rrdfunctions-internals.h"
7
8
#define MAX_FUNCTION_LENGTH (PLUGINSD_LINE_MAX - 512) // we need some space for the rest of the line
9
@@ -17,7 +18,7 @@ static unsigned char functions_allowed_chars[256] = {
18
[30] = '_', [31] = '_',
19
20
// symbols
20
- [' '] = ' ', ['!'] = '!', ['"'] = '"', ['#'] = '#', ['$'] = '$', ['%'] = '%', ['&'] = '&', ['\''] = '\'',
21
+ [' '] = ' ', ['!'] = '!', ['"'] = '\'', ['#'] = '#', ['$'] = '$', ['%'] = '%', ['&'] = '&', ['\''] = '\'',
22
['('] = '(', [')'] = ')', ['*'] = '*', ['+'] = '+', [','] = ',', ['-'] = '-', ['.'] = '.', ['/'] = '/',
23
24
// numbers
@@ -65,99 +66,15 @@ static unsigned char functions_allowed_chars[256] = {
66
[255] = '_'
67
};
68
68
-static inline size_t sanitize_function_text(char *dst, const char *src, size_t dst_len) {
69
+size_t rrd_functions_sanitize(char *dst, const char *src, size_t dst_len) {
70
return text_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_len,
71
functions_allowed_chars, true, "", NULL);
72
}
73
73
-// we keep a dictionary per RRDSET with these functions
74
-// the dictionary is created on demand (only when a function is added to an RRDSET)
75
-
76
-typedef enum __attribute__((packed)) {
77
- RRD_FUNCTION_LOCAL = (1 << 0),
78
- RRD_FUNCTION_GLOBAL = (1 << 1),
79
-
80
- // this is 8-bit
81
-} RRD_FUNCTION_OPTIONS;
82
-
74
// ----------------------------------------------------------------------------
75
85
-struct rrd_host_function {
86
- bool sync; // when true, the function is called synchronously
87
- RRD_FUNCTION_OPTIONS options; // RRD_FUNCTION_OPTIONS
88
- HTTP_ACCESS access;
89
- STRING *help;
90
- STRING *tags;
91
- int timeout; // the default timeout of the function
92
- int priority;
93
-
94
- rrd_function_execute_cb_t execute_cb;
95
- void *execute_cb_data;
96
-
97
- struct rrd_collector *collector;
98
-};
99
-
100
-struct rrd_function_inflight {
101
- bool used;
102
-
103
- RRDHOST *host;
104
- uuid_t transaction_uuid;
105
- const char *transaction;
106
- const char *cmd;
107
- const char *sanitized_cmd;
108
- size_t sanitized_cmd_length;
109
- int timeout;
110
- bool cancelled;
111
- usec_t stop_monotonic_ut;
112
-
113
- const DICTIONARY_ITEM *host_function_acquired;
114
-
115
- // the collector
116
- // we acquire this structure at the beginning,
117
- // and we release it at the end
118
- struct rrd_host_function *rdcf;
119
-
120
- struct {
121
- BUFFER *wb;
122
-
123
- // in async mode,
124
- // the function to call to send the result back
125
- rrd_function_result_callback_t cb;
126
- void *data;
127
- } result;
128
-
129
- struct {
130
- // to be called in sync mode
131
- // while the function is running
132
- // to check if the function has been canceled
133
- rrd_function_is_cancelled_cb_t cb;
134
- void *data;
135
- } is_cancelled;
136
-
137
- struct {
138
- // to be registered by the function itself
139
- // used to signal the function to cancel
140
- rrd_function_cancel_cb_t cb;
141
- void *data;
142
- } canceller;
143
-
144
- struct {
145
- // callback to receive progress reports from function
146
- rrd_function_progress_cb_t cb;
147
- void *data;
148
- } progress;
149
-
150
- struct {
151
- // to be registered by the function itself
152
- // used to send progress requests to function
153
- rrd_function_progresser_cb_t cb;
154
- void *data;
155
- } progresser;
156
-};
157
-
158
-static DICTIONARY *rrd_functions_inflight_requests = NULL;
159
-
160
-static void rrd_function_cancel_inflight(struct rrd_function_inflight *r);
76
+// we keep a dictionary per RRDSET with these functions
77
+// the dictionary is created on demand (only when a function is added to an RRDSET)
78
79
// ----------------------------------------------------------------------------
80
@@ -291,7 +208,7 @@ static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_
208
return changed;
209
}
210
294
-void rrdfunctions_host_init(RRDHOST *host) {
211
+void rrd_functions_host_init(RRDHOST *host) {
212
if(host->functions) return;
213
214
host->functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
@@ -302,15 +219,29 @@ void rrdfunctions_host_init(RRDHOST *host) {
219
dictionary_register_conflict_callback(host->functions, rrd_functions_conflict_callback, host);
220
}
221
305
-void rrdfunctions_host_destroy(RRDHOST *host) {
222
+void rrd_functions_host_destroy(RRDHOST *host) {
223
dictionary_destroy(host->functions);
224
}
225
226
// ----------------------------------------------------------------------------
227
311
-void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority, const char *help, const char *tags,
312
- HTTP_ACCESS access, bool sync, rrd_function_execute_cb_t execute_cb,
313
- void *execute_cb_data) {
228
+static inline bool is_function_dyncfg(const char *name) {
229
+ if(!name || !*name)
230
+ return false;
231
+
232
+ if(strncmp(name, PLUGINSD_FUNCTION_CONFIG, sizeof(PLUGINSD_FUNCTION_CONFIG) - 1) != 0)
233
+ return false;
234
+
235
+ char c = name[sizeof(PLUGINSD_FUNCTION_CONFIG) - 1];
236
+ if(c == 0 || isspace(c))
237
+ return true;
238
+
239
+ return false;
240
+}
241
+
242
+void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority,
243
+ const char *help, const char *tags, HTTP_ACCESS access, bool sync,
244
+ rrd_function_execute_cb_t execute_cb, void *execute_cb_data) {
245
246
// RRDSET *st may be NULL in this function
247
// to create a GLOBAL function
@@ -325,13 +256,13 @@ void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout,
256
if(st && !st->functions_view)
257
st->functions_view = dictionary_create_view(host->functions);
258
328
- char key[PLUGINSD_LINE_MAX + 1];
329
- sanitize_function_text(key, name, PLUGINSD_LINE_MAX);
259
+ char key[strlen(name) + 1];
260
+ rrd_functions_sanitize(key, name, sizeof(key));
261
262
struct rrd_host_function tmp = {
263
.sync = sync,
264
.timeout = timeout,
334
- .options = (st)?RRD_FUNCTION_LOCAL:RRD_FUNCTION_GLOBAL,
265
+ .options = st ? RRD_FUNCTION_LOCAL: (is_function_dyncfg(name) ? RRD_FUNCTION_DYNCFG : RRD_FUNCTION_GLOBAL),
266
.access = access,
267
.execute_cb = execute_cb,
268
.execute_cb_data = execute_cb_data,
@@ -349,81 +280,17 @@ void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout,
280
dictionary_acquired_item_release(host->functions, item);
281
}
282
352
-void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb) {
353
- if(!st->functions_view)
354
- return;
355
-
356
- struct rrd_host_function *tmp;
357
- dfe_start_read(st->functions_view, tmp) {
358
- buffer_sprintf(wb
359
- , PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\" \"%s\" \"%s\" %d\n"
360
- , tmp_dfe.name
361
- , tmp->timeout
362
- , string2str(tmp->help)
363
- , string2str(tmp->tags)
364
- , http_id2access(tmp->access)
365
- , tmp->priority
366
- );
367
- }
368
- dfe_done(tmp);
369
-}
370
-
371
-void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb) {
372
- rrdhost_flag_clear(host, RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED);
373
-
374
- struct rrd_host_function *tmp;
375
- dfe_start_read(host->functions, tmp) {
376
- if(!(tmp->options & RRD_FUNCTION_GLOBAL))
377
- continue;
378
-
379
- buffer_sprintf(wb
380
- , PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"%s\" \"%s\" %d\n"
381
- , tmp_dfe.name
382
- , tmp->timeout
383
- , string2str(tmp->help)
384
- , string2str(tmp->tags)
385
- , http_id2access(tmp->access)
386
- , tmp->priority
387
- );
388
- }
389
- dfe_done(tmp);
390
-}
391
-
392
-struct {
393
- const char *format;
394
- HTTP_CONTENT_TYPE content_type;
395
-} function_formats[] = {
396
- { .format = "application/json", CT_APPLICATION_JSON },
397
- { .format = "text/plain", CT_TEXT_PLAIN },
398
- { .format = "application/xml", CT_APPLICATION_XML },
399
- { .format = "prometheus", CT_PROMETHEUS },
400
- { .format = "text", CT_TEXT_PLAIN },
401
- { .format = "txt", CT_TEXT_PLAIN },
402
- { .format = "json", CT_APPLICATION_JSON },
403
- { .format = "html", CT_TEXT_HTML },
404
- { .format = "text/html", CT_TEXT_HTML },
405
- { .format = "xml", CT_APPLICATION_XML },
406
-
407
- // terminator
408
- { .format = NULL, CT_TEXT_PLAIN },
409
-};
410
-
411
-uint8_t functions_format_to_content_type(const char *format) {
412
- if(format && *format) {
413
- for (int i = 0; function_formats[i].format; i++)
414
- if (strcmp(function_formats[i].format, format) == 0)
415
- return function_formats[i].content_type;
416
- }
283
+void rrd_function_del(RRDHOST *host, RRDSET *st, const char *name) {
284
+ char key[strlen(name) + 1];
285
+ rrd_functions_sanitize(key, name, sizeof(key));
286
+ dictionary_del(host->functions, key);
287
418
- return CT_TEXT_PLAIN;
419
-}
420
-
421
-const char *functions_content_type_to_format(HTTP_CONTENT_TYPE content_type) {
422
- for (int i = 0; function_formats[i].format; i++)
423
- if (function_formats[i].content_type == content_type)
424
- return function_formats[i].format;
288
+ if(st)
289
+ dictionary_del(st->functions_view, key);
290
+ else
291
+ rrdhost_flag_set(host, RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED);
292
426
- return "text/plain";
293
+ dictionary_garbage_collect(host->functions);
294
}
295
296
int rrd_call_function_error(BUFFER *wb, const char *msg, int code) {
@@ -437,10 +304,9 @@ int rrd_call_function_error(BUFFER *wb, const char *msg, int code) {
304
return code;
305
}
306
440
-static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, size_t key_length, const DICTIONARY_ITEM **item) {
307
+int rrd_functions_find_by_name(RRDHOST *host, BUFFER *wb, const char *name, size_t key_length, const DICTIONARY_ITEM **item) {
308
char buffer[MAX_FUNCTION_LENGTH + 1];
442
-
443
- strncpyz(buffer, name, MAX_FUNCTION_LENGTH);
309
+ strncpyz(buffer, name, sizeof(buffer) - 1);
310
char *s = NULL;
311
312
bool found = false;
@@ -489,1235 +355,19 @@ static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, s
355
return HTTP_RESP_OK;
356
}
357
492
-// ----------------------------------------------------------------------------
493
-
494
-static void rrd_functions_inflight_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
495
- struct rrd_function_inflight *r = value;
496
-
497
- // internal_error(true, "FUNCTIONS: transaction '%s' finished", r->transaction);
498
-
499
- freez((void *)r->transaction);
500
- freez((void *)r->cmd);
501
- freez((void *)r->sanitized_cmd);
502
- dictionary_acquired_item_release(r->host->functions, r->host_function_acquired);
503
-}
504
-
505
-void rrd_functions_inflight_init(void) {
506
- if(rrd_functions_inflight_requests)
507
- return;
508
-
509
- rrd_functions_inflight_requests = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct rrd_function_inflight));
510
-
511
- dictionary_register_delete_callback(rrd_functions_inflight_requests, rrd_functions_inflight_delete_cb, NULL);
512
-}
513
-
514
-void rrd_functions_inflight_destroy(void) {
515
- if(!rrd_functions_inflight_requests)
516
- return;
517
-
518
- dictionary_destroy(rrd_functions_inflight_requests);
519
- rrd_functions_inflight_requests = NULL;
520
-}
521
-
522
-static void rrd_inflight_async_function_register_canceller_cb(void *register_canceller_cb_data, rrd_function_cancel_cb_t canceller_cb, void *canceller_cb_data) {
523
- struct rrd_function_inflight *r = register_canceller_cb_data;
524
- r->canceller.cb = canceller_cb;
525
- r->canceller.data = canceller_cb_data;
526
-}
527
-
528
-static void rrd_inflight_async_function_register_progresser_cb(void *register_progresser_cb_data, rrd_function_progresser_cb_t progresser_cb, void *progresser_cb_data) {
529
- struct rrd_function_inflight *r = register_progresser_cb_data;
530
- r->progresser.cb = progresser_cb;
531
- r->progresser.data = progresser_cb_data;
532
-}
533
-
534
-// ----------------------------------------------------------------------------
535
-// waiting for async function completion
536
-
537
-struct rrd_function_call_wait {
538
- RRDHOST *host;
539
- const DICTIONARY_ITEM *host_function_acquired;
540
- char *transaction;
541
-
542
- bool free_with_signal;
543
- bool data_are_ready;
544
- netdata_mutex_t mutex;
545
- pthread_cond_t cond;
546
- int code;
547
-};
548
-
549
-static void rrd_inflight_function_cleanup(RRDHOST *host __maybe_unused,
550
- const char *transaction) {
551
- dictionary_del(rrd_functions_inflight_requests, transaction);
552
- dictionary_garbage_collect(rrd_functions_inflight_requests);
553
-}
554
-
555
-static void rrd_function_call_wait_free(struct rrd_function_call_wait *tmp) {
556
- rrd_inflight_function_cleanup(tmp->host, tmp->transaction);
557
- freez(tmp->transaction);
558
-
559
- pthread_cond_destroy(&tmp->cond);
560
- netdata_mutex_destroy(&tmp->mutex);
561
- freez(tmp);
562
-}
563
-
564
-static void rrd_async_function_signal_when_ready(BUFFER *temp_wb __maybe_unused, int code, void *callback_data) {
565
- struct rrd_function_call_wait *tmp = callback_data;
566
- bool we_should_free = false;
567
-
568
- netdata_mutex_lock(&tmp->mutex);
569
-
570
- // since we got the mutex,
571
- // the waiting thread is either in pthread_cond_timedwait()
572
- // or gave up and left.
573
-
574
- tmp->code = code;
575
- tmp->data_are_ready = true;
576
-
577
- if(tmp->free_with_signal)
578
- we_should_free = true;
579
-
580
- pthread_cond_signal(&tmp->cond);
581
-
582
- netdata_mutex_unlock(&tmp->mutex);
583
-
584
- if(we_should_free) {
585
- buffer_free(temp_wb);
586
- rrd_function_call_wait_free(tmp);
587
- }
588
-}
589
-
590
-static void rrd_inflight_async_function_nowait_finished(BUFFER *wb, int code, void *data) {
591
- struct rrd_function_inflight *r = data;
592
-
593
- if(r->result.cb)
594
- r->result.cb(wb, code, r->result.data);
595
-
596
- rrd_inflight_function_cleanup(r->host, r->transaction);
597
-}
598
-
599
-static bool rrd_inflight_async_function_is_cancelled(void *data) {
600
- struct rrd_function_inflight *r = data;
601
- return __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED);
602
-}
603
-
604
-static inline int rrd_call_function_async_and_dont_wait(struct rrd_function_inflight *r) {
605
- int code = r->rdcf->execute_cb(&r->transaction_uuid, r->result.wb,
606
- &r->stop_monotonic_ut, r->sanitized_cmd, r->rdcf->execute_cb_data,
607
- rrd_inflight_async_function_nowait_finished, r,
608
- r->progress.cb, r->progress.data,
609
- rrd_inflight_async_function_is_cancelled, r,
610
- rrd_inflight_async_function_register_canceller_cb, r,
611
- rrd_inflight_async_function_register_progresser_cb, r);
612
-
613
- if(code != HTTP_RESP_OK) {
614
- if (!buffer_strlen(r->result.wb))
615
- rrd_call_function_error(r->result.wb, "Failed to send request to the collector.", code);
616
-
617
- rrd_inflight_function_cleanup(r->host, r->transaction);
618
- }
619
-
620
- return code;
621
-}
622
-
623
-static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) {
624
- struct rrd_function_call_wait *tmp = mallocz(sizeof(struct rrd_function_call_wait));
625
- tmp->free_with_signal = false;
626
- tmp->data_are_ready = false;
627
- tmp->host = r->host;
628
- tmp->host_function_acquired = r->host_function_acquired;
629
- tmp->transaction = strdupz(r->transaction);
630
- netdata_mutex_init(&tmp->mutex);
631
- pthread_cond_init(&tmp->cond, NULL);
632
-
633
- // we need a temporary BUFFER, because we may time out and the caller supplied one may vanish,
634
- // so we create a new one we guarantee will survive until the collector finishes...
635
-
636
- bool we_should_free = true;
637
- BUFFER *temp_wb = buffer_create(PLUGINSD_LINE_MAX + 1, &netdata_buffers_statistics.buffers_functions); // we need it because we may give up on it
638
- temp_wb->content_type = r->result.wb->content_type;
639
-
640
- int code = r->rdcf->execute_cb(&r->transaction_uuid, temp_wb, &r->stop_monotonic_ut,
641
- r->sanitized_cmd, r->rdcf->execute_cb_data,
642
- // we overwrite the result callbacks,
643
- // so that we can clean up the allocations made
644
- rrd_async_function_signal_when_ready, tmp,
645
- r->progress.cb, r->progress.data,
646
- rrd_inflight_async_function_is_cancelled, r,
647
- rrd_inflight_async_function_register_canceller_cb, r,
648
- rrd_inflight_async_function_register_progresser_cb, r);
649
-
650
- if (code == HTTP_RESP_OK) {
651
- netdata_mutex_lock(&tmp->mutex);
652
-
653
- bool cancelled = false;
654
- int rc = 0;
655
- while (rc == 0 && !cancelled && !tmp->data_are_ready) {
656
- usec_t now_mono_ut = now_monotonic_usec();
657
- usec_t stop_mono_ut = __atomic_load_n(&r->stop_monotonic_ut, __ATOMIC_RELAXED) + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
658
- if(now_mono_ut > stop_mono_ut) {
659
- rc = ETIMEDOUT;
660
- break;
661
- }
662
-
663
- // wait for 10ms, and loop again...
664
- struct timespec tp;
665
- clock_gettime(CLOCK_REALTIME, &tp);
666
- tp.tv_nsec += 10 * NSEC_PER_MSEC;
667
- if(tp.tv_nsec > (long)(1 * NSEC_PER_SEC)) {
668
- tp.tv_sec++;
669
- tp.tv_nsec -= 1 * NSEC_PER_SEC;
670
- }
671
-
672
- // the mutex is unlocked within pthread_cond_timedwait()
673
- rc = pthread_cond_timedwait(&tmp->cond, &tmp->mutex, &tp);
674
- // the mutex is again ours
675
-
676
- if(rc == ETIMEDOUT) {
677
- // 10ms have passed
678
-
679
- rc = 0;
680
- if (!tmp->data_are_ready && r->is_cancelled.cb &&
681
- r->is_cancelled.cb(r->is_cancelled.data)) {
682
-// internal_error(true, "FUNCTIONS: transaction '%s' is cancelled while waiting for response",
683
-// r->transaction);
684
- cancelled = true;
685
- rrd_function_cancel_inflight(r);
686
- break;
687
- }
688
- }
689
- }
690
-
691
- if (tmp->data_are_ready) {
692
- // we have a response
693
- buffer_fast_strcat(r->result.wb, buffer_tostring(temp_wb), buffer_strlen(temp_wb));
694
- r->result.wb->content_type = temp_wb->content_type;
695
- r->result.wb->expires = temp_wb->expires;
696
-
697
- if(r->result.wb->expires)
698
- buffer_cacheable(r->result.wb);
699
- else
700
- buffer_no_cacheable(r->result.wb);
701
-
702
- code = tmp->code;
703
- }
704
- else if (rc == ETIMEDOUT || cancelled) {
705
- // timeout
706
- // we will go away and let the callback free the structure
707
- tmp->free_with_signal = true;
708
- we_should_free = false;
709
-
710
- if(cancelled)
711
- code = rrd_call_function_error(r->result.wb,
712
- "Request cancelled",
713
- HTTP_RESP_CLIENT_CLOSED_REQUEST);
714
- else
715
- code = rrd_call_function_error(r->result.wb,
716
- "Timeout while waiting for a response from the collector.",
717
- HTTP_RESP_GATEWAY_TIMEOUT);
718
- }
719
- else
720
- code = rrd_call_function_error(r->result.wb,
721
- "Internal error while communicating with the collector",
722
- HTTP_RESP_INTERNAL_SERVER_ERROR);
723
-
724
- netdata_mutex_unlock(&tmp->mutex);
725
- }
726
- else {
727
- if(!buffer_strlen(r->result.wb))
728
- rrd_call_function_error(r->result.wb, "The collector returned an error.", code);
729
- }
730
-
731
- if (we_should_free) {
732
- rrd_function_call_wait_free(tmp);
733
- buffer_free(temp_wb);
734
- }
735
-
736
- return code;
737
-}
738
-
739
-static inline int rrd_call_function_async(struct rrd_function_inflight *r, bool wait) {
740
- if(wait)
741
- return rrd_call_function_async_and_wait(r);
742
- else
743
- return rrd_call_function_async_and_dont_wait(r);
744
-}
745
-
746
-
747
-void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data);
748
-// ----------------------------------------------------------------------------
749
-
750
-int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, HTTP_ACCESS access, const char *cmd,
751
- bool wait, const char *transaction,
752
- rrd_function_result_callback_t result_cb, void *result_cb_data,
753
- rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
754
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload) {
755
-
756
- int code;
757
- char sanitized_cmd[PLUGINSD_LINE_MAX + 1];
758
- const DICTIONARY_ITEM *host_function_acquired = NULL;
759
-
760
- // ------------------------------------------------------------------------
761
- // find the function
762
-
763
- size_t sanitized_cmd_length = sanitize_function_text(sanitized_cmd, cmd, PLUGINSD_LINE_MAX);
764
-
765
- if (is_dyncfg_function(sanitized_cmd, DYNCFG_FUNCTION_TYPE_ALL)) {
766
- call_virtual_function_async(result_wb, host, sanitized_cmd, payload, result_cb, result_cb_data);
767
- return HTTP_RESP_OK;
768
- }
769
-
770
- code = rrd_call_function_find(host, result_wb, sanitized_cmd, sanitized_cmd_length, &host_function_acquired);
771
- if(code != HTTP_RESP_OK)
772
- return code;
773
-
774
- struct rrd_host_function *rdcf = dictionary_acquired_item_value(host_function_acquired);
775
-
776
- if(access != HTTP_ACCESS_ADMINS && rdcf->access != HTTP_ACCESS_ANY && access > rdcf->access) {
777
-
778
- if(!aclk_connected)
779
- rrd_call_function_error(result_wb, "This Netdata must be connected to Netdata Cloud to access this function.", HTTP_RESP_PRECOND_FAIL);
780
- else if(access >= HTTP_ACCESS_ANY)
781
- rrd_call_function_error(result_wb, "You need to login to the Netdata Cloud space this agent is claimed to, to access this function.", HTTP_RESP_PRECOND_FAIL);
782
- else /* if(access < HTTP_ACCESS_ANY && rdcf->access < access) */
783
- rrd_call_function_error(result_wb, "To access this function you need to be an admin in this Netdata Cloud space.", HTTP_RESP_PRECOND_FAIL);
784
-
785
- dictionary_acquired_item_release(host->functions, host_function_acquired);
786
- return HTTP_RESP_PRECOND_FAIL;
787
- }
788
-
789
- if(timeout_s <= 0)
790
- timeout_s = rdcf->timeout;
791
-
792
- // ------------------------------------------------------------------------
793
- // validate and parse the transaction, or generate a new transaction id
794
-
795
- char uuid_str[UUID_COMPACT_STR_LEN];
796
- uuid_t uuid;
797
-
798
- if(!transaction || !*transaction || uuid_parse_flexi(transaction, uuid) != 0)
799
- uuid_generate_random(uuid);
800
-
801
- uuid_unparse_lower_compact(uuid, uuid_str);
802
- transaction = uuid_str;
803
-
804
- // ------------------------------------------------------------------------
805
- // the function can only be executed in async mode
806
- // put the function into the inflight requests
807
-
808
- struct rrd_function_inflight t = {
809
- .used = false,
810
- .host = host,
811
- .cmd = strdupz(cmd),
812
- .sanitized_cmd = strdupz(sanitized_cmd),
813
- .sanitized_cmd_length = sanitized_cmd_length,
814
- .transaction = strdupz(transaction),
815
- .timeout = timeout_s,
816
- .cancelled = false,
817
- .stop_monotonic_ut = now_monotonic_usec() + timeout_s * USEC_PER_SEC,
818
- .host_function_acquired = host_function_acquired,
819
- .rdcf = rdcf,
820
- .result = {
821
- .wb = result_wb,
822
- .cb = result_cb,
823
- .data = result_cb_data,
824
- },
825
- .is_cancelled = {
826
- .cb = is_cancelled_cb,
827
- .data = is_cancelled_cb_data,
828
- },
829
- .progress = {
830
- .cb = progress_cb,
831
- .data = progress_cb_data,
832
- },
833
- };
834
- uuid_copy(t.transaction_uuid, uuid);
835
-
836
- struct rrd_function_inflight *r = dictionary_set(rrd_functions_inflight_requests, transaction, &t, sizeof(t));
837
- if(r->used) {
838
- nd_log(NDLS_DAEMON, NDLP_NOTICE,
839
- "FUNCTIONS: duplicate transaction '%s', function: '%s'",
840
- t.transaction, t.cmd);
841
-
842
- code = rrd_call_function_error(result_wb, "duplicate transaction", HTTP_RESP_BAD_REQUEST);
843
- freez((void *)t.transaction);
844
- freez((void *)t.cmd);
845
- freez((void *)t.sanitized_cmd);
846
- dictionary_acquired_item_release(r->host->functions, t.host_function_acquired);
847
- return code;
848
- }
849
- r->used = true;
850
- // internal_error(true, "FUNCTIONS: transaction '%s' started", r->transaction);
851
-
852
- if(r->rdcf->sync) {
853
- // the caller has to wait
854
- code = r->rdcf->execute_cb(&r->transaction_uuid, r->result.wb,
855
- &r->stop_monotonic_ut, r->sanitized_cmd, r->rdcf->execute_cb_data,
856
- r->result.cb, r->result.data,
857
- r->progress.cb, r->progress.data,
858
- r->is_cancelled.cb, r->is_cancelled.data, // it is ok to pass these, we block the caller
859
- NULL, NULL, // no need to register canceller, we will wait
860
- NULL, NULL // ?? do we need a progresser in this case?
861
- );
862
-
863
- if(code != HTTP_RESP_OK && !buffer_strlen(result_wb))
864
- rrd_call_function_error(result_wb, "Collector reported error.", code);
865
-
866
- rrd_inflight_function_cleanup(host, r->transaction);
867
- return code;
868
- }
869
-
870
- return rrd_call_function_async(r, wait);
871
-}
872
-
873
-static void rrd_function_cancel_inflight(struct rrd_function_inflight *r) {
874
- if(!r)
875
- return;
876
-
877
- bool cancelled = __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED);
878
- if(cancelled) {
879
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
880
- "FUNCTIONS: received a CANCEL request for transaction '%s', but it is already cancelled.",
881
- r->transaction);
882
- return;
883
- }
884
-
885
- __atomic_store_n(&r->cancelled, true, __ATOMIC_RELAXED);
886
-
887
- if(!rrd_collector_dispatcher_acquire(r->rdcf->collector)) {
888
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
889
- "FUNCTIONS: received a CANCEL request for transaction '%s', but the collector is not running.",
890
- r->transaction);
891
- return;
892
- }
893
-
894
- if(r->canceller.cb)
895
- r->canceller.cb(r->canceller.data);
896
-
897
- rrd_collector_dispatcher_release(r->rdcf->collector);
898
-}
899
-
900
-void rrd_function_cancel(const char *transaction) {
901
- // internal_error(true, "FUNCTIONS: request to cancel transaction '%s'", transaction);
902
-
903
- const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction);
904
- if(!item) {
905
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
906
- "FUNCTIONS: received a CANCEL request for transaction '%s', but the transaction is not running.",
907
- transaction);
908
- return;
909
- }
910
-
911
- struct rrd_function_inflight *r = dictionary_acquired_item_value(item);
912
- rrd_function_cancel_inflight(r);
913
- dictionary_acquired_item_release(rrd_functions_inflight_requests, item);
914
-}
915
-
916
-void rrd_function_progress(const char *transaction) {
917
- const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction);
918
- if(!item) {
919
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
920
- "FUNCTIONS: received a PROGRESS request for transaction '%s', but the transaction is not running.",
921
- transaction);
922
- return;
923
- }
924
-
925
- struct rrd_function_inflight *r = dictionary_acquired_item_value(item);
926
-
927
- if(!rrd_collector_dispatcher_acquire(r->rdcf->collector)) {
928
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
929
- "FUNCTIONS: received a PROGRESS request for transaction '%s', but the collector is not running.",
930
- transaction);
931
- goto cleanup;
932
- }
933
-
934
- functions_stop_monotonic_update_on_progress(&r->stop_monotonic_ut);
935
-
936
- if(r->progresser.cb)
937
- r->progresser.cb(r->progresser.data);
938
-
939
- rrd_collector_dispatcher_release(r->rdcf->collector);
940
-
941
-cleanup:
942
- dictionary_acquired_item_release(rrd_functions_inflight_requests, item);
943
-}
944
-
945
-void rrd_function_call_progresser(uuid_t *transaction) {
946
- char str[UUID_COMPACT_STR_LEN];
947
- uuid_unparse_lower_compact(*transaction, str);
948
- rrd_function_progress(str);
949
-}
950
-
951
-// ----------------------------------------------------------------------------
952
-
953
-static void functions2json(DICTIONARY *functions, BUFFER *wb)
954
-{
955
- struct rrd_host_function *t;
956
- dfe_start_read(functions, t)
957
- {
958
- if (!rrd_collector_running(t->collector))
959
- continue;
960
-
961
- buffer_json_member_add_object(wb, t_dfe.name);
962
- {
963
- buffer_json_member_add_string_or_empty(wb, "help", string2str(t->help));
964
- buffer_json_member_add_int64(wb, "timeout", (int64_t) t->timeout);
965
-
966
- char options[65];
967
- snprintfz(
968
- options, 64
969
- , "%s%s"
970
- , (t->options & RRD_FUNCTION_LOCAL) ? "LOCAL " : ""
971
- , (t->options & RRD_FUNCTION_GLOBAL) ? "GLOBAL" : ""
972
- );
973
-
974
- buffer_json_member_add_string_or_empty(wb, "options", options);
975
- buffer_json_member_add_string_or_empty(wb, "tags", string2str(t->tags));
976
- buffer_json_member_add_string(wb, "access", http_id2access(t->access));
977
- buffer_json_member_add_uint64(wb, "priority", t->priority);
978
- }
979
- buffer_json_object_close(wb);
980
- }
981
- dfe_done(t);
982
-}
983
-
984
-void chart_functions2json(RRDSET *st, BUFFER *wb) {
985
- if(!st || !st->functions_view) return;
986
-
987
- functions2json(st->functions_view, wb);
988
-}
989
-
990
-void host_functions2json(RRDHOST *host, BUFFER *wb) {
991
- if(!host || !host->functions) return;
992
-
993
- buffer_json_member_add_object(wb, "functions");
994
-
995
- struct rrd_host_function *t;
996
- dfe_start_read(host->functions, t) {
997
- if(!rrd_collector_running(t->collector)) continue;
998
-
999
- buffer_json_member_add_object(wb, t_dfe.name);
1000
- {
1001
- buffer_json_member_add_string(wb, "help", string2str(t->help));
1002
- buffer_json_member_add_int64(wb, "timeout", t->timeout);
1003
- buffer_json_member_add_array(wb, "options");
1004
- {
1005
- if (t->options & RRD_FUNCTION_GLOBAL)
1006
- buffer_json_add_array_item_string(wb, "GLOBAL");
1007
- if (t->options & RRD_FUNCTION_LOCAL)
1008
- buffer_json_add_array_item_string(wb, "LOCAL");
1009
- }
1010
- buffer_json_array_close(wb);
1011
- buffer_json_member_add_string(wb, "tags", string2str(t->tags));
1012
- buffer_json_member_add_string(wb, "access", http_id2access(t->access));
1013
- buffer_json_member_add_uint64(wb, "priority", t->priority);
1014
- }
1015
- buffer_json_object_close(wb);
1016
- }
1017
- dfe_done(t);
1018
-
1019
- buffer_json_object_close(wb);
1020
-}
1021
-
1022
-void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst, void *value, size_t value_size) {
1023
- if(!rrdset_functions_view || !dst) return;
358
+bool rrd_function_available(RRDHOST *host, const char *function) {
359
+ if(!host || !host->functions)
360
+ return false;
361
1025
- struct rrd_host_function *t;
1026
- dfe_start_read(rrdset_functions_view, t) {
1027
- if(!rrd_collector_running(t->collector)) continue;
362
+ bool ret = false;
363
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(host->functions, function);
364
+ if(item) {
365
+ struct rrd_host_function *rdcf = dictionary_acquired_item_value(item);
366
+ if(rrd_collector_running(rdcf->collector))
367
+ ret = true;
368
1029
- dictionary_set(dst, t_dfe.name, value, value_size);
369
+ dictionary_acquired_item_release(host->functions, item);
370
}
1031
- dfe_done(t);
1032
-}
1033
-
1034
-void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help, STRING **tags, HTTP_ACCESS *access, int *priority) {
1035
- if(!host || !host->functions || !dictionary_entries(host->functions) || !dst) return;
1036
-
1037
- struct rrd_host_function *t;
1038
- dfe_start_read(host->functions, t) {
1039
- if(!rrd_collector_running(t->collector)) continue;
1040
-
1041
- if(help)
1042
- *help = t->help;
1043
-
1044
- if(tags)
1045
- *tags = t->tags;
1046
-
1047
- if(access)
1048
- *access = t->access;
1049
-
1050
- if(priority)
1051
- *priority = t->priority;
1052
-
1053
- dictionary_set(dst, t_dfe.name, value, value_size);
1054
- }
1055
- dfe_done(t);
1056
-}
1057
-
1058
-// ----------------------------------------------------------------------------
1059
-
1060
-int rrdhost_function_progress(uuid_t *transaction __maybe_unused, BUFFER *wb,
1061
- usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
1062
- void *collector_data __maybe_unused,
1063
- rrd_function_result_callback_t result_cb, void *result_cb_data,
1064
- rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
1065
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
1066
- rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
1067
- void *register_canceller_cb_data __maybe_unused,
1068
- rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
1069
- void *register_progresser_cb_data __maybe_unused) {
1070
-
1071
- int response = progress_function_result(wb, rrdhost_hostname(localhost));
1072
-
1073
- if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) {
1074
- buffer_flush(wb);
1075
- response = HTTP_RESP_CLIENT_CLOSED_REQUEST;
1076
- }
1077
-
1078
- if(result_cb)
1079
- result_cb(wb, response, result_cb_data);
1080
-
1081
- return response;
1082
-}
1083
-
1084
-int rrdhost_function_streaming(uuid_t *transaction __maybe_unused, BUFFER *wb,
1085
- usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
1086
- void *collector_data __maybe_unused,
1087
- rrd_function_result_callback_t result_cb, void *result_cb_data,
1088
- rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
1089
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
1090
- rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
1091
- void *register_canceller_cb_data __maybe_unused,
1092
- rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
1093
- void *register_progresser_cb_data __maybe_unused) {
1094
-
1095
- time_t now = now_realtime_sec();
1096
-
1097
- buffer_flush(wb);
1098
- wb->content_type = CT_APPLICATION_JSON;
1099
- buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
1100
-
1101
- buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(localhost));
1102
- buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
1103
- buffer_json_member_add_string(wb, "type", "table");
1104
- buffer_json_member_add_time_t(wb, "update_every", 1);
1105
- buffer_json_member_add_string(wb, "help", RRDFUNCTIONS_STREAMING_HELP);
1106
- buffer_json_member_add_array(wb, "data");
1107
-
1108
- size_t max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_MAX] = { 0 };
1109
- size_t max_db_metrics = 0, max_db_instances = 0, max_db_contexts = 0;
1110
- size_t max_collection_replication_instances = 0, max_streaming_replication_instances = 0;
1111
- size_t max_ml_anomalous = 0, max_ml_normal = 0, max_ml_trained = 0, max_ml_pending = 0, max_ml_silenced = 0;
1112
- {
1113
- RRDHOST *host;
1114
- dfe_start_read(rrdhost_root_index, host) {
1115
- RRDHOST_STATUS s;
1116
- rrdhost_status(host, now, &s);
1117
- buffer_json_add_array_item_array(wb);
1118
-
1119
- if(s.db.metrics > max_db_metrics)
1120
- max_db_metrics = s.db.metrics;
1121
-
1122
- if(s.db.instances > max_db_instances)
1123
- max_db_instances = s.db.instances;
1124
-
1125
- if(s.db.contexts > max_db_contexts)
1126
- max_db_contexts = s.db.contexts;
1127
-
1128
- if(s.ingest.replication.instances > max_collection_replication_instances)
1129
- max_collection_replication_instances = s.ingest.replication.instances;
1130
-
1131
- if(s.stream.replication.instances > max_streaming_replication_instances)
1132
- max_streaming_replication_instances = s.stream.replication.instances;
1133
-
1134
- for(int i = 0; i < STREAM_TRAFFIC_TYPE_MAX ;i++) {
1135
- if (s.stream.sent_bytes_on_this_connection_per_type[i] >
1136
- max_sent_bytes_on_this_connection_per_type[i])
1137
- max_sent_bytes_on_this_connection_per_type[i] =
1138
- s.stream.sent_bytes_on_this_connection_per_type[i];
1139
- }
1140
-
1141
- // retention
1142
- buffer_json_add_array_item_string(wb, rrdhost_hostname(s.host)); // Node
1143
- buffer_json_add_array_item_uint64(wb, s.db.first_time_s * MSEC_PER_SEC); // dbFrom
1144
- buffer_json_add_array_item_uint64(wb, s.db.last_time_s * MSEC_PER_SEC); // dbTo
1145
-
1146
- if(s.db.first_time_s && s.db.last_time_s && s.db.last_time_s > s.db.first_time_s)
1147
- buffer_json_add_array_item_uint64(wb, s.db.last_time_s - s.db.first_time_s); // dbDuration
1148
- else
1149
- buffer_json_add_array_item_string(wb, NULL); // dbDuration
1150
-
1151
- buffer_json_add_array_item_uint64(wb, s.db.metrics); // dbMetrics
1152
- buffer_json_add_array_item_uint64(wb, s.db.instances); // dbInstances
1153
- buffer_json_add_array_item_uint64(wb, s.db.contexts); // dbContexts
1154
-
1155
- // statuses
1156
- buffer_json_add_array_item_string(wb, rrdhost_ingest_status_to_string(s.ingest.status)); // InStatus
1157
- buffer_json_add_array_item_string(wb, rrdhost_streaming_status_to_string(s.stream.status)); // OutStatus
1158
- buffer_json_add_array_item_string(wb, rrdhost_ml_status_to_string(s.ml.status)); // MLStatus
1159
-
1160
- // collection
1161
- if(s.ingest.since) {
1162
- buffer_json_add_array_item_uint64(wb, s.ingest.since * MSEC_PER_SEC); // InSince
1163
- buffer_json_add_array_item_time_t(wb, s.now - s.ingest.since); // InAge
1164
- }
1165
- else {
1166
- buffer_json_add_array_item_string(wb, NULL); // InSince
1167
- buffer_json_add_array_item_string(wb, NULL); // InAge
1168
- }
1169
- buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(s.ingest.reason)); // InReason
1170
- buffer_json_add_array_item_uint64(wb, s.ingest.hops); // InHops
1171
- buffer_json_add_array_item_double(wb, s.ingest.replication.completion); // InReplCompletion
1172
- buffer_json_add_array_item_uint64(wb, s.ingest.replication.instances); // InReplInstances
1173
- buffer_json_add_array_item_string(wb, s.ingest.peers.local.ip); // InLocalIP
1174
- buffer_json_add_array_item_uint64(wb, s.ingest.peers.local.port); // InLocalPort
1175
- buffer_json_add_array_item_string(wb, s.ingest.peers.peer.ip); // InRemoteIP
1176
- buffer_json_add_array_item_uint64(wb, s.ingest.peers.peer.port); // InRemotePort
1177
- buffer_json_add_array_item_string(wb, s.ingest.ssl ? "SSL" : "PLAIN"); // InSSL
1178
- stream_capabilities_to_json_array(wb, s.ingest.capabilities, NULL); // InCapabilities
1179
-
1180
- // streaming
1181
- if(s.stream.since) {
1182
- buffer_json_add_array_item_uint64(wb, s.stream.since * MSEC_PER_SEC); // OutSince
1183
- buffer_json_add_array_item_time_t(wb, s.now - s.stream.since); // OutAge
1184
- }
1185
- else {
1186
- buffer_json_add_array_item_string(wb, NULL); // OutSince
1187
- buffer_json_add_array_item_string(wb, NULL); // OutAge
1188
- }
1189
- buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(s.stream.reason)); // OutReason
1190
- buffer_json_add_array_item_uint64(wb, s.stream.hops); // OutHops
1191
- buffer_json_add_array_item_double(wb, s.stream.replication.completion); // OutReplCompletion
1192
- buffer_json_add_array_item_uint64(wb, s.stream.replication.instances); // OutReplInstances
1193
- buffer_json_add_array_item_string(wb, s.stream.peers.local.ip); // OutLocalIP
1194
- buffer_json_add_array_item_uint64(wb, s.stream.peers.local.port); // OutLocalPort
1195
- buffer_json_add_array_item_string(wb, s.stream.peers.peer.ip); // OutRemoteIP
1196
- buffer_json_add_array_item_uint64(wb, s.stream.peers.peer.port); // OutRemotePort
1197
- buffer_json_add_array_item_string(wb, s.stream.ssl ? "SSL" : "PLAIN"); // OutSSL
1198
- buffer_json_add_array_item_string(wb, s.stream.compression ? "COMPRESSED" : "UNCOMPRESSED"); // OutCompression
1199
- stream_capabilities_to_json_array(wb, s.stream.capabilities, NULL); // OutCapabilities
1200
- buffer_json_add_array_item_uint64(wb, s.stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA]);
1201
- buffer_json_add_array_item_uint64(wb, s.stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA]);
1202
- buffer_json_add_array_item_uint64(wb, s.stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION]);
1203
- buffer_json_add_array_item_uint64(wb, s.stream.sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS]);
1204
-
1205
- buffer_json_add_array_item_array(wb); // OutAttemptHandshake
1206
- time_t last_attempt = 0;
1207
- for(struct rrdpush_destinations *d = host->destinations; d ; d = d->next) {
1208
- if(d->since > last_attempt)
1209
- last_attempt = d->since;
1210
-
1211
- buffer_json_add_array_item_string(wb, stream_handshake_error_to_string(d->reason));
1212
- }
1213
- buffer_json_array_close(wb); // // OutAttemptHandshake
1214
-
1215
- if(!last_attempt) {
1216
- buffer_json_add_array_item_string(wb, NULL); // OutAttemptSince
1217
- buffer_json_add_array_item_string(wb, NULL); // OutAttemptAge
1218
- }
1219
- else {
1220
- buffer_json_add_array_item_uint64(wb, last_attempt * 1000); // OutAttemptSince
1221
- buffer_json_add_array_item_time_t(wb, s.now - last_attempt); // OutAttemptAge
1222
- }
1223
-
1224
- // ML
1225
- if(s.ml.status == RRDHOST_ML_STATUS_RUNNING) {
1226
- buffer_json_add_array_item_uint64(wb, s.ml.metrics.anomalous); // MlAnomalous
1227
- buffer_json_add_array_item_uint64(wb, s.ml.metrics.normal); // MlNormal
1228
- buffer_json_add_array_item_uint64(wb, s.ml.metrics.trained); // MlTrained
1229
- buffer_json_add_array_item_uint64(wb, s.ml.metrics.pending); // MlPending
1230
- buffer_json_add_array_item_uint64(wb, s.ml.metrics.silenced); // MlSilenced
1231
-
1232
- if(s.ml.metrics.anomalous > max_ml_anomalous)
1233
- max_ml_anomalous = s.ml.metrics.anomalous;
1234
-
1235
- if(s.ml.metrics.normal > max_ml_normal)
1236
- max_ml_normal = s.ml.metrics.normal;
1237
-
1238
- if(s.ml.metrics.trained > max_ml_trained)
1239
- max_ml_trained = s.ml.metrics.trained;
1240
-
1241
- if(s.ml.metrics.pending > max_ml_pending)
1242
- max_ml_pending = s.ml.metrics.pending;
1243
-
1244
- if(s.ml.metrics.silenced > max_ml_silenced)
1245
- max_ml_silenced = s.ml.metrics.silenced;
1246
-
1247
- }
1248
- else {
1249
- buffer_json_add_array_item_string(wb, NULL); // MlAnomalous
1250
- buffer_json_add_array_item_string(wb, NULL); // MlNormal
1251
- buffer_json_add_array_item_string(wb, NULL); // MlTrained
1252
- buffer_json_add_array_item_string(wb, NULL); // MlPending
1253
- buffer_json_add_array_item_string(wb, NULL); // MlSilenced
1254
- }
1255
-
1256
- // close
1257
- buffer_json_array_close(wb);
1258
- }
1259
- dfe_done(host);
1260
- }
1261
- buffer_json_array_close(wb); // data
1262
- buffer_json_member_add_object(wb, "columns");
1263
- {
1264
- size_t field_id = 0;
1265
-
1266
- // Node
1267
- buffer_rrdf_table_add_field(wb, field_id++, "Node", "Node's Hostname",
1268
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1269
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1270
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1271
- RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY | RRDF_FIELD_OPTS_STICKY,
1272
- NULL);
1273
-
1274
- buffer_rrdf_table_add_field(wb, field_id++, "dbFrom", "DB Data Retention From",
1275
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1276
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1277
- RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1278
- RRDF_FIELD_OPTS_NONE, NULL);
1279
-
1280
- buffer_rrdf_table_add_field(wb, field_id++, "dbTo", "DB Data Retention To",
1281
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1282
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1283
- RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1284
- RRDF_FIELD_OPTS_NONE, NULL);
1285
-
1286
- buffer_rrdf_table_add_field(wb, field_id++, "dbDuration", "DB Data Retention Duration",
1287
- RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
1288
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1289
- RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1290
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1291
-
1292
- buffer_rrdf_table_add_field(wb, field_id++, "dbMetrics", "Time-series Metrics in the DB",
1293
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1294
- 0, NULL, (double)max_db_metrics, RRDF_FIELD_SORT_DESCENDING, NULL,
1295
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1296
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1297
-
1298
- buffer_rrdf_table_add_field(wb, field_id++, "dbInstances", "Instances in the DB",
1299
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1300
- 0, NULL, (double)max_db_instances, RRDF_FIELD_SORT_DESCENDING, NULL,
1301
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1302
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1303
-
1304
- buffer_rrdf_table_add_field(wb, field_id++, "dbContexts", "Contexts in the DB",
1305
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1306
- 0, NULL, (double)max_db_contexts, RRDF_FIELD_SORT_DESCENDING, NULL,
1307
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1308
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1309
-
1310
- // --- statuses ---
1311
-
1312
- buffer_rrdf_table_add_field(wb, field_id++, "InStatus", "Data Collection Online Status",
1313
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1314
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1315
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1316
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1317
-
1318
-
1319
- buffer_rrdf_table_add_field(wb, field_id++, "OutStatus", "Streaming Online Status",
1320
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1321
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1322
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1323
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1324
-
1325
- buffer_rrdf_table_add_field(wb, field_id++, "MlStatus", "ML Status",
1326
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1327
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1328
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1329
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1330
-
1331
- // --- collection ---
1332
-
1333
- buffer_rrdf_table_add_field(wb, field_id++, "InSince", "Last Data Collection Status Change",
1334
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1335
- 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
1336
- RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1337
- RRDF_FIELD_OPTS_NONE, NULL);
1338
-
1339
- buffer_rrdf_table_add_field(wb, field_id++, "InAge", "Last Data Collection Online Status Change Age",
1340
- RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
1341
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1342
- RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1343
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1344
-
1345
- buffer_rrdf_table_add_field(wb, field_id++, "InReason", "Data Collection Online Status Reason",
1346
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1347
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1348
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1349
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1350
-
1351
- buffer_rrdf_table_add_field(wb, field_id++, "InHops", "Data Collection Distance Hops from Origin Node",
1352
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1353
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1354
- RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1355
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1356
-
1357
- buffer_rrdf_table_add_field(wb, field_id++, "InReplCompletion", "Inbound Replication Completion",
1358
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
1359
- 1, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
1360
- RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1361
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1362
-
1363
- buffer_rrdf_table_add_field(wb, field_id++, "InReplInstances", "Inbound Replicating Instances",
1364
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1365
- 0, "instances", (double)max_collection_replication_instances, RRDF_FIELD_SORT_DESCENDING,
1366
- NULL,
1367
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1368
- RRDF_FIELD_OPTS_NONE, NULL);
1369
-
1370
- buffer_rrdf_table_add_field(wb, field_id++, "InLocalIP", "Inbound Local IP",
1371
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1372
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1373
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1374
- RRDF_FIELD_OPTS_NONE, NULL);
1375
-
1376
- buffer_rrdf_table_add_field(wb, field_id++, "InLocalPort", "Inbound Local Port",
1377
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1378
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1379
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE,
1380
- RRDF_FIELD_OPTS_NONE, NULL);
1381
-
1382
- buffer_rrdf_table_add_field(wb, field_id++, "InRemoteIP", "Inbound Remote IP",
1383
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1384
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1385
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1386
- RRDF_FIELD_OPTS_NONE, NULL);
1387
-
1388
- buffer_rrdf_table_add_field(wb, field_id++, "InRemotePort", "Inbound Remote Port",
1389
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1390
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1391
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE,
1392
- RRDF_FIELD_OPTS_NONE, NULL);
1393
-
1394
- buffer_rrdf_table_add_field(wb, field_id++, "InSSL", "Inbound SSL Connection",
1395
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1396
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1397
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1398
- RRDF_FIELD_OPTS_NONE, NULL);
1399
-
1400
- buffer_rrdf_table_add_field(wb, field_id++, "InCapabilities", "Inbound Connection Capabilities",
1401
- RRDF_FIELD_TYPE_ARRAY, RRDF_FIELD_VISUAL_PILL, RRDF_FIELD_TRANSFORM_NONE,
1402
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1403
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1404
- RRDF_FIELD_OPTS_NONE, NULL);
1405
-
1406
- // --- streaming ---
1407
-
1408
- buffer_rrdf_table_add_field(wb, field_id++, "OutSince", "Last Streaming Status Change",
1409
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1410
- 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
1411
- RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1412
- RRDF_FIELD_OPTS_NONE, NULL);
1413
-
1414
- buffer_rrdf_table_add_field(wb, field_id++, "OutAge", "Last Streaming Status Change Age",
1415
- RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
1416
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1417
- RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1418
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1419
-
1420
- buffer_rrdf_table_add_field(wb, field_id++, "OutReason", "Streaming Status Reason",
1421
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1422
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1423
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1424
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1425
-
1426
- buffer_rrdf_table_add_field(wb, field_id++, "OutHops", "Streaming Distance Hops from Origin Node",
1427
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1428
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1429
- RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1430
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1431
-
1432
- buffer_rrdf_table_add_field(wb, field_id++, "OutReplCompletion", "Outbound Replication Completion",
1433
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_BAR, RRDF_FIELD_TRANSFORM_NUMBER,
1434
- 1, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
1435
- RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1436
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1437
-
1438
- buffer_rrdf_table_add_field(wb, field_id++, "OutReplInstances", "Outbound Replicating Instances",
1439
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1440
- 0, "instances", (double)max_streaming_replication_instances, RRDF_FIELD_SORT_DESCENDING,
1441
- NULL,
1442
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1443
- RRDF_FIELD_OPTS_NONE, NULL);
1444
-
1445
- buffer_rrdf_table_add_field(wb, field_id++, "OutLocalIP", "Outbound Local IP",
1446
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1447
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1448
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1449
- RRDF_FIELD_OPTS_NONE, NULL);
1450
-
1451
- buffer_rrdf_table_add_field(wb, field_id++, "OutLocalPort", "Outbound Local Port",
1452
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1453
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1454
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE,
1455
- RRDF_FIELD_OPTS_NONE, NULL);
1456
-
1457
- buffer_rrdf_table_add_field(wb, field_id++, "OutRemoteIP", "Outbound Remote IP",
1458
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1459
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1460
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1461
- RRDF_FIELD_OPTS_NONE, NULL);
1462
-
1463
- buffer_rrdf_table_add_field(wb, field_id++, "OutRemotePort", "Outbound Remote Port",
1464
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1465
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1466
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE,
1467
- RRDF_FIELD_OPTS_NONE, NULL);
1468
-
1469
- buffer_rrdf_table_add_field(wb, field_id++, "OutSSL", "Outbound SSL Connection",
1470
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1471
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1472
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1473
- RRDF_FIELD_OPTS_NONE, NULL);
1474
-
1475
- buffer_rrdf_table_add_field(wb, field_id++, "OutCompression", "Outbound Compressed Connection",
1476
- RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
1477
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1478
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1479
- RRDF_FIELD_OPTS_NONE, NULL);
1480
-
1481
- buffer_rrdf_table_add_field(wb, field_id++, "OutCapabilities", "Outbound Connection Capabilities",
1482
- RRDF_FIELD_TYPE_ARRAY, RRDF_FIELD_VISUAL_PILL, RRDF_FIELD_TRANSFORM_NONE,
1483
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1484
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1485
- RRDF_FIELD_OPTS_NONE, NULL);
1486
-
1487
- buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficData", "Outbound Metric Data Traffic",
1488
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1489
- 0, "bytes", (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA],
1490
- RRDF_FIELD_SORT_DESCENDING, NULL,
1491
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1492
- RRDF_FIELD_OPTS_NONE, NULL);
1493
-
1494
- buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficMetadata", "Outbound Metric Metadata Traffic",
1495
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1496
- 0, "bytes",
1497
- (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA],
1498
- RRDF_FIELD_SORT_DESCENDING, NULL,
1499
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1500
- RRDF_FIELD_OPTS_NONE, NULL);
1501
-
1502
- buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficReplication", "Outbound Metric Replication Traffic",
1503
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1504
- 0, "bytes",
1505
- (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION],
1506
- RRDF_FIELD_SORT_DESCENDING, NULL,
1507
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1508
- RRDF_FIELD_OPTS_NONE, NULL);
1509
-
1510
- buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficFunctions", "Outbound Metric Functions Traffic",
1511
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1512
- 0, "bytes",
1513
- (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS],
1514
- RRDF_FIELD_SORT_DESCENDING, NULL,
1515
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1516
- RRDF_FIELD_OPTS_NONE, NULL);
1517
-
1518
- buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptHandshake",
1519
- "Outbound Connection Attempt Handshake Status",
1520
- RRDF_FIELD_TYPE_ARRAY, RRDF_FIELD_VISUAL_PILL, RRDF_FIELD_TRANSFORM_NONE,
1521
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1522
- RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
1523
- RRDF_FIELD_OPTS_NONE, NULL);
1524
-
1525
- buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptSince",
1526
- "Last Outbound Connection Attempt Status Change Time",
1527
- RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_MS,
1528
- 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
1529
- RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
1530
- RRDF_FIELD_OPTS_NONE, NULL);
1531
-
1532
- buffer_rrdf_table_add_field(wb, field_id++, "OutAttemptAge",
1533
- "Last Outbound Connection Attempt Status Change Age",
1534
- RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DURATION_S,
1535
- 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
1536
- RRDF_FIELD_SUMMARY_MIN, RRDF_FIELD_FILTER_RANGE,
1537
- RRDF_FIELD_OPTS_VISIBLE, NULL);
1538
-
1539
- // --- ML ---
1540
-
1541
- buffer_rrdf_table_add_field(wb, field_id++, "MlAnomalous", "Number of Anomalous Metrics",
1542
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1543
- 0, "metrics",
1544
- (double)max_ml_anomalous,
1545
- RRDF_FIELD_SORT_DESCENDING, NULL,
1546
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1547
- RRDF_FIELD_OPTS_NONE, NULL);
1548
-
1549
- buffer_rrdf_table_add_field(wb, field_id++, "MlNormal", "Number of Not Anomalous Metrics",
1550
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1551
- 0, "metrics",
1552
- (double)max_ml_normal,
1553
- RRDF_FIELD_SORT_DESCENDING, NULL,
1554
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1555
- RRDF_FIELD_OPTS_NONE, NULL);
1556
-
1557
- buffer_rrdf_table_add_field(wb, field_id++, "MlTrained", "Number of Trained Metrics",
1558
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1559
- 0, "metrics",
1560
- (double)max_ml_trained,
1561
- RRDF_FIELD_SORT_DESCENDING, NULL,
1562
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1563
- RRDF_FIELD_OPTS_NONE, NULL);
1564
-
1565
- buffer_rrdf_table_add_field(wb, field_id++, "MlPending", "Number of Pending Metrics",
1566
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1567
- 0, "metrics",
1568
- (double)max_ml_pending,
1569
- RRDF_FIELD_SORT_DESCENDING, NULL,
1570
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1571
- RRDF_FIELD_OPTS_NONE, NULL);
1572
-
1573
- buffer_rrdf_table_add_field(wb, field_id++, "MlSilenced", "Number of Silenced Metrics",
1574
- RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1575
- 0, "metrics",
1576
- (double)max_ml_silenced,
1577
- RRDF_FIELD_SORT_DESCENDING, NULL,
1578
- RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1579
- RRDF_FIELD_OPTS_NONE, NULL);
1580
- }
1581
- buffer_json_object_close(wb); // columns
1582
- buffer_json_member_add_string(wb, "default_sort_column", "Node");
1583
- buffer_json_member_add_object(wb, "charts");
1584
- {
1585
- // Data Collection Age chart
1586
- buffer_json_member_add_object(wb, "InAge");
1587
- {
1588
- buffer_json_member_add_string(wb, "name", "Data Collection Age");
1589
- buffer_json_member_add_string(wb, "type", "stacked-bar");
1590
- buffer_json_member_add_array(wb, "columns");
1591
- {
1592
- buffer_json_add_array_item_string(wb, "InAge");
1593
- }
1594
- buffer_json_array_close(wb);
1595
- }
1596
- buffer_json_object_close(wb);
1597
-
1598
- // Streaming Age chart
1599
- buffer_json_member_add_object(wb, "OutAge");
1600
- {
1601
- buffer_json_member_add_string(wb, "name", "Streaming Age");
1602
- buffer_json_member_add_string(wb, "type", "stacked-bar");
1603
- buffer_json_member_add_array(wb, "columns");
1604
- {
1605
- buffer_json_add_array_item_string(wb, "OutAge");
1606
- }
1607
- buffer_json_array_close(wb);
1608
- }
1609
- buffer_json_object_close(wb);
1610
-
1611
- // DB Duration
1612
- buffer_json_member_add_object(wb, "dbDuration");
1613
- {
1614
- buffer_json_member_add_string(wb, "name", "Retention Duration");
1615
- buffer_json_member_add_string(wb, "type", "stacked-bar");
1616
- buffer_json_member_add_array(wb, "columns");
1617
- {
1618
- buffer_json_add_array_item_string(wb, "dbDuration");
1619
- }
1620
- buffer_json_array_close(wb);
1621
- }
1622
- buffer_json_object_close(wb);
1623
- }
1624
- buffer_json_object_close(wb); // charts
1625
-
1626
- buffer_json_member_add_array(wb, "default_charts");
1627
- {
1628
- buffer_json_add_array_item_array(wb);
1629
- buffer_json_add_array_item_string(wb, "InAge");
1630
- buffer_json_add_array_item_string(wb, "Node");
1631
- buffer_json_array_close(wb);
1632
-
1633
- buffer_json_add_array_item_array(wb);
1634
- buffer_json_add_array_item_string(wb, "OutAge");
1635
- buffer_json_add_array_item_string(wb, "Node");
1636
- buffer_json_array_close(wb);
1637
- }
1638
- buffer_json_array_close(wb);
1639
-
1640
- buffer_json_member_add_object(wb, "group_by");
1641
- {
1642
- buffer_json_member_add_object(wb, "Node");
1643
- {
1644
- buffer_json_member_add_string(wb, "name", "Node");
1645
- buffer_json_member_add_array(wb, "columns");
1646
- {
1647
- buffer_json_add_array_item_string(wb, "Node");
1648
- }
1649
- buffer_json_array_close(wb);
1650
- }
1651
- buffer_json_object_close(wb);
1652
-
1653
- buffer_json_member_add_object(wb, "InStatus");
1654
- {
1655
- buffer_json_member_add_string(wb, "name", "Nodes by Collection Status");
1656
- buffer_json_member_add_array(wb, "columns");
1657
- {
1658
- buffer_json_add_array_item_string(wb, "InStatus");
1659
- }
1660
- buffer_json_array_close(wb);
1661
- }
1662
- buffer_json_object_close(wb);
1663
-
1664
- buffer_json_member_add_object(wb, "OutStatus");
1665
- {
1666
- buffer_json_member_add_string(wb, "name", "Nodes by Streaming Status");
1667
- buffer_json_member_add_array(wb, "columns");
1668
- {
1669
- buffer_json_add_array_item_string(wb, "OutStatus");
1670
- }
1671
- buffer_json_array_close(wb);
1672
- }
1673
- buffer_json_object_close(wb);
1674
-
1675
- buffer_json_member_add_object(wb, "MlStatus");
1676
- {
1677
- buffer_json_member_add_string(wb, "name", "Nodes by ML Status");
1678
- buffer_json_member_add_array(wb, "columns");
1679
- {
1680
- buffer_json_add_array_item_string(wb, "MlStatus");
1681
- }
1682
- buffer_json_array_close(wb);
1683
- }
1684
- buffer_json_object_close(wb);
1685
-
1686
- buffer_json_member_add_object(wb, "InRemoteIP");
1687
- {
1688
- buffer_json_member_add_string(wb, "name", "Nodes by Inbound IP");
1689
- buffer_json_member_add_array(wb, "columns");
1690
- {
1691
- buffer_json_add_array_item_string(wb, "InRemoteIP");
1692
- }
1693
- buffer_json_array_close(wb);
1694
- }
1695
- buffer_json_object_close(wb);
1696
-
1697
- buffer_json_member_add_object(wb, "OutRemoteIP");
1698
- {
1699
- buffer_json_member_add_string(wb, "name", "Nodes by Outbound IP");
1700
- buffer_json_member_add_array(wb, "columns");
1701
- {
1702
- buffer_json_add_array_item_string(wb, "OutRemoteIP");
1703
- }
1704
- buffer_json_array_close(wb);
1705
- }
1706
- buffer_json_object_close(wb);
1707
- }
1708
- buffer_json_object_close(wb); // group_by
1709
-
1710
- buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + 1);
1711
- buffer_json_finalize(wb);
1712
-
1713
- int response = HTTP_RESP_OK;
1714
- if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) {
1715
- buffer_flush(wb);
1716
- response = HTTP_RESP_CLIENT_CLOSED_REQUEST;
1717
- }
1718
-
1719
- if(result_cb)
1720
- result_cb(wb, response, result_cb_data);
371
1722
- return response;
372
+ return ret;
373
}
database/rrdfunctions.h
+56
-45
@@ -4,7 +4,7 @@
4
5
// ----------------------------------------------------------------------------
6
7
-#include "rrd.h"
7
+#include "../libnetdata/libnetdata.h"
8
9
#define RRDFUNCTIONS_PRIORITY_DEFAULT 100
10
@@ -18,65 +18,76 @@ typedef void (*rrd_function_progress_cb_t)(void *data, size_t done, size_t all);
18
typedef void (*rrd_function_progresser_cb_t)(void *data);
19
typedef void (*rrd_function_register_progresser_cb_t)(void *register_progresser_cb_data, rrd_function_progresser_cb_t progresser_cb, void *progresser_cb_data);
20
21
-typedef int (*rrd_function_execute_cb_t)(uuid_t *transaction, BUFFER *wb,
22
- usec_t *stop_monotonic_ut, const char *function, void *collector_data,
23
- rrd_function_result_callback_t result_cb, void *result_cb_data,
24
- rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
25
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
26
- rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
27
- rrd_function_register_progresser_cb_t register_progresser_cb, void *register_progresser_cb_data);
21
+struct rrd_function_execute {
22
+ uuid_t *transaction;
23
+ const char *function;
24
+ BUFFER *payload;
25
+ const char *source;
26
+
27
+ usec_t *stop_monotonic_ut;
28
+
29
+ struct {
30
+ BUFFER *wb; // the response should be written here
31
+ rrd_function_result_callback_t cb;
32
+ void *data;
33
+ } result;
34
+
35
+ struct {
36
+ rrd_function_progress_cb_t cb;
37
+ void *data;
38
+ } progress;
39
+
40
+ struct {
41
+ rrd_function_is_cancelled_cb_t cb;
42
+ void *data;
43
+ } is_cancelled;
44
29
-void rrd_functions_inflight_init(void);
30
-void rrdfunctions_host_init(RRDHOST *host);
31
-void rrdfunctions_host_destroy(RRDHOST *host);
45
+ struct {
46
+ rrd_function_register_canceller_cb_t cb;
47
+ void *data;
48
+ } register_canceller;
49
+
50
+ struct {
51
+ rrd_function_register_progresser_cb_t cb;
52
+ void *data;
53
+ } register_progresser;
54
+};
55
+
56
+typedef int (*rrd_function_execute_cb_t)(struct rrd_function_execute *rfe, void *data);
57
+
58
+
59
+// ----------------------------------------------------------------------------
60
+
61
+#include "rrd.h"
62
+
63
+void rrd_functions_host_init(RRDHOST *host);
64
+void rrd_functions_host_destroy(RRDHOST *host);
65
66
// add a function, to be run from the collector
67
void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority, const char *help, const char *tags,
68
HTTP_ACCESS access, bool sync, rrd_function_execute_cb_t execute_cb,
69
void *execute_cb_data);
70
71
+void rrd_function_del(RRDHOST *host, RRDSET *st, const char *name);
72
+
73
// call a function, to be run from anywhere
74
int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, HTTP_ACCESS access, const char *cmd,
75
bool wait, const char *transaction,
76
rrd_function_result_callback_t result_cb, void *result_cb_data,
77
rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
43
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload);
44
-
45
-// cancel a running function, to be run from anywhere
46
-void rrd_function_cancel(const char *transaction);
47
-void rrd_function_progress(const char *transaction);
48
-void rrd_function_call_progresser(uuid_t *transaction);
78
+ rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
79
+ BUFFER *payload, const char *source);
80
50
-void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb);
51
-void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb);
81
+int rrd_call_function_error(BUFFER *wb, const char *msg, int code);
82
53
-void chart_functions2json(RRDSET *st, BUFFER *wb);
54
-void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst, void *value, size_t value_size);
55
-void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help, STRING **tags, HTTP_ACCESS *access, int *priority);
56
-void host_functions2json(RRDHOST *host, BUFFER *wb);
83
+bool rrd_function_available(RRDHOST *host, const char *function);
84
58
-uint8_t functions_format_to_content_type(const char *format);
59
-const char *functions_content_type_to_format(HTTP_CONTENT_TYPE content_type);
60
-int rrd_call_function_error(BUFFER *wb, const char *msg, int code);
85
+bool rrd_function_has_this_original_result_callback(uuid_t *transaction, rrd_function_result_callback_t cb);
86
62
-int rrdhost_function_progress(uuid_t *transaction, BUFFER *wb,
63
- usec_t *stop_monotonic_ut, const char *function, void *collector_data,
64
- rrd_function_result_callback_t result_cb, void *result_cb_data,
65
- rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
66
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
67
- rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
68
- rrd_function_register_progresser_cb_t register_progresser_cb,
69
- void *register_progresser_cb_data);
70
-
71
-int rrdhost_function_streaming(uuid_t *transaction, BUFFER *wb,
72
- usec_t *stop_monotonic_ut, const char *function, void *collector_data,
73
- rrd_function_result_callback_t result_cb, void *result_cb_data,
74
- rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
75
- rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
76
- rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
77
- rrd_function_register_progresser_cb_t register_progresser_cb,
78
- void *register_progresser_cb_data);
79
-
80
-#define RRDFUNCTIONS_STREAMING_HELP "Streaming status for parents and children."
87
+#include "rrdfunctions-inline.h"
88
+#include "rrdfunctions-inflight.h"
89
+#include "rrdfunctions-exporters.h"
90
+#include "rrdfunctions-streaming.h"
91
+#include "rrdfunctions-progress.h"
92
93
#endif // NETDATA_RRDFUNCTIONS_H
database/rrdhost.c
+15
-15
@@ -337,7 +337,7 @@ int is_legacy = 1;
337
netdata_mutex_init(&host->receiver_lock);
338
339
if (likely(!archived)) {
340
- rrdfunctions_host_init(host);
340
+ rrd_functions_host_init(host);
341
host->last_connected = now_realtime_sec();
342
host->rrdlabels = rrdlabels_create();
343
rrdhost_initialize_rrdpush_sender(
@@ -573,9 +573,6 @@ int is_legacy = 1;
573
, string2str(host->health.health_default_recipient)
574
);
575
576
- host->configurable_plugins = dyncfg_dictionary_create();
577
- dictionary_register_delete_callback(host->configurable_plugins, plugin_del_cb, NULL);
578
-
576
if(!archived) {
577
metaqueue_host_update_info(host);
578
rrdhost_load_rrdcontext_data(host);
@@ -694,7 +691,7 @@ static void rrdhost_update(RRDHOST *host
691
if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
692
rrdhost_flag_clear(host, RRDHOST_FLAG_ARCHIVED);
693
697
- rrdfunctions_host_init(host);
694
+ rrd_functions_host_init(host);
695
696
if(!host->rrdlabels)
697
host->rrdlabels = rrdlabels_create();
@@ -1115,18 +1112,17 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unitt
1112
if (unlikely(!localhost))
1113
return 1;
1114
1115
+ dyncfg_host_init(localhost);
1116
+
1117
// we register this only on localhost
1118
// for the other nodes, the origin server should register it
1120
- rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
1121
- rrd_function_add(localhost, NULL, "streaming", 10, RRDFUNCTIONS_PRIORITY_DEFAULT + 1,
1122
- RRDFUNCTIONS_STREAMING_HELP, "top",
1123
- HTTP_ACCESS_MEMBERS, true,
1124
- rrdhost_function_streaming, NULL);
1119
+ rrd_function_add_inline(localhost, NULL, "streaming", 10,
1120
+ RRDFUNCTIONS_PRIORITY_DEFAULT + 1, RRDFUNCTIONS_STREAMING_HELP, "top",
1121
+ HTTP_ACCESS_MEMBER, rrdhost_function_streaming);
1122
1126
- rrd_function_add(localhost, NULL, "netdata-api-calls", 10, RRDFUNCTIONS_PRIORITY_DEFAULT + 2,
1127
- RRDFUNCTIONS_PROGRESS_HELP, "top",
1128
- HTTP_ACCESS_MEMBERS, true,
1129
- rrdhost_function_progress, NULL);
1123
+ rrd_function_add_inline(localhost, NULL, "netdata-api-calls", 10,
1124
+ RRDFUNCTIONS_PRIORITY_DEFAULT + 2, RRDFUNCTIONS_PROGRESS_HELP, "top",
1125
+ HTTP_ACCESS_MEMBER, rrdhost_function_progress);
1126
1127
if (likely(system_info)) {
1128
migrate_localhost(&localhost->host_uuid);
@@ -1328,7 +1324,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1324
freez(host->node_id);
1325
1326
rrdfamily_index_destroy(host);
1331
- rrdfunctions_host_destroy(host);
1327
+ rrd_functions_host_destroy(host);
1328
rrdvariables_destroy(host->rrdvars);
1329
if (host == localhost)
1330
rrdvariables_destroy(health_rrdvars);
@@ -1849,6 +1845,10 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1845
1846
RRDHOST_FLAGS flags = __atomic_load_n(&host->flags, __ATOMIC_RELAXED);
1847
1848
+ // --- dyncfg ---
1849
+
1850
+ s->dyncfg.status = dyncfg_available_for_rrdhost(host) ? RRDHOST_DYNCFG_STATUS_AVAILABLE : RRDHOST_DYNCFG_STATUS_UNAVAILABLE;
1851
+
1852
// --- db ---
1853
1854
bool online = rrdhost_is_online(host);
database/rrdlabels.c
+4
-4
@@ -448,7 +448,7 @@ __attribute__((constructor)) void initialize_labels_keys_char_map(void) {
448
label_names_char_map[' '] = '_';
449
label_names_char_map['\\'] = '/';
450
451
- // create the spaces map
451
+ // create the space map
452
for(i = 0; i < 256 ;i++)
453
label_spaces_char_map[i] = (isspace(i) || iscntrl(i) || !isprint(i))?1:0;
454
@@ -460,8 +460,8 @@ __attribute__((constructor)) void initialize_label_stats(void) {
460
dictionary_stats_category_rrdlabels.memory.values = 0;
461
}
462
463
-size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length) {
464
- if(unlikely(!dst_size)) return 0;
463
+size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, const unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length) {
464
+ if(unlikely(!src || !dst_size)) return 0;
465
466
if(unlikely(!src || !*src)) {
467
strncpyz((char *)dst, empty, dst_size);
@@ -476,7 +476,7 @@ size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_si
476
// make room for the final string termination
477
unsigned char *end = &d[dst_size - 1];
478
479
- // copy while converting, but keep only one white space
479
+ // copy while converting, but keep only one space
480
// we start wil last_is_space = 1 to skip leading spaces
481
int last_is_space = 1;
482
database/rrdlabels.h
+1
-1
@@ -20,7 +20,7 @@ typedef enum __attribute__ ((__packed__)) rrdlabel_source {
20
21
#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_DONT_DELETE)
22
23
-size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length);
23
+size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, const unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length);
24
25
RRDLABELS *rrdlabels_create(void);
26
void rrdlabels_destroy(RRDLABELS *labels_dict);
database/rrdsetvar.c
+1
-1
@@ -237,7 +237,7 @@ void rrdsetvar_rename_all(RRDSET *st) {
237
void rrdsetvar_release_and_delete_all(RRDSET *st) {
238
RRDSETVAR *rs;
239
dfe_start_write(st->rrdsetvar_root_index, rs) {
240
- dictionary_del_advanced(st->rrdsetvar_root_index, string2str(rs->name), (ssize_t)string_strlen(rs->name) + 1);
240
+ dictionary_del_advanced(st->rrdsetvar_root_index, string2str(rs->name), (ssize_t)string_strlen(rs->name));
241
}
242
dfe_done(rs);
243
}
database/rrdvar.c
+4
-4
@@ -107,7 +107,7 @@ void rrdvariables_destroy(DICTIONARY *dict) {
107
}
108
109
static inline const RRDVAR_ACQUIRED *rrdvar_get_and_acquire(DICTIONARY *dict, STRING *name) {
110
- return (const RRDVAR_ACQUIRED *)dictionary_get_and_acquire_item_advanced(dict, string2str(name), (ssize_t)string_strlen(name) + 1);
110
+ return (const RRDVAR_ACQUIRED *)dictionary_get_and_acquire_item_advanced(dict, string2str(name), (ssize_t)string_strlen(name));
111
}
112
113
inline void rrdvar_release_and_del(DICTIONARY *dict, const RRDVAR_ACQUIRED *rva) {
@@ -115,7 +115,7 @@ inline void rrdvar_release_and_del(DICTIONARY *dict, const RRDVAR_ACQUIRED *rva)
115
116
RRDVAR *rv = dictionary_acquired_item_value((const DICTIONARY_ITEM *)rva);
117
118
- dictionary_del_advanced(dict, string2str(rv->name), (ssize_t)string_strlen(rv->name) + 1);
118
+ dictionary_del_advanced(dict, string2str(rv->name), (ssize_t)string_strlen(rv->name));
119
120
dictionary_acquired_item_release(dict, (const DICTIONARY_ITEM *)rva);
121
}
@@ -130,7 +130,7 @@ inline const RRDVAR_ACQUIRED *rrdvar_add_and_acquire(const char *scope __maybe_u
130
.options = options,
131
.react_action = RRDVAR_REACT_NONE,
132
};
133
- return (const RRDVAR_ACQUIRED *)dictionary_set_and_acquire_item_advanced(dict, string2str(name), (ssize_t)string_strlen(name) + 1, NULL, sizeof(RRDVAR), &tmp);
133
+ return (const RRDVAR_ACQUIRED *)dictionary_set_and_acquire_item_advanced(dict, string2str(name), (ssize_t)string_strlen(name), NULL, sizeof(RRDVAR), &tmp);
134
}
135
136
inline void rrdvar_add(const char *scope __maybe_unused, DICTIONARY *dict, STRING *name, RRDVAR_TYPE type, RRDVAR_FLAGS options, void *value) {
@@ -143,7 +143,7 @@ inline void rrdvar_add(const char *scope __maybe_unused, DICTIONARY *dict, STRIN
143
.options = options,
144
.react_action = RRDVAR_REACT_NONE,
145
};
146
- dictionary_set_advanced(dict, string2str(name), (ssize_t)string_strlen(name) + 1, NULL, sizeof(RRDVAR), &tmp);
146
+ dictionary_set_advanced(dict, string2str(name), (ssize_t)string_strlen(name), NULL, sizeof(RRDVAR), &tmp);
147
}
148
149
void rrdvar_delete_all(DICTIONARY *dict) {
libnetdata/buffer/buffer.h
+25
-31
@@ -35,37 +35,6 @@ typedef enum __attribute__ ((__packed__)) {
35
WB_CONTENT_NO_CACHEABLE = (1 << 1),
36
} BUFFER_OPTIONS;
37
38
-typedef enum __attribute__ ((__packed__)) {
39
- CT_NONE = 0,
40
- CT_APPLICATION_JSON,
41
- CT_TEXT_PLAIN,
42
- CT_TEXT_HTML,
43
- CT_APPLICATION_X_JAVASCRIPT,
44
- CT_TEXT_CSS,
45
- CT_TEXT_XML,
46
- CT_APPLICATION_XML,
47
- CT_TEXT_XSL,
48
- CT_APPLICATION_OCTET_STREAM,
49
- CT_APPLICATION_X_FONT_TRUETYPE,
50
- CT_APPLICATION_X_FONT_OPENTYPE,
51
- CT_APPLICATION_FONT_WOFF,
52
- CT_APPLICATION_FONT_WOFF2,
53
- CT_APPLICATION_VND_MS_FONTOBJ,
54
- CT_IMAGE_SVG_XML,
55
- CT_IMAGE_PNG,
56
- CT_IMAGE_JPG,
57
- CT_IMAGE_GIF,
58
- CT_IMAGE_XICON,
59
- CT_IMAGE_ICNS,
60
- CT_IMAGE_BMP,
61
- CT_PROMETHEUS,
62
- CT_AUDIO_MPEG,
63
- CT_AUDIO_OGG,
64
- CT_VIDEO_MP4,
65
- CT_APPLICATION_PDF,
66
- CT_APPLICATION_ZIP,
67
-} HTTP_CONTENT_TYPE;
68
-
38
typedef enum __attribute__ ((__packed__)) {
39
BUFFER_JSON_OPTIONS_DEFAULT = 0,
40
BUFFER_JSON_OPTIONS_MINIFY = (1 << 0),
@@ -164,6 +133,9 @@ void buffer_json_finalize(BUFFER *wb);
133
134
static const char *buffer_tostring(BUFFER *wb)
135
{
136
+ if(unlikely(!wb))
137
+ return NULL;
138
+
139
buffer_need_bytes(wb, 1);
140
wb->buffer[wb->len] = '\0';
141
@@ -1249,4 +1221,26 @@ buffer_rrdf_table_add_field(BUFFER *wb, size_t field_id, const char *key, const
1221
buffer_json_object_close(wb);
1222
}
1223
1224
+static inline void buffer_copy(BUFFER *dst, BUFFER *src) {
1225
+ if(!src || !dst)
1226
+ return;
1227
+
1228
+ buffer_contents_replace(dst, buffer_tostring(src), buffer_strlen(src));
1229
+
1230
+ dst->content_type = src->content_type;
1231
+ dst->options = src->options;
1232
+ dst->date = src->date;
1233
+ dst->expires = src->expires;
1234
+ dst->json = src->json;
1235
+}
1236
+
1237
+static inline BUFFER *buffer_dup(BUFFER *src) {
1238
+ if(!src)
1239
+ return NULL;
1240
+
1241
+ BUFFER *dst = buffer_create(buffer_strlen(src) + 1, src->statistics);
1242
+ buffer_copy(dst, src);
1243
+ return dst;
1244
+}
1245
+
1246
#endif /* NETDATA_WEB_BUFFER_H */
libnetdata/config/dyncfg.c
new
+297
@@ -0,0 +1,297 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "../libnetdata.h"
4
+
5
+// ----------------------------------------------------------------------------
6
+
7
+static struct {
8
+ DYNCFG_TYPE type;
9
+ const char *name;
10
+} dyncfg_types[] = {
11
+ { .type = DYNCFG_TYPE_SINGLE, .name = "single" },
12
+ { .type = DYNCFG_TYPE_TEMPLATE, .name = "template" },
13
+ { .type = DYNCFG_TYPE_JOB, .name = "job" },
14
+};
15
+
16
+DYNCFG_TYPE dyncfg_type2id(const char *type) {
17
+ if(!type || !*type)
18
+ return DYNCFG_TYPE_SINGLE;
19
+
20
+ size_t entries = sizeof(dyncfg_types) / sizeof(dyncfg_types[0]);
21
+ for(size_t i = 0; i < entries ;i++) {
22
+ if(strcmp(dyncfg_types[i].name, type) == 0)
23
+ return dyncfg_types[i].type;
24
+ }
25
+
26
+ return DYNCFG_TYPE_SINGLE;
27
+}
28
+
29
+const char *dyncfg_id2type(DYNCFG_TYPE type) {
30
+ size_t entries = sizeof(dyncfg_types) / sizeof(dyncfg_types[0]);
31
+ for(size_t i = 0; i < entries ;i++) {
32
+ if(type == dyncfg_types[i].type)
33
+ return dyncfg_types[i].name;
34
+ }
35
+
36
+ return "single";
37
+}
38
+
39
+// ----------------------------------------------------------------------------
40
+
41
+static struct {
42
+ DYNCFG_SOURCE_TYPE source_type;
43
+ const char *name;
44
+} dyncfg_source_types[] = {
45
+ { .source_type = DYNCFG_SOURCE_TYPE_INTERNAL, .name = "internal" },
46
+ { .source_type = DYNCFG_SOURCE_TYPE_STOCK, .name = "stock" },
47
+ { .source_type = DYNCFG_SOURCE_TYPE_USER, .name = "user" },
48
+ { .source_type = DYNCFG_SOURCE_TYPE_DYNCFG, .name = "dyncfg" },
49
+ { .source_type = DYNCFG_SOURCE_TYPE_DISCOVERED, .name = "discovered" },
50
+};
51
+
52
+DYNCFG_SOURCE_TYPE dyncfg_source_type2id(const char *source_type) {
53
+ if(!source_type || !*source_type)
54
+ return DYNCFG_SOURCE_TYPE_INTERNAL;
55
+
56
+ size_t entries = sizeof(dyncfg_source_types) / sizeof(dyncfg_source_types[0]);
57
+ for(size_t i = 0; i < entries ;i++) {
58
+ if(strcmp(dyncfg_source_types[i].name, source_type) == 0)
59
+ return dyncfg_source_types[i].source_type;
60
+ }
61
+
62
+ return DYNCFG_SOURCE_TYPE_INTERNAL;
63
+}
64
+
65
+const char *dyncfg_id2source_type(DYNCFG_SOURCE_TYPE source_type) {
66
+ size_t entries = sizeof(dyncfg_source_types) / sizeof(dyncfg_source_types[0]);
67
+ for(size_t i = 0; i < entries ;i++) {
68
+ if(source_type == dyncfg_source_types[i].source_type)
69
+ return dyncfg_source_types[i].name;
70
+ }
71
+
72
+ return "internal";
73
+}
74
+
75
+// ----------------------------------------------------------------------------
76
+
77
+static struct {
78
+ DYNCFG_STATUS status;
79
+ const char *name;
80
+} dyncfg_statuses[] = {
81
+ { .status = DYNCFG_STATUS_NONE, .name = "none" },
82
+ { .status = DYNCFG_STATUS_ACCEPTED, .name = "accepted" },
83
+ { .status = DYNCFG_STATUS_RUNNING, .name = "running" },
84
+ { .status = DYNCFG_STATUS_FAILED, .name = "failed" },
85
+ { .status = DYNCFG_STATUS_DISABLED, .name = "disabled" },
86
+ { .status = DYNCFG_STATUS_ORPHAN, .name = "orphan" },
87
+ { .status = DYNCFG_STATUS_INCOMPLETE, .name = "incomplete" },
88
+};
89
+
90
+DYNCFG_STATUS dyncfg_status2id(const char *status) {
91
+ if(!status || !*status)
92
+ return DYNCFG_STATUS_NONE;
93
+
94
+ size_t entries = sizeof(dyncfg_statuses) / sizeof(dyncfg_statuses[0]);
95
+ for(size_t i = 0; i < entries ;i++) {
96
+ if(strcmp(dyncfg_statuses[i].name, status) == 0)
97
+ return dyncfg_statuses[i].status;
98
+ }
99
+
100
+ return DYNCFG_STATUS_NONE;
101
+}
102
+
103
+const char *dyncfg_id2status(DYNCFG_STATUS status) {
104
+ size_t entries = sizeof(dyncfg_statuses) / sizeof(dyncfg_statuses[0]);
105
+ for(size_t i = 0; i < entries ;i++) {
106
+ if(status == dyncfg_statuses[i].status)
107
+ return dyncfg_statuses[i].name;
108
+ }
109
+
110
+ return "none";
111
+}
112
+
113
+// ----------------------------------------------------------------------------
114
+
115
+static struct {
116
+ DYNCFG_CMDS cmd;
117
+ const char *name;
118
+} cmd_map[] = {
119
+ { .cmd = DYNCFG_CMD_GET, .name = "get" },
120
+ { .cmd = DYNCFG_CMD_SCHEMA, .name = "schema" },
121
+ { .cmd = DYNCFG_CMD_UPDATE, .name = "update" },
122
+ { .cmd = DYNCFG_CMD_ADD, .name = "add" },
123
+ { .cmd = DYNCFG_CMD_TEST, .name = "test" },
124
+ { .cmd = DYNCFG_CMD_REMOVE, .name = "remove" },
125
+ { .cmd = DYNCFG_CMD_ENABLE, .name = "enable" },
126
+ { .cmd = DYNCFG_CMD_DISABLE, .name = "disable" },
127
+ { .cmd = DYNCFG_CMD_RESTART, .name = "restart" }
128
+};
129
+
130
+const char *dyncfg_id2cmd_one(DYNCFG_CMDS cmd) {
131
+ for (size_t i = 0; i < sizeof(cmd_map) / sizeof(cmd_map[0]); i++) {
132
+ if(cmd == cmd_map[i].cmd)
133
+ return cmd_map[i].name;
134
+ }
135
+
136
+ return NULL;
137
+}
138
+
139
+DYNCFG_CMDS dyncfg_cmds2id(const char *cmds) {
140
+ if(!cmds || !*cmds)
141
+ return DYNCFG_CMD_NONE;
142
+
143
+ DYNCFG_CMDS result = DYNCFG_CMD_NONE;
144
+ const char *p = cmds;
145
+ size_t len, i;
146
+
147
+ while (*p) {
148
+ // Skip any leading spaces
149
+ while (*p == ' ') p++;
150
+
151
+ // Find the end of the current word
152
+ const char *end = p;
153
+ while (*end && *end != ' ') end++;
154
+ len = end - p;
155
+
156
+ // Compare with known commands
157
+ for (i = 0; i < sizeof(cmd_map) / sizeof(cmd_map[0]); i++) {
158
+ if (strncmp(p, cmd_map[i].name, len) == 0 && cmd_map[i].name[len] == '\0') {
159
+ result |= cmd_map[i].cmd;
160
+ break;
161
+ }
162
+ }
163
+
164
+ // Move to the next word
165
+ p = end;
166
+ }
167
+
168
+ return result;
169
+}
170
+
171
+void dyncfg_cmds2fp(DYNCFG_CMDS cmds, FILE *fp) {
172
+ for (size_t i = 0; i < sizeof(cmd_map) / sizeof(cmd_map[0]); i++) {
173
+ if(cmds & cmd_map[i].cmd)
174
+ fprintf(fp, "%s ", cmd_map[i].name);
175
+ }
176
+}
177
+
178
+void dyncfg_cmds2json_array(DYNCFG_CMDS cmds, const char *key, BUFFER *wb) {
179
+ buffer_json_member_add_array(wb, key);
180
+ for (size_t i = 0; i < sizeof(cmd_map) / sizeof(cmd_map[0]); i++) {
181
+ if(cmds & cmd_map[i].cmd)
182
+ buffer_json_add_array_item_string(wb, cmd_map[i].name);
183
+ }
184
+ buffer_json_array_close(wb);
185
+}
186
+
187
+void dyncfg_cmds2buffer(DYNCFG_CMDS cmds, BUFFER *wb) {
188
+ size_t added = 0;
189
+ for (size_t i = 0; i < sizeof(cmd_map) / sizeof(cmd_map[0]); i++) {
190
+ if(cmds & cmd_map[i].cmd) {
191
+ if(added)
192
+ buffer_fast_strcat(wb, " ", 1);
193
+
194
+ buffer_strcat(wb, cmd_map[i].name);
195
+ added++;
196
+ }
197
+ }
198
+}
199
+
200
+// ----------------------------------------------------------------------------
201
+
202
+bool dyncfg_is_valid_id(const char *id) {
203
+ const char *s = id;
204
+
205
+ while(*s) {
206
+ if(isspace(*s) || *s == '\'') return false;
207
+ s++;
208
+ }
209
+
210
+ return true;
211
+}
212
+
213
+char *dyncfg_escape_id_for_filename(const char *id) {
214
+ if (id == NULL) return NULL;
215
+
216
+ // Allocate memory for the worst case, where every character is escaped.
217
+ char *escaped = mallocz(strlen(id) * 3 + 1); // Each char can become '%XX', plus '\0'
218
+ if (!escaped) return NULL;
219
+
220
+ const char *src = id;
221
+ char *dest = escaped;
222
+
223
+ while (*src) {
224
+ if (*src == '/' || isspace(*src) || !isprint(*src)) {
225
+ sprintf(dest, "%%%02X", (unsigned char)*src);
226
+ dest += 3;
227
+ } else {
228
+ *dest++ = *src;
229
+ }
230
+ src++;
231
+ }
232
+
233
+ *dest = '\0';
234
+ return escaped;
235
+}
236
+
237
+// ----------------------------------------------------------------------------
238
+
239
+int dyncfg_default_response(BUFFER *wb, int code, const char *msg) {
240
+ buffer_flush(wb);
241
+ wb->content_type = CT_APPLICATION_JSON;
242
+ wb->expires = now_realtime_sec();
243
+
244
+ buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
245
+ buffer_json_member_add_uint64(wb, "status", code);
246
+ buffer_json_member_add_string(wb, "message", msg);
247
+ buffer_json_finalize(wb);
248
+
249
+ return code;
250
+}
251
+
252
+int dyncfg_node_find_and_call(DICTIONARY *dyncfg_nodes, const char *transaction, const char *function,
253
+ usec_t *stop_monotonic_ut, bool *cancelled,
254
+ BUFFER *payload, const char *source, BUFFER *result) {
255
+ if(!function || !*function)
256
+ return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "command received is empty");
257
+
258
+ char buf[strlen(function) + 1];
259
+ memcpy(buf, function, sizeof(buf));
260
+
261
+ char *words[MAX_FUNCTION_PARAMETERS]; // an array of pointers for the words in this line
262
+ size_t num_words = quoted_strings_splitter_pluginsd(buf, words, MAX_FUNCTION_PARAMETERS);
263
+
264
+ const char *id = get_word(words, num_words, 1);
265
+ const char *action = get_word(words, num_words, 2);
266
+
267
+ if(!id || !*id)
268
+ return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "dyncfg node: id is missing from the request");
269
+
270
+ if(!action || !*action)
271
+ return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "dyncfg node: action is missing from the request");
272
+
273
+ DYNCFG_CMDS cmd = dyncfg_cmds2id(action);
274
+ if(cmd == DYNCFG_CMD_NONE)
275
+ return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "dyncfg node: action given in request is unknown");
276
+
277
+ const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(dyncfg_nodes, id);
278
+ if(!item)
279
+ return dyncfg_default_response(result, HTTP_RESP_NOT_FOUND, "dyncfg node: id is not found");
280
+
281
+ struct dyncfg_node *df = dictionary_acquired_item_value(item);
282
+
283
+ buffer_flush(result);
284
+ result->content_type = CT_APPLICATION_JSON;
285
+
286
+ int code = df->cb(transaction, id, cmd, payload, stop_monotonic_ut, cancelled, result, source, df->data);
287
+
288
+ if(!result->expires)
289
+ result->expires = now_realtime_sec();
290
+
291
+ if(!buffer_tostring(result))
292
+ dyncfg_default_response(result, code, "");
293
+
294
+ dictionary_acquired_item_release(dyncfg_nodes, item);
295
+
296
+ return code;
297
+}
libnetdata/config/dyncfg.h
new
+85
@@ -0,0 +1,85 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef LIBNETDATA_DYNCFG_H
4
+#define LIBNETDATA_DYNCFG_H
5
+
6
+#define DYNCFG_VERSION (size_t)1
7
+
8
+#define DYNCFG_RESP_SUCCESS(code) (code >= 200 && code <= 299)
9
+#define DYNCFG_RESP_RUNNING 200 // accepted and running
10
+#define DYNCFG_RESP_ACCEPTED 202 // accepted, but not running yet
11
+#define DYNCFG_RESP_ACCEPTED_RESTART_REQUIRED 299 // accepted, but restart is required to apply it
12
+
13
+typedef enum __attribute__((packed)) {
14
+ DYNCFG_TYPE_SINGLE = 0,
15
+ DYNCFG_TYPE_TEMPLATE,
16
+ DYNCFG_TYPE_JOB,
17
+} DYNCFG_TYPE;
18
+DYNCFG_TYPE dyncfg_type2id(const char *type);
19
+const char *dyncfg_id2type(DYNCFG_TYPE type);
20
+
21
+typedef enum __attribute__((packed)) {
22
+ DYNCFG_SOURCE_TYPE_INTERNAL = 0,
23
+ DYNCFG_SOURCE_TYPE_STOCK,
24
+ DYNCFG_SOURCE_TYPE_USER,
25
+ DYNCFG_SOURCE_TYPE_DYNCFG,
26
+ DYNCFG_SOURCE_TYPE_DISCOVERED,
27
+} DYNCFG_SOURCE_TYPE;
28
+DYNCFG_SOURCE_TYPE dyncfg_source_type2id(const char *source_type);
29
+const char *dyncfg_id2source_type(DYNCFG_SOURCE_TYPE source_type);
30
+
31
+typedef enum __attribute__((packed)) {
32
+ DYNCFG_STATUS_NONE = 0,
33
+ DYNCFG_STATUS_ACCEPTED, // the plugin has accepted the configuration
34
+ DYNCFG_STATUS_RUNNING, // the plugin runs the accepted configuration
35
+ DYNCFG_STATUS_FAILED, // the plugin fails to run the accepted configuration
36
+ DYNCFG_STATUS_DISABLED, // the configuration is disabled by a user
37
+ DYNCFG_STATUS_ORPHAN, // no plugin has claimed this configurations
38
+ DYNCFG_STATUS_INCOMPLETE, // a special kind of failed configuration
39
+} DYNCFG_STATUS;
40
+DYNCFG_STATUS dyncfg_status2id(const char *status);
41
+const char *dyncfg_id2status(DYNCFG_STATUS status);
42
+
43
+typedef enum __attribute__((packed)) {
44
+ DYNCFG_CMD_NONE = 0,
45
+ DYNCFG_CMD_GET = (1 << 0),
46
+ DYNCFG_CMD_SCHEMA = (1 << 1),
47
+ DYNCFG_CMD_UPDATE = (1 << 2),
48
+ DYNCFG_CMD_ADD = (1 << 3),
49
+ DYNCFG_CMD_TEST = (1 << 4),
50
+ DYNCFG_CMD_REMOVE = (1 << 5),
51
+ DYNCFG_CMD_ENABLE = (1 << 6),
52
+ DYNCFG_CMD_DISABLE = (1 << 7),
53
+ DYNCFG_CMD_RESTART = (1 << 8),
54
+} DYNCFG_CMDS;
55
+DYNCFG_CMDS dyncfg_cmds2id(const char *cmds);
56
+void dyncfg_cmds2buffer(DYNCFG_CMDS cmds, struct web_buffer *wb);
57
+void dyncfg_cmds2json_array(DYNCFG_CMDS cmds, const char *key, struct web_buffer *wb);
58
+void dyncfg_cmds2fp(DYNCFG_CMDS cmds, FILE *fp);
59
+const char *dyncfg_id2cmd_one(DYNCFG_CMDS cmd);
60
+
61
+bool dyncfg_is_valid_id(const char *id);
62
+char *dyncfg_escape_id_for_filename(const char *id);
63
+
64
+#include "../clocks/clocks.h"
65
+#include "../buffer/buffer.h"
66
+#include "../dictionary/dictionary.h"
67
+
68
+typedef int (*dyncfg_cb_t)(const char *transaction, const char *id, DYNCFG_CMDS cmd, BUFFER *payload, usec_t *stop_monotonic_ut, bool *cancelled, BUFFER *result, const char *source, void *data);
69
+
70
+struct dyncfg_node {
71
+ DYNCFG_TYPE type;
72
+ DYNCFG_CMDS cmds;
73
+ dyncfg_cb_t cb;
74
+ void *data;
75
+};
76
+
77
+#define dyncfg_nodes_dictionary_create() dictionary_create_advanced(DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct dyncfg_node))
78
+
79
+int dyncfg_default_response(BUFFER *wb, int code, const char *msg);
80
+
81
+int dyncfg_node_find_and_call(DICTIONARY *dyncfg_nodes, const char *transaction, const char *function,
82
+ usec_t *stop_monotonic_ut, bool *cancelled,
83
+ BUFFER *payload, const char *source, BUFFER *result);
84
+
85
+#endif //LIBNETDATA_DYNCFG_H
libnetdata/dictionary/dictionary.c
+30
-26
@@ -996,7 +996,9 @@ static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *it
996
if (having_index_lock) {
997
// delete it from the hashtable
998
if(hashtable_delete_unsafe(dict, item_get_name(item), item->key_len, item) == 0)
999
- netdata_log_error("DICTIONARY: INTERNAL ERROR VIEW: tried to delete item with name '%s', name_len %u that is not in the index", item_get_name(item), (KEY_LEN_TYPE)(item->key_len - 1));
999
+ netdata_log_error("DICTIONARY: INTERNAL ERROR VIEW: tried to delete item with name '%s', "
1000
+ "name_len %u that is not in the index",
1001
+ item_get_name(item), (KEY_LEN_TYPE)(item->key_len));
1002
else
1003
pointer_del(dict, item);
1004
@@ -1237,7 +1239,7 @@ static inline size_t item_set_name(DICTIONARY *dict, DICTIONARY_ITEM *item, cons
1239
}
1240
else {
1241
item->string_name = string_strdupz(name);
1240
- item->key_len = string_strlen(item->string_name) + 1;
1242
+ item->key_len = string_strlen(item->string_name);
1243
item->options |= ITEM_OPTION_ALLOCATED_NAME;
1244
}
1245
@@ -1584,7 +1586,7 @@ static inline void dict_item_release_and_check_if_it_is_deleted_and_can_be_remov
1586
1587
static bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t name_len) {
1588
if(name_len == -1)
1587
- name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too
1589
+ name_len = (ssize_t)strlen(name);
1590
1591
netdata_log_debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name);
1592
@@ -1602,9 +1604,9 @@ static bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t name_len)
1604
}
1605
else {
1606
if(hashtable_delete_unsafe(dict, name, name_len, item) == 0)
1605
- netdata_log_error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s', name_len %zd that is not in the index",
1606
- name,
1607
- name_len - 1);
1607
+ netdata_log_error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s', "
1608
+ "name_len %zd that is not in the index",
1609
+ name, name_len);
1610
else
1611
pointer_del(dict, item);
1612
@@ -1635,7 +1637,7 @@ static DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONARY *dic
1637
}
1638
1639
if(name_len == -1)
1638
- name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too
1640
+ name_len = (ssize_t)strlen(name);
1641
1642
netdata_log_debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name);
1643
@@ -1754,7 +1756,7 @@ static DICTIONARY_ITEM *dict_item_find_and_acquire(DICTIONARY *dict, const char
1756
}
1757
1758
if(name_len == -1)
1757
- name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too
1759
+ name_len = (ssize_t)strlen(name);
1760
1761
netdata_log_debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name);
1762
@@ -1990,11 +1992,12 @@ static bool api_is_name_good_with_trace(DICTIONARY *dict __maybe_unused, const c
1992
}
1993
1994
internal_error(
1993
- name_len > 0 && name_len != (ssize_t)(strlen(name) + 1),
1994
- "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu (incl. '\\0'), but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.",
1995
+ name_len > 0 && name_len != (ssize_t)strlen(name),
1996
+ "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu, "
1997
+ "but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.",
1998
function,
1999
name,
1997
- strlen(name) + 1,
2000
+ strlen(name),
2001
(long int) name_len,
2002
dict?dict->creation_function:"unknown",
2003
dict?dict->creation_line:0,
@@ -2002,10 +2005,11 @@ static bool api_is_name_good_with_trace(DICTIONARY *dict __maybe_unused, const c
2005
2006
internal_error(
2007
name_len <= 0 && name_len != -1,
2005
- "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu (incl. '\\0'), but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.",
2008
+ "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu, "
2009
+ "but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.",
2010
function,
2011
name,
2008
- strlen(name) + 1,
2012
+ strlen(name),
2013
(long int) name_len,
2014
dict?dict->creation_function:"unknown",
2015
dict?dict->creation_line:0,
@@ -2109,7 +2113,7 @@ void dictionary_flush(DICTIONARY *dict) {
2113
DICTIONARY_ITEM *item, *next = NULL;
2114
for(item = dict->items.list; item ;item = next) {
2115
next = item->next;
2112
- dict_item_del(dict, item_get_name(item), (ssize_t) item_get_name_len(item) + 1);
2116
+ dict_item_del(dict, item_get_name(item), (ssize_t)item_get_name_len(item));
2117
}
2118
2119
ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
@@ -2580,7 +2584,7 @@ void *thread_cache_entry_get_or_set(void *key,
2584
if(unlikely(!key || !key_length)) return NULL;
2585
2586
if(key_length == -1)
2583
- key_length = (ssize_t)strlen((char *)key) + 1;
2587
+ key_length = (ssize_t)strlen((char *)key);
2588
2589
JError_t J_Error;
2590
Pvoid_t *Rc = JudyHSIns(&thread_cache_judy_array, key, key_length, &J_Error);
@@ -2627,7 +2631,7 @@ static char **dictionary_unittest_generate_names(size_t entries) {
2631
char **names = mallocz(sizeof(char *) * entries);
2632
for(size_t i = 0; i < entries ;i++) {
2633
char buf[25 + 1] = "";
2630
- snprintfz(buf, sizeof(buf) - 1, "name.%zu.0123456789.%zu!@#$%%^&*(),./[]{}\\|~`", i, entries / 2 + i);
2634
+ snprintfz(buf, sizeof(buf), "name.%zu.0123456789.%zu!@#$%%^&*(),./[]{}\\|~`", i, entries / 2 + i);
2635
names[i] = strdupz(buf);
2636
}
2637
return names;
@@ -2637,7 +2641,7 @@ static char **dictionary_unittest_generate_values(size_t entries) {
2641
char **values = mallocz(sizeof(char *) * entries);
2642
for(size_t i = 0; i < entries ;i++) {
2643
char buf[25 + 1] = "";
2640
- snprintfz(buf, sizeof(buf) - 1, "value-%zu-0987654321.%zu%%^&*(),. \t !@#$/[]{}\\|~`", i, entries / 2 + i);
2644
+ snprintfz(buf, sizeof(buf), "value-%zu-0987654321.%zu%%^&*(),. \t !@#$/[]{}\\|~`", i, entries / 2 + i);
2645
values[i] = strdupz(buf);
2646
}
2647
return values;
@@ -2646,7 +2650,7 @@ static char **dictionary_unittest_generate_values(size_t entries) {
2650
static size_t dictionary_unittest_set_clone(DICTIONARY *dict, char **names, char **values, size_t entries) {
2651
size_t errors = 0;
2652
for(size_t i = 0; i < entries ;i++) {
2649
- size_t vallen = strlen(values[i]) + 1;
2653
+ size_t vallen = strlen(values[i]);
2654
char *val = (char *)dictionary_set(dict, names[i], values[i], vallen);
2655
if(val == values[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
2656
if(!val || memcmp(val, values[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
@@ -2673,7 +2677,7 @@ static size_t dictionary_unittest_set_null(DICTIONARY *dict, char **names, char
2677
static size_t dictionary_unittest_set_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
2678
size_t errors = 0;
2679
for(size_t i = 0; i < entries ;i++) {
2676
- size_t vallen = strlen(values[i]) + 1;
2680
+ size_t vallen = strlen(values[i]);
2681
char *val = (char *)dictionary_set(dict, names[i], values[i], vallen);
2682
if(val != values[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
2683
}
@@ -2683,7 +2687,7 @@ static size_t dictionary_unittest_set_nonclone(DICTIONARY *dict, char **names, c
2687
static size_t dictionary_unittest_get_clone(DICTIONARY *dict, char **names, char **values, size_t entries) {
2688
size_t errors = 0;
2689
for(size_t i = 0; i < entries ;i++) {
2686
- size_t vallen = strlen(values[i]) + 1;
2690
+ size_t vallen = strlen(values[i]);
2691
char *val = (char *)dictionary_get(dict, names[i]);
2692
if(val == values[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
2693
if(!val || memcmp(val, values[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
@@ -2751,7 +2755,7 @@ static size_t dictionary_unittest_reset_clone(DICTIONARY *dict, char **names, ch
2755
// set the name as value too
2756
size_t errors = 0;
2757
for(size_t i = 0; i < entries ;i++) {
2754
- size_t vallen = strlen(names[i]) + 1;
2758
+ size_t vallen = strlen(names[i]);
2759
char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
2760
if(val == names[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
2761
if(!val || memcmp(val, names[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
@@ -2764,7 +2768,7 @@ static size_t dictionary_unittest_reset_nonclone(DICTIONARY *dict, char **names,
2768
// set the name as value too
2769
size_t errors = 0;
2770
for(size_t i = 0; i < entries ;i++) {
2767
- size_t vallen = strlen(names[i]) + 1;
2771
+ size_t vallen = strlen(names[i]);
2772
char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
2773
if(val != names[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
2774
if(!val) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
@@ -2776,7 +2780,7 @@ static size_t dictionary_unittest_reset_dont_overwrite_nonclone(DICTIONARY *dict
2780
// set the name as value too
2781
size_t errors = 0;
2782
for(size_t i = 0; i < entries ;i++) {
2779
- size_t vallen = strlen(names[i]) + 1;
2783
+ size_t vallen = strlen(names[i]);
2784
char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
2785
if(val != values[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
2786
}
@@ -3253,13 +3257,13 @@ static void *unittest_dict_thread(void *arg) {
3257
char buf [256 + 1];
3258
3259
for (int i = 0; i < 1000; i++) {
3256
- snprintfz(buf, sizeof(buf) - 1, "del/flush test %d", i);
3260
+ snprintfz(buf, sizeof(buf), "del/flush test %d", i);
3261
dictionary_set(tu->dict, buf, NULL, 0);
3262
tu->stats.ops.inserts++;
3263
}
3264
3265
for (int i = 0; i < 1000; i++) {
3262
- snprintfz(buf, sizeof(buf) - 1, "del/flush test %d", i);
3266
+ snprintfz(buf, sizeof(buf), "del/flush test %d", i);
3267
dictionary_del(tu->dict, buf);
3268
tu->stats.ops.deletes++;
3269
}
@@ -3392,7 +3396,7 @@ static void *unittest_dict_master_thread(void *arg) {
3396
while(!__atomic_load_n(&tv->join, __ATOMIC_RELAXED)) {
3397
3398
if(!item)
3395
- item = dictionary_set_and_acquire_item(tv->master, "ITEM1", "123", strlen("123") + 1);
3399
+ item = dictionary_set_and_acquire_item(tv->master, "ITEM1", "123", strlen("123"));
3400
3401
if(__atomic_load_n(&tv->item_master, __ATOMIC_RELAXED) != NULL) {
3402
dictionary_acquired_item_release(tv->master, item);
libnetdata/dyn_conf/README.md
deleted
-188
@@ -1,188 +0,0 @@
1
-# Netdata Dynamic Configuration
2
-
3
-Purpose of Netdata Dynamic Configuration is to allow configuration of select Netdata plugins and options through the
4
-Netdata API and by extension by UI.
5
-
6
-## HTTP API documentation
7
-
8
-### Summary API
9
-
10
-For summary of all jobs and their statuses (for all children that stream to parent) use the following URL:
11
-
12
-| Method | Endpoint | Description |
13
-|:-------:|-------------------------------|------------------------------------------------------------|
14
-| **GET** | `api/v2/job_statuses` | list of Jobs |
15
-| **GET** | `api/v2/job_statuses?grouped` | list of Jobs (hierarchical, grouped by host/plugin/module) |
16
-
17
-### Dyncfg API
18
-
19
-### Top level
20
-
21
-| Method | Endpoint | Description |
22
-|:-------:|------------------|-----------------------------------------|
23
-| **GET** | `/api/v2/config` | registered Plugins (sent DYNCFG_ENABLE) |
24
-
25
-### Plugin level
26
-
27
-| Method | Endpoint | Description |
28
-|:-------:|-----------------------------------|------------------------------|
29
-| **GET** | `/api/v2/config/[plugin]` | Plugin config |
30
-| **PUT** | `/api/v2/config/[plugin]` | update Plugin config |
31
-| **GET** | `/api/v2/config/[plugin]/modules` | Modules registered by Plugin |
32
-| **GET** | `/api/v2/config/[plugin]/schema` | Plugin config schema |
33
-
34
-### Module level
35
-
36
-| Method | Endpoint | Description |
37
-|:-------:|-----------------------------------------------|---------------------------|
38
-| **GET** | `/api/v2/config/<plugin>/[module]` | Module config |
39
-| **PUT** | `/api/v2/config/[plugin]/[module]` | update Module config |
40
-| **GET** | `/api/v2/config/[plugin]/[module]/jobs` | Jobs registered by Module |
41
-| **GET** | `/api/v2/config/[plugin]/[module]/job_schema` | Job config schema |
42
-| **GET** | `/api/v2/config/[plugin]/[module]/schema` | Module config schema |
43
-
44
-### Job level - only for modules where `module_type == job_array`
45
-
46
-| Method | Endpoint | Description |
47
-|:----------:|------------------------------------------|--------------------------------|
48
-| **GET** | `/api/v2/config/[plugin]/[module]/[job]` | Job config |
49
-| **PUT** | `/api/v2/config/[plugin]/[module]/[job]` | update Job config |
50
-| **POST** | `/api/v2/config/[plugin]/[module]/[job]` | create Job |
51
-| **DELETE** | `/api/v2/config/[plugin]/[module]/[job]` | delete Job (created by Dyncfg) |
52
-
53
-## Internal Plugins API
54
-
55
-TBD
56
-
57
-## External Plugins API
58
-
59
-### Commands plugins can use
60
-
61
-#### DYNCFG_ENABLE
62
-
63
-Plugin signifies to agent its ability to use new dynamic config and the name it wishes to use by sending
64
-
65
-```
66
-DYNCFG_ENABLE [{PLUGIN_NAME}]
67
-```
68
-
69
-This can be sent only once per lifetime of the plugin (at startup or later) sending it multiple times is considered a
70
-protocol violation and plugin might get terminated.
71
-
72
-After this command is sent the plugin has to be ready to accept all the new commands/keywords related to dynamic
73
-configuration (this command lets agent know this plugin is dyncfg capable and wishes to use dyncfg functionality).
74
-
75
-#### DYNCFG_RESET
76
-
77
-Sending this, will reset the internal state of the agent, considering this a `DYNCFG_ENABLE`.
78
-
79
-```
80
-DYNCFG_RESET
81
-```
82
-
83
-
84
-#### DYNCFG_REGISTER_MODULE
85
-
86
-```
87
-DYNCFG_REGISTER_MODULE {MODULE_NAME} {MODULE_TYPE}
88
-```
89
-
90
-Module has to choose one of following types at registration:
91
-
92
-- `single` - module itself has configuration but does not accept any jobs *(this is useful mainly for internal netdata
93
- configurable things like webserver etc.)*
94
-
95
-- `job_array` - module itself **can** *(not must)* have configuration and it has an array of jobs which can be added,
96
- modified and deleted. **this is what plugin developer needs in most cases**
97
-
98
-After a module has been registered agent can call `set_module_config`, `get_module_config` and `get_module_config_schema`.
99
-
100
-When `MODULE_TYPE` is `job_array` the agent may also send `set_job_config`, `get_job_config` and `get_job_config_schema`.
101
-
102
-#### DYNCFG_REGISTER_JOB
103
-
104
-The plugin can use `DYNCFG_REGISTER_JOB` to register its own configuration jobs. It should not register jobs configured
105
-via DYNCFG (doing so, the agent will shutdown the plugin).
106
-
107
-
108
-```
109
-DYNCFG_REGISTER_JOB {MODULE_NAME} {JOB_NAME} {JOB_TYPE} {FLAGS}
110
-```
111
-
112
-Where:
113
-
114
-- `MODULE_NAME` is the name of the module.
115
-- `JOB_NAME` is the name of the job.
116
-- `JOB_TYPE` is either `stock` or `autodiscovered`.
117
-- `FLAGS`, just send zero.
118
-
119
-#### REPORT_JOB_STATUS
120
-
121
-```
122
-REPORT_JOB_STATUS {MODULE_NAME} {JOB_NAME} {STATUS} {STATE} ["REASON"]
123
-```
124
-
125
-Note the REASON parameter is optional and can be entirelly ommited (for example when state is OK there is no need to send any reason).
126
-
127
-Where:
128
-
129
-- `MODULE_NAME` is the name of the module.
130
-- `JOB_NAME` is the name of the job.
131
-- `STATUS` is one of `stopped`, `running`, or `error`.
132
-- `STATE`, just send zero.
133
-- `REASON` is a message describing the status. In case you don't want to send any reason string it is preferable to omit this parameter altogether (as opposed to sending empty string `""`).
134
-
135
-
136
-### Commands plugins must serve
137
-
138
-Once a plugin calls `DYNCFG_ENABLE`, the must be able to handle these calls.
139
-
140
-function|parameters|prerequisites|request payload|response payload|
141
-:---:|:---:|:---:|:---:|:---:|
142
-`set_plugin_config`|none|`DYNCFG_ENABLE`|plugin configuration|none|
143
-`get_plugin_config`|none|`DYNCFG_ENABLE`|none|plugin configuration|
144
-`get_plugin_config_schema`|none|`DYNCFG_ENABLE`|none|plugin configuration schema|
145
-`set_module_config`|`module_name`|`DYNCFG_REGISTER_MODULE`|module configuration|none|
146
-`get_module_config`|`module_name`|`DYNCFG_REGISTER_MODULE`|none|module configuration|
147
-`get_module_config_schema`|`module_name`|`DYNCFG_REGISTER_MODULE`|none|module configuration schema|
148
-`set_job_config`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|job configuration|none|
149
-`get_job_config`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|none|job configuration|
150
-`get_job_config_schema`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|none|job configuration schema|
151
-
152
-All of them work like this:
153
-
154
-If the request payload is `none`, then the request looks like this:
155
-
156
-```bash
157
-FUNCTION {TRANSACTION_UUID} {TIMEOUT_SECONDS} "{function} {parameters}"
158
-```
159
-
160
-When there is payload, the request looks like this:
161
-
162
-```bash
163
-FUNCTION_PAYLOAD {TRANSACTION_UUID} {TIMEOUT_SECONDS} "{function} {parameters}"
164
-<payload>
165
-FUNCTION_PAYLOAD_END
166
-```
167
-
168
-In all cases, the response is like this:
169
-
170
-```bash
171
-FUNCTION_RESULT_BEGIN {TRANSACTION_UUID} {HTTP_RESPONSE_CODE} "{CONTENT_TYPE}" {EXPIRATION_TIMESTAMP}
172
-<payload>
173
-FUNCTION_RESULT_END
174
-```
175
-Where:
176
-- `TRANSACTION_UUID` is the same UUID received with the request.
177
-- `HTTP_RESPONSE_CODE` is either `0` (rejected) or `1` (accepted).
178
-- `CONTENT_TYPE` should reflect the `payload` returned.
179
-- `EXPIRATION_TIMESTAMP` can be zero.
180
-
181
-
182
-## DYNCFG with streaming
183
-
184
-When above commands are transferred trough streaming additionally `plugin_name` is prefixed as first parameter. This is
185
-done to allow routing to appropriate plugin @child.
186
-
187
-As a plugin developer you don't need to concern yourself with this detail as that parameter is stripped when sent to the
188
-plugin *(and added when sent trough streaming)* automagically.
libnetdata/dyn_conf/dyn_conf.c
deleted
-1140
@@ -1,1140 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "dyn_conf.h"
4
-
5
-#define DYN_CONF_PATH_MAX (4096)
6
-#define DYN_CONF_DIR VARLIB_DIR "/dynconf"
7
-
8
-#define DYN_CONF_JOB_SCHEMA "job_schema"
9
-#define DYN_CONF_SCHEMA "schema"
10
-#define DYN_CONF_MODULE_LIST "modules"
11
-#define DYN_CONF_JOB_LIST "jobs"
12
-#define DYN_CONF_CFG_EXT ".cfg"
13
-
14
-void job_flags_wallkthrough(dyncfg_job_flg_t flags, void (*cb)(const char *str, void *data), void *data)
15
-{
16
- if (flags & JOB_FLG_PS_LOADED)
17
- cb("JOB_FLG_PS_LOADED", data);
18
- if (flags & JOB_FLG_PLUGIN_PUSHED)
19
- cb("JOB_FLG_PLUGIN_PUSHED", data);
20
- if (flags & JOB_FLG_STREAMING_PUSHED)
21
- cb("JOB_FLG_STREAMING_PUSHED", data);
22
- if (flags & JOB_FLG_USER_CREATED)
23
- cb("JOB_FLG_USER_CREATED", data);
24
-}
25
-
26
-struct deferred_cfg_send {
27
- DICTIONARY *plugins_dict;
28
- char *plugin_name;
29
- char *module_name;
30
- char *job_name;
31
- struct deferred_cfg_send *next;
32
-};
33
-
34
-bool dyncfg_shutdown = false;
35
-struct deferred_cfg_send *deferred_configs = NULL;
36
-pthread_mutex_t deferred_configs_lock = PTHREAD_MUTEX_INITIALIZER;
37
-pthread_cond_t deferred_configs_cond = PTHREAD_COND_INITIALIZER;
38
-
39
-static void deferred_config_free(struct deferred_cfg_send *dcs)
40
-{
41
- freez(dcs->plugin_name);
42
- freez(dcs->module_name);
43
- freez(dcs->job_name);
44
- freez(dcs);
45
-}
46
-
47
-static void deferred_config_push_back(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name)
48
-{
49
- struct deferred_cfg_send *deferred = callocz(1, sizeof(struct deferred_cfg_send));
50
- deferred->plugin_name = strdupz(plugin_name);
51
- if (module_name != NULL) {
52
- deferred->module_name = strdupz(module_name);
53
- if (job_name != NULL)
54
- deferred->job_name = strdupz(job_name);
55
- }
56
- deferred->plugins_dict = plugins_dict;
57
- pthread_mutex_lock(&deferred_configs_lock);
58
- if (dyncfg_shutdown) {
59
- pthread_mutex_unlock(&deferred_configs_lock);
60
- deferred_config_free(deferred);
61
- return;
62
- }
63
- struct deferred_cfg_send *last = deferred_configs;
64
- if (last == NULL)
65
- deferred_configs = deferred;
66
- else {
67
- while (last->next != NULL)
68
- last = last->next;
69
- last->next = deferred;
70
- }
71
- pthread_cond_signal(&deferred_configs_cond);
72
- pthread_mutex_unlock(&deferred_configs_lock);
73
-}
74
-
75
-static void deferred_configs_unlock()
76
-{
77
- dyncfg_shutdown = true;
78
- // if we get cancelled in pthread_cond_wait
79
- // we will arrive at cancelled cleanup handler
80
- // with mutex locked we need to unlock it
81
- pthread_mutex_unlock(&deferred_configs_lock);
82
-}
83
-
84
-static struct deferred_cfg_send *deferred_config_pop(void *ptr)
85
-{
86
- pthread_mutex_lock(&deferred_configs_lock);
87
- while (deferred_configs == NULL) {
88
- netdata_thread_cleanup_push(deferred_configs_unlock, ptr);
89
- pthread_cond_wait(&deferred_configs_cond, &deferred_configs_lock);
90
- netdata_thread_cleanup_pop(0);
91
- }
92
- struct deferred_cfg_send *deferred = deferred_configs;
93
- deferred_configs = deferred_configs->next;
94
- pthread_mutex_unlock(&deferred_configs_lock);
95
- return deferred;
96
-}
97
-
98
-static int _get_list_of_plugins_json_cb(const DICTIONARY_ITEM *item, void *entry, void *data)
99
-{
100
- UNUSED(item);
101
- json_object *obj = (json_object *)data;
102
- struct configurable_plugin *plugin = (struct configurable_plugin *)entry;
103
-
104
- json_object *plugin_name = json_object_new_string(plugin->name);
105
- json_object_array_add(obj, plugin_name);
106
-
107
- return 0;
108
-}
109
-
110
-json_object *get_list_of_plugins_json(DICTIONARY *plugins_dict)
111
-{
112
- json_object *obj = json_object_new_array();
113
-
114
- dictionary_walkthrough_read(plugins_dict, _get_list_of_plugins_json_cb, obj);
115
-
116
- return obj;
117
-}
118
-
119
-static int _get_list_of_modules_json_cb(const DICTIONARY_ITEM *item, void *entry, void *data)
120
-{
121
- UNUSED(item);
122
- json_object *obj = (json_object *)data;
123
- struct module *module = (struct module *)entry;
124
-
125
- json_object *json_module = json_object_new_object();
126
-
127
- json_object *json_item = json_object_new_string(module->name);
128
- json_object_object_add(json_module, "name", json_item);
129
- const char *module_type = module_type2str(module->type);
130
- json_item = json_object_new_string(module_type);
131
- json_object_object_add(json_module, "type", json_item);
132
-
133
- json_object_array_add(obj, json_module);
134
-
135
- return 0;
136
-}
137
-
138
-json_object *get_list_of_modules_json(struct configurable_plugin *plugin)
139
-{
140
- json_object *obj = json_object_new_array();
141
-
142
- pthread_mutex_lock(&plugin->lock);
143
-
144
- dictionary_walkthrough_read(plugin->modules, _get_list_of_modules_json_cb, obj);
145
-
146
- pthread_mutex_unlock(&plugin->lock);
147
-
148
- return obj;
149
-}
150
-
151
-const char *job_status2str(enum job_status status)
152
-{
153
- switch (status) {
154
- case JOB_STATUS_UNKNOWN:
155
- return "unknown";
156
- case JOB_STATUS_STOPPED:
157
- return "stopped";
158
- case JOB_STATUS_RUNNING:
159
- return "running";
160
- case JOB_STATUS_ERROR:
161
- return "error";
162
- default:
163
- return "unknown";
164
- }
165
-}
166
-
167
-static void _job_flags2str_cb(const char *str, void *data)
168
-{
169
- json_object *json_item = json_object_new_string(str);
170
- json_object_array_add((json_object *)data, json_item);
171
-}
172
-
173
-json_object *job2json(struct job *job) {
174
- json_object *json_job = json_object_new_object();
175
-
176
- json_object *json_item = json_object_new_string(job->name);
177
- json_object_object_add(json_job, "name", json_item);
178
-
179
- json_item = json_object_new_string(job_type2str(job->type));
180
- json_object_object_add(json_job, "type", json_item);
181
-
182
- netdata_mutex_lock(&job->lock);
183
- json_item = json_object_new_string(job_status2str(job->status));
184
- json_object_object_add(json_job, "status", json_item);
185
-
186
- json_item = json_object_new_int(job->state);
187
- json_object_object_add(json_job, "state", json_item);
188
-
189
- json_item = job->reason == NULL ? NULL : json_object_new_string(job->reason);
190
- json_object_object_add(json_job, "reason", json_item);
191
-
192
- int64_t last_state_update_s = job->last_state_update / USEC_PER_SEC;
193
- int64_t last_state_update_us = job->last_state_update % USEC_PER_SEC;
194
-
195
- json_item = json_object_new_int64(last_state_update_s);
196
- json_object_object_add(json_job, "last_state_update_s", json_item);
197
-
198
- json_item = json_object_new_int64(last_state_update_us);
199
- json_object_object_add(json_job, "last_state_update_us", json_item);
200
-
201
- json_item = json_object_new_array();
202
- job_flags_wallkthrough(job->flags, _job_flags2str_cb, json_item);
203
- json_object_object_add(json_job, "flags", json_item);
204
-
205
- netdata_mutex_unlock(&job->lock);
206
-
207
- return json_job;
208
-}
209
-
210
-static int _get_list_of_jobs_json_cb(const DICTIONARY_ITEM *item, void *entry, void *data)
211
-{
212
- UNUSED(item);
213
- json_object *obj = (json_object *)data;
214
-
215
- json_object *json_job = job2json((struct job *)entry);
216
-
217
- json_object_array_add(obj, json_job);
218
-
219
- return 0;
220
-}
221
-
222
-json_object *get_list_of_jobs_json(struct module *module)
223
-{
224
- json_object *obj = json_object_new_array();
225
-
226
- pthread_mutex_lock(&module->lock);
227
-
228
- dictionary_walkthrough_read(module->jobs, _get_list_of_jobs_json_cb, obj);
229
-
230
- pthread_mutex_unlock(&module->lock);
231
-
232
- return obj;
233
-}
234
-
235
-struct job *get_job_by_name(struct module *module, const char *job_name)
236
-{
237
- return dictionary_get(module->jobs, job_name);
238
-}
239
-
240
-void unlink_job(const char *plugin_name, const char *module_name, const char *job_name)
241
-{
242
- // as we are going to do unlink here we better make sure we have all to build proper path
243
- if (unlikely(job_name == NULL || module_name == NULL || plugin_name == NULL))
244
- return;
245
- BUFFER *buffer = buffer_create(DYN_CONF_PATH_MAX, NULL);
246
- buffer_sprintf(buffer, DYN_CONF_DIR "/%s/%s/%s" DYN_CONF_CFG_EXT, plugin_name, module_name, job_name);
247
- if (unlink(buffer_tostring(buffer)))
248
- netdata_log_error("Cannot remove file %s", buffer_tostring(buffer));
249
-
250
- buffer_free(buffer);
251
-}
252
-
253
-void delete_job(struct configurable_plugin *plugin, const char *module_name, const char *job_name)
254
-{
255
- struct module *module = get_module_by_name(plugin, module_name);
256
- if (module == NULL) {
257
- error_report("DYNCFG module \"%s\" not found", module_name);
258
- return;
259
- }
260
-
261
- struct job *job_item = get_job_by_name(module, job_name);
262
- if (job_item == NULL) {
263
- error_report("DYNCFG job \"%s\" not found", job_name);
264
- return;
265
- }
266
-
267
- dictionary_del(module->jobs, job_name);
268
-}
269
-
270
-void delete_job_pname(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name)
271
-{
272
- const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name);
273
- if (plugin_item == NULL) {
274
- error_report("DYNCFG plugin \"%s\" not found", plugin_name);
275
- return;
276
- }
277
- struct configurable_plugin *plugin = dictionary_acquired_item_value(plugin_item);
278
-
279
- delete_job(plugin, module_name, job_name);
280
-
281
- dictionary_acquired_item_release(plugins_dict, plugin_item);
282
-}
283
-
284
-int remove_job(struct module *module, struct job *job)
285
-{
286
- enum set_config_result rc = module->delete_job_cb(module->job_config_cb_usr_ctx, module->plugin->name, module->name, job->name);
287
-
288
- if (rc != SET_CONFIG_ACCEPTED) {
289
- error_report("DYNCFG module \"%s\" rejected delete job for \"%s\"", module->name, job->name);
290
- return 0;
291
- }
292
- return 1;
293
-}
294
-
295
-struct module *get_module_by_name(struct configurable_plugin *plugin, const char *module_name)
296
-{
297
- return dictionary_get(plugin->modules, module_name);
298
-}
299
-
300
-inline struct configurable_plugin *get_plugin_by_name(DICTIONARY *plugins_dict, const char *name)
301
-{
302
- return dictionary_get(plugins_dict, name);
303
-}
304
-
305
-static int store_config(const char *module_name, const char *submodule_name, const char *cfg_idx, dyncfg_config_t cfg)
306
-{
307
- BUFFER *filename = buffer_create(DYN_CONF_PATH_MAX, NULL);
308
- buffer_sprintf(filename, DYN_CONF_DIR "/%s", module_name);
309
- if (mkdir(buffer_tostring(filename), 0755) == -1) {
310
- if (errno != EEXIST) {
311
- netdata_log_error("DYNCFG store_config: failed to create module directory %s", buffer_tostring(filename));
312
- buffer_free(filename);
313
- return 1;
314
- }
315
- }
316
-
317
- if (submodule_name != NULL) {
318
- buffer_sprintf(filename, "/%s", submodule_name);
319
- if (mkdir(buffer_tostring(filename), 0755) == -1) {
320
- if (errno != EEXIST) {
321
- netdata_log_error("DYNCFG store_config: failed to create submodule directory %s", buffer_tostring(filename));
322
- buffer_free(filename);
323
- return 1;
324
- }
325
- }
326
- }
327
-
328
- if (cfg_idx != NULL)
329
- buffer_sprintf(filename, "/%s", cfg_idx);
330
-
331
- buffer_strcat(filename, DYN_CONF_CFG_EXT);
332
-
333
-
334
- error_report("DYNCFG store_config: %s", buffer_tostring(filename));
335
-
336
- //write to file
337
- FILE *f = fopen(buffer_tostring(filename), "w");
338
- if (f == NULL) {
339
- error_report("DYNCFG store_config: failed to open %s for writing", buffer_tostring(filename));
340
- buffer_free(filename);
341
- return 1;
342
- }
343
-
344
- fwrite(cfg.data, cfg.data_size, 1, f);
345
- fclose(f);
346
-
347
- buffer_free(filename);
348
- return 0;
349
-}
350
-
351
-#ifdef NETDATA_DEV_MODE
352
-#define netdata_dev_fatal(...) fatal(__VA_ARGS__)
353
-#else
354
-#define netdata_dev_fatal(...) error_report(__VA_ARGS__)
355
-#endif
356
-
357
-void dyn_conf_store_config(const char *function, const char *payload, struct configurable_plugin *plugin) {
358
- dyncfg_config_t config = {
359
- .data = (char*)payload,
360
- .data_size = strlen(payload)
361
- };
362
-
363
- char *fnc = strdupz(function);
364
- // split fnc to words
365
- char *words[DYNCFG_MAX_WORDS];
366
- size_t words_c = quoted_strings_splitter(fnc, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd);
367
-
368
- const char *fnc_name = get_word(words, words_c, 0);
369
- if (fnc_name == NULL) {
370
- error_report("Function name expected \"%s\"", function);
371
- goto CLEANUP;
372
- }
373
- if (strncmp(fnc_name, FUNCTION_NAME_SET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_SET_PLUGIN_CONFIG)) == 0) {
374
- store_config(plugin->name, NULL, NULL, config);
375
- goto CLEANUP;
376
- }
377
-
378
- if (words_c < 2) {
379
- error_report("Module name expected \"%s\"", function);
380
- goto CLEANUP;
381
- }
382
- const char *module_name = get_word(words, words_c, 1);
383
- if (strncmp(fnc_name, FUNCTION_NAME_SET_MODULE_CONFIG, strlen(FUNCTION_NAME_SET_MODULE_CONFIG)) == 0) {
384
- store_config(plugin->name, module_name, NULL, config);
385
- goto CLEANUP;
386
- }
387
-
388
- if (words_c < 3) {
389
- error_report("Job name expected \"%s\"", function);
390
- goto CLEANUP;
391
- }
392
- const char *job_name = get_word(words, words_c, 2);
393
- if (strncmp(fnc_name, FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0) {
394
- store_config(plugin->name, module_name, job_name, config);
395
- goto CLEANUP;
396
- }
397
-
398
- netdata_dev_fatal("Unknown function \"%s\"", function);
399
-
400
-CLEANUP:
401
- freez(fnc);
402
-}
403
-
404
-dyncfg_config_t load_config(const char *plugin_name, const char *module_name, const char *job_id)
405
-{
406
- BUFFER *filename = buffer_create(DYN_CONF_PATH_MAX, NULL);
407
- buffer_sprintf(filename, DYN_CONF_DIR "/%s", plugin_name);
408
- if (module_name != NULL)
409
- buffer_sprintf(filename, "/%s", module_name);
410
-
411
- if (job_id != NULL)
412
- buffer_sprintf(filename, "/%s", job_id);
413
-
414
- buffer_strcat(filename, DYN_CONF_CFG_EXT);
415
-
416
- dyncfg_config_t config;
417
- long bytes;
418
- config.data = read_by_filename(buffer_tostring(filename), &bytes);
419
-
420
- if (config.data == NULL)
421
- error_report("DYNCFG load_config: failed to load config from %s", buffer_tostring(filename));
422
-
423
- config.data_size = bytes;
424
-
425
- buffer_free(filename);
426
-
427
- return config;
428
-}
429
-
430
-char *set_plugin_config(struct configurable_plugin *plugin, dyncfg_config_t cfg)
431
-{
432
- enum set_config_result rc = plugin->set_config_cb(plugin->cb_usr_ctx, plugin->name, &cfg);
433
- if (rc != SET_CONFIG_ACCEPTED) {
434
- error_report("DYNCFG plugin \"%s\" rejected config", plugin->name);
435
- return "plugin rejected config";
436
- }
437
-
438
- return NULL;
439
-}
440
-
441
-static char *set_module_config(struct module *mod, dyncfg_config_t cfg)
442
-{
443
- struct configurable_plugin *plugin = mod->plugin;
444
-
445
- enum set_config_result rc = mod->set_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name, &cfg);
446
- if (rc != SET_CONFIG_ACCEPTED) {
447
- error_report("DYNCFG module \"%s\" rejected config", plugin->name);
448
- return "module rejected config";
449
- }
450
-
451
- return NULL;
452
-}
453
-
454
-struct job *job_new(const char *job_id)
455
-{
456
- struct job *job = callocz(1, sizeof(struct job));
457
- job->state = JOB_STATUS_UNKNOWN;
458
- job->last_state_update = now_realtime_usec();
459
- job->name = strdupz(job_id);
460
- netdata_mutex_init(&job->lock);
461
- return job;
462
-}
463
-
464
-static inline void job_del(struct job *job)
465
-{
466
- netdata_mutex_destroy(&job->lock);
467
- freez(job->reason);
468
- freez((void*)job->name);
469
- freez(job);
470
-}
471
-
472
-void job_del_cb(const DICTIONARY_ITEM *item, void *value, void *data)
473
-{
474
- UNUSED(item);
475
- UNUSED(data);
476
- job_del((struct job *)value);
477
-}
478
-
479
-void module_del_cb(const DICTIONARY_ITEM *item, void *value, void *data)
480
-{
481
- UNUSED(item);
482
- UNUSED(data);
483
- struct module *mod = (struct module *)value;
484
- dictionary_destroy(mod->jobs);
485
- freez(mod->name);
486
- freez(mod);
487
-}
488
-
489
-const DICTIONARY_ITEM *register_plugin(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, bool localhost)
490
-{
491
- if (get_plugin_by_name(plugins_dict, plugin->name) != NULL) {
492
- error_report("DYNCFG plugin \"%s\" already registered", plugin->name);
493
- return NULL;
494
- }
495
-
496
- if (plugin->set_config_cb == NULL) {
497
- error_report("DYNCFG plugin \"%s\" has no set_config_cb", plugin->name);
498
- return NULL;
499
- }
500
-
501
- pthread_mutex_init(&plugin->lock, NULL);
502
-
503
- plugin->modules = dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE);
504
- dictionary_register_delete_callback(plugin->modules, module_del_cb, NULL);
505
-
506
- if (localhost)
507
- deferred_config_push_back(plugins_dict, plugin->name, NULL, NULL);
508
-
509
- dictionary_set(plugins_dict, plugin->name, plugin, sizeof(plugin));
510
-
511
- // the plugin keeps the pointer to the dictionary item, so we need to acquire it
512
- return dictionary_get_and_acquire_item(plugins_dict, plugin->name);
513
-}
514
-
515
-void unregister_plugin(DICTIONARY *plugins_dict, const DICTIONARY_ITEM *plugin)
516
-{
517
- struct configurable_plugin *plug = dictionary_acquired_item_value(plugin);
518
- dictionary_acquired_item_release(plugins_dict, plugin);
519
- dictionary_del(plugins_dict, plug->name);
520
-}
521
-
522
-int register_module(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, struct module *module, bool localhost)
523
-{
524
- if (get_module_by_name(plugin, module->name) != NULL) {
525
- error_report("DYNCFG module \"%s\" already registered", module->name);
526
- return 1;
527
- }
528
-
529
- pthread_mutex_init(&module->lock, NULL);
530
-
531
- if (localhost)
532
- deferred_config_push_back(plugins_dict, plugin->name, module->name, NULL);
533
-
534
- module->plugin = plugin;
535
-
536
- if (module->type == MOD_TYPE_ARRAY) {
537
- module->jobs = dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE);
538
- dictionary_register_delete_callback(module->jobs, job_del_cb, NULL);
539
-
540
- if (localhost) {
541
- // load all jobs from disk
542
- BUFFER *path = buffer_create(DYN_CONF_PATH_MAX, NULL);
543
- buffer_sprintf(path, "%s/%s/%s", DYN_CONF_DIR, plugin->name, module->name);
544
- DIR *dir = opendir(buffer_tostring(path));
545
- if (dir != NULL) {
546
- struct dirent *ent;
547
- while ((ent = readdir(dir)) != NULL) {
548
- if (ent->d_name[0] == '.')
549
- continue;
550
- if (ent->d_type != DT_REG)
551
- continue;
552
- size_t len = strnlen(ent->d_name, NAME_MAX);
553
- if (len <= strlen(DYN_CONF_CFG_EXT))
554
- continue;
555
- if (strcmp(ent->d_name + len - strlen(DYN_CONF_CFG_EXT), DYN_CONF_CFG_EXT) != 0)
556
- continue;
557
- ent->d_name[len - strlen(DYN_CONF_CFG_EXT)] = '\0';
558
-
559
- struct job *job = job_new(ent->d_name);
560
- job->module = module;
561
- job->flags = JOB_FLG_PS_LOADED;
562
- job->type = JOB_TYPE_USER;
563
-
564
- dictionary_set(module->jobs, job->name, job, sizeof(job));
565
-
566
- deferred_config_push_back(plugins_dict, plugin->name, module->name, ent->d_name);
567
- }
568
- closedir(dir);
569
- }
570
- buffer_free(path);
571
- }
572
- }
573
-
574
- dictionary_set(plugin->modules, module->name, module, sizeof(module));
575
-
576
- return 0;
577
-}
578
-
579
-int register_job(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_type job_type, dyncfg_job_flg_t flags, int ignore_existing)
580
-{
581
- int rc = 1;
582
- const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name);
583
- if (plugin_item == NULL) {
584
- error_report("plugin \"%s\" not registered", plugin_name);
585
- return rc;
586
- }
587
- struct configurable_plugin *plugin = dictionary_acquired_item_value(plugin_item);
588
- struct module *mod = get_module_by_name(plugin, module_name);
589
- if (mod == NULL) {
590
- error_report("module \"%s\" not registered", module_name);
591
- goto ERR_EXIT;
592
- }
593
- if (mod->type != MOD_TYPE_ARRAY) {
594
- error_report("module \"%s\" is not an array", module_name);
595
- goto ERR_EXIT;
596
- }
597
- if (get_job_by_name(mod, job_name) != NULL) {
598
- if (!ignore_existing)
599
- error_report("job \"%s\" already registered", job_name);
600
- goto ERR_EXIT;
601
- }
602
-
603
- struct job *job = job_new(job_name);
604
- job->module = mod;
605
- job->flags = flags;
606
- job->type = job_type;
607
-
608
- dictionary_set(mod->jobs, job->name, job, sizeof(job));
609
-
610
- rc = 0;
611
-ERR_EXIT:
612
- dictionary_acquired_item_release(plugins_dict, plugin_item);
613
- return rc;
614
-}
615
-
616
-void freez_dyncfg(void *ptr) {
617
- freez(ptr);
618
-}
619
-
620
-#ifdef NETDATA_TEST_DYNCFG
621
-static void handle_dyncfg_root(DICTIONARY *plugins_dict, struct uni_http_response *resp, int method)
622
-{
623
- if (method != HTTP_REQUEST_MODE_GET) {
624
- resp->content = "method not allowed";
625
- resp->content_length = strlen(resp->content);
626
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
627
- return;
628
- }
629
- json_object *obj = get_list_of_plugins_json(plugins_dict);
630
- json_object *wrapper = json_object_new_object();
631
- json_object_object_add(wrapper, "configurable_plugins", obj);
632
- resp->content = strdupz(json_object_to_json_string_ext(wrapper, JSON_C_TO_STRING_PRETTY));
633
- json_object_put(wrapper);
634
- resp->status = HTTP_RESP_OK;
635
- resp->content_type = CT_APPLICATION_JSON;
636
- resp->content_free = freez_dyncfg;
637
- resp->content_length = strlen(resp->content);
638
-}
639
-
640
-static void handle_plugin_root(struct uni_http_response *resp, int method, struct configurable_plugin *plugin, void *post_payload, size_t post_payload_size)
641
-{
642
- switch(method) {
643
- case HTTP_REQUEST_MODE_GET:
644
- {
645
- dyncfg_config_t cfg = plugin->get_config_cb(plugin->cb_usr_ctx, plugin->name);
646
- resp->content = mallocz(cfg.data_size);
647
- memcpy(resp->content, cfg.data, cfg.data_size);
648
- resp->status = HTTP_RESP_OK;
649
- resp->content_free = freez_dyncfg;
650
- resp->content_length = cfg.data_size;
651
- return;
652
- }
653
- case HTTP_REQUEST_MODE_PUT:
654
- {
655
- char *response;
656
- if (post_payload == NULL) {
657
- resp->content = "no payload";
658
- resp->content_length = strlen(resp->content);
659
- resp->status = HTTP_RESP_BAD_REQUEST;
660
- return;
661
- }
662
- dyncfg_config_t cont = {
663
- .data = post_payload,
664
- .data_size = post_payload_size
665
- };
666
- response = set_plugin_config(plugin, cont);
667
- if (response == NULL) {
668
- resp->status = HTTP_RESP_OK;
669
- resp->content = "OK";
670
- resp->content_length = strlen(resp->content);
671
- } else {
672
- resp->status = HTTP_RESP_BAD_REQUEST;
673
- resp->content = response;
674
- resp->content_length = strlen(resp->content);
675
- }
676
- return;
677
- }
678
- default:
679
- resp->content = "method not allowed";
680
- resp->content_length = strlen(resp->content);
681
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
682
- return;
683
- }
684
-}
685
-#endif
686
-
687
-void handle_module_root(struct uni_http_response *resp, int method, struct configurable_plugin *plugin, const char *module, void *post_payload, size_t post_payload_size)
688
-{
689
- if (strncmp(module, DYN_CONF_SCHEMA, sizeof(DYN_CONF_SCHEMA)) == 0) {
690
- dyncfg_config_t cfg = plugin->get_config_schema_cb(plugin->cb_usr_ctx, plugin->name);
691
- resp->content = mallocz(cfg.data_size);
692
- memcpy(resp->content, cfg.data, cfg.data_size);
693
- resp->status = HTTP_RESP_OK;
694
- resp->content_free = freez_dyncfg;
695
- resp->content_length = cfg.data_size;
696
- return;
697
- }
698
- if (strncmp(module, DYN_CONF_MODULE_LIST, sizeof(DYN_CONF_MODULE_LIST)) == 0) {
699
- if (method != HTTP_REQUEST_MODE_GET) {
700
- resp->content = "method not allowed (only GET)";
701
- resp->content_length = strlen(resp->content);
702
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
703
- return;
704
- }
705
- json_object *obj = get_list_of_modules_json(plugin);
706
- json_object *wrapper = json_object_new_object();
707
- json_object_object_add(wrapper, "modules", obj);
708
- resp->content = strdupz(json_object_to_json_string_ext(wrapper, JSON_C_TO_STRING_PRETTY));
709
- json_object_put(wrapper);
710
- resp->status = HTTP_RESP_OK;
711
- resp->content_type = CT_APPLICATION_JSON;
712
- resp->content_free = freez_dyncfg;
713
- resp->content_length = strlen(resp->content);
714
- return;
715
- }
716
- struct module *mod = get_module_by_name(plugin, module);
717
- if (mod == NULL) {
718
- resp->content = "module not found";
719
- resp->content_length = strlen(resp->content);
720
- resp->status = HTTP_RESP_NOT_FOUND;
721
- return;
722
- }
723
- if (method == HTTP_REQUEST_MODE_GET) {
724
- dyncfg_config_t cfg = mod->get_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name);
725
- resp->content = mallocz(cfg.data_size);
726
- memcpy(resp->content, cfg.data, cfg.data_size);
727
- resp->status = HTTP_RESP_OK;
728
- resp->content_free = freez_dyncfg;
729
- resp->content_length = cfg.data_size;
730
- return;
731
- } else if (method == HTTP_REQUEST_MODE_PUT) {
732
- char *response;
733
- if (post_payload == NULL) {
734
- resp->content = "no payload";
735
- resp->content_length = strlen(resp->content);
736
- resp->status = HTTP_RESP_BAD_REQUEST;
737
- return;
738
- }
739
- dyncfg_config_t cont = {
740
- .data = post_payload,
741
- .data_size = post_payload_size
742
- };
743
- response = set_module_config(mod, cont);
744
- if (response == NULL) {
745
- resp->status = HTTP_RESP_OK;
746
- resp->content = "OK";
747
- resp->content_length = strlen(resp->content);
748
- } else {
749
- resp->status = HTTP_RESP_BAD_REQUEST;
750
- resp->content = response;
751
- resp->content_length = strlen(resp->content);
752
- }
753
- return;
754
- }
755
- resp->content = "method not allowed";
756
- resp->content_length = strlen(resp->content);
757
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
758
-}
759
-
760
-static inline void _handle_job_root(struct uni_http_response *resp, int method, struct module *mod, const char *job_id, void *post_payload, size_t post_payload_size, struct job *job)
761
-{
762
- if (method == HTTP_REQUEST_MODE_POST) {
763
- if (job != NULL) {
764
- resp->content = "can't POST, job already exists (use PUT to update?)";
765
- resp->content_length = strlen(resp->content);
766
- resp->status = HTTP_RESP_BAD_REQUEST;
767
- return;
768
- }
769
- if (post_payload == NULL) {
770
- resp->content = "no payload";
771
- resp->content_length = strlen(resp->content);
772
- resp->status = HTTP_RESP_BAD_REQUEST;
773
- return;
774
- }
775
- dyncfg_config_t cont = {
776
- .data = post_payload,
777
- .data_size = post_payload_size
778
- };
779
- if (mod->set_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job_id, &cont)) {
780
- resp->content = "failed to add job";
781
- resp->content_length = strlen(resp->content);
782
- resp->status = HTTP_RESP_INTERNAL_SERVER_ERROR;
783
- return;
784
- }
785
- resp->status = HTTP_RESP_OK;
786
- resp->content = "OK";
787
- resp->content_length = strlen(resp->content);
788
- return;
789
- }
790
- if (job == NULL) {
791
- resp->content = "job not found";
792
- resp->content_length = strlen(resp->content);
793
- resp->status = HTTP_RESP_NOT_FOUND;
794
- return;
795
- }
796
- switch (method) {
797
- case HTTP_REQUEST_MODE_GET:
798
- {
799
- dyncfg_config_t cfg = mod->get_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job->name);
800
- resp->content = mallocz(cfg.data_size);
801
- memcpy(resp->content, cfg.data, cfg.data_size);
802
- resp->status = HTTP_RESP_OK;
803
- resp->content_free = freez_dyncfg;
804
- resp->content_length = cfg.data_size;
805
- return;
806
- }
807
- case HTTP_REQUEST_MODE_PUT:
808
- {
809
- if (post_payload == NULL) {
810
- resp->content = "missing payload";
811
- resp->content_length = strlen(resp->content);
812
- resp->status = HTTP_RESP_BAD_REQUEST;
813
- return;
814
- }
815
- dyncfg_config_t cont = {
816
- .data = post_payload,
817
- .data_size = post_payload_size
818
- };
819
- if (mod->set_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job->name, &cont) != SET_CONFIG_ACCEPTED) {
820
- error_report("DYNCFG module \"%s\" rejected config for job \"%s\"", mod->name, job->name);
821
- resp->content = "failed to set job config";
822
- resp->content_length = strlen(resp->content);
823
- resp->status = HTTP_RESP_INTERNAL_SERVER_ERROR;
824
- return;
825
- }
826
- resp->status = HTTP_RESP_OK;
827
- resp->content = "OK";
828
- resp->content_length = strlen(resp->content);
829
- return;
830
- }
831
- case HTTP_REQUEST_MODE_DELETE:
832
- {
833
- if (!remove_job(mod, job)) {
834
- resp->content = "failed to remove job";
835
- resp->content_length = strlen(resp->content);
836
- resp->status = HTTP_RESP_INTERNAL_SERVER_ERROR;
837
- return;
838
- }
839
- resp->status = HTTP_RESP_OK;
840
- resp->content = "OK";
841
- resp->content_length = strlen(resp->content);
842
- return;
843
- }
844
- default:
845
- resp->content = "method not allowed (only GET, PUT, DELETE)";
846
- resp->content_length = strlen(resp->content);
847
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
848
- return;
849
- }
850
-}
851
-
852
-void handle_job_root(struct uni_http_response *resp, int method, struct module *mod, const char *job_id, void *post_payload, size_t post_payload_size)
853
-{
854
- if (strncmp(job_id, DYN_CONF_SCHEMA, sizeof(DYN_CONF_SCHEMA)) == 0) {
855
- dyncfg_config_t cfg = mod->get_config_schema_cb(mod->config_cb_usr_ctx, mod->plugin->name, mod->name);
856
- resp->content = mallocz(cfg.data_size);
857
- memcpy(resp->content, cfg.data, cfg.data_size);
858
- resp->status = HTTP_RESP_OK;
859
- resp->content_free = freez_dyncfg;
860
- resp->content_length = cfg.data_size;
861
- return;
862
- }
863
- if (strncmp(job_id, DYN_CONF_JOB_SCHEMA, sizeof(DYN_CONF_JOB_SCHEMA)) == 0) {
864
- dyncfg_config_t cfg = mod->get_job_config_schema_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name);
865
- resp->content = mallocz(cfg.data_size);
866
- memcpy(resp->content, cfg.data, cfg.data_size);
867
- resp->status = HTTP_RESP_OK;
868
- resp->content_free = freez_dyncfg;
869
- resp->content_length = cfg.data_size;
870
- return;
871
- }
872
- if (strncmp(job_id, DYN_CONF_JOB_LIST, sizeof(DYN_CONF_JOB_LIST)) == 0) {
873
- if (mod->type != MOD_TYPE_ARRAY) {
874
- resp->content = "module type is not job_array (can't get the list of jobs)";
875
- resp->content_length = strlen(resp->content);
876
- resp->status = HTTP_RESP_NOT_FOUND;
877
- return;
878
- }
879
- if (method != HTTP_REQUEST_MODE_GET) {
880
- resp->content = "method not allowed (only GET)";
881
- resp->content_length = strlen(resp->content);
882
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
883
- return;
884
- }
885
- json_object *obj = get_list_of_jobs_json(mod);
886
- json_object *wrapper = json_object_new_object();
887
- json_object_object_add(wrapper, "jobs", obj);
888
- resp->content = strdupz(json_object_to_json_string_ext(wrapper, JSON_C_TO_STRING_PRETTY));
889
- json_object_put(wrapper);
890
- resp->status = HTTP_RESP_OK;
891
- resp->content_type = CT_APPLICATION_JSON;
892
- resp->content_free = freez_dyncfg;
893
- resp->content_length = strlen(resp->content);
894
- return;
895
- }
896
- const DICTIONARY_ITEM *job_item = dictionary_get_and_acquire_item(mod->jobs, job_id);
897
- struct job *job = dictionary_acquired_item_value(job_item);
898
-
899
- _handle_job_root(resp, method, mod, job_id, post_payload, post_payload_size, job);
900
-
901
- dictionary_acquired_item_release(mod->jobs, job_item);
902
-}
903
-
904
-struct uni_http_response dyn_conf_process_http_request(
905
- DICTIONARY *plugins_dict __maybe_unused,
906
- int method __maybe_unused,
907
- const char *plugin __maybe_unused,
908
- const char *module __maybe_unused,
909
- const char *job_id __maybe_unused,
910
- void *post_payload __maybe_unused,
911
- size_t post_payload_size __maybe_unused)
912
-{
913
- struct uni_http_response resp = {
914
- .status = HTTP_RESP_INTERNAL_SERVER_ERROR,
915
- .content_type = CT_TEXT_PLAIN,
916
- .content = (char *) http_response_code2string(HTTP_RESP_INTERNAL_SERVER_ERROR),
917
- .content_free = NULL,
918
- .content_length = 0
919
- };
920
-#ifndef NETDATA_TEST_DYNCFG
921
- resp.content = "DYNCFG is disabled (as it is for now developer only feature). This will be enabled by default when ready for technical preview.";
922
- resp.content_length = strlen(resp.content);
923
- resp.status = HTTP_RESP_PRECOND_FAIL;
924
- return resp;
925
-#else
926
- if (plugin == NULL) {
927
- handle_dyncfg_root(plugins_dict, &resp, method);
928
- return resp;
929
- }
930
- const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin);
931
- if (plugin_item == NULL) {
932
- resp.content = "plugin not found";
933
- resp.content_length = strlen(resp.content);
934
- resp.status = HTTP_RESP_NOT_FOUND;
935
- return resp;
936
- }
937
- struct configurable_plugin *plug = dictionary_acquired_item_value(plugin_item);
938
- if (module == NULL) {
939
- handle_plugin_root(&resp, method, plug, post_payload, post_payload_size);
940
- goto EXIT_PLUGIN;
941
- }
942
- if (job_id == NULL) {
943
- handle_module_root(&resp, method, plug, module, post_payload, post_payload_size);
944
- goto EXIT_PLUGIN;
945
- }
946
- // for modules we do not do get_and_acquire as modules are never removed (only together with the plugin)
947
- struct module *mod = get_module_by_name(plug, module);
948
- if (mod == NULL) {
949
- resp.content = "module not found";
950
- resp.content_length = strlen(resp.content);
951
- resp.status = HTTP_RESP_NOT_FOUND;
952
- goto EXIT_PLUGIN;
953
- }
954
- if (mod->type != MOD_TYPE_ARRAY) {
955
- resp.content = "400 - this module is not array type";
956
- resp.content_length = strlen(resp.content);
957
- resp.status = HTTP_RESP_BAD_REQUEST;
958
- goto EXIT_PLUGIN;
959
- }
960
- handle_job_root(&resp, method, mod, job_id, post_payload, post_payload_size);
961
-
962
-EXIT_PLUGIN:
963
- dictionary_acquired_item_release(plugins_dict, plugin_item);
964
- return resp;
965
-#endif
966
-}
967
-
968
-void plugin_del_cb(const DICTIONARY_ITEM *item, void *value, void *data)
969
-{
970
- UNUSED(item);
971
- UNUSED(data);
972
- struct configurable_plugin *plugin = (struct configurable_plugin *)value;
973
- dictionary_destroy(plugin->modules);
974
- freez(plugin->name);
975
- freez(plugin);
976
-}
977
-
978
-// on failure - return NULL - all unlocked, nothing acquired
979
-// on success - return pointer to job item - keep job and plugin acquired and locked!!!
980
-// for caller convenience (to prevent another lock and races)
981
-// caller is responsible to unlock the job and release it when not needed anymore
982
-// this also avoids dependency creep
983
-const DICTIONARY_ITEM *report_job_status_acq_lock(DICTIONARY *plugins_dict, const DICTIONARY_ITEM **plugin_acq_item, DICTIONARY **job_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_status status, int status_code, char *reason)
984
-{
985
- *plugin_acq_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name);
986
- if (*plugin_acq_item == NULL) {
987
- netdata_log_error("plugin %s not found", plugin_name);
988
- return NULL;
989
- }
990
-
991
- struct configurable_plugin *plug = dictionary_acquired_item_value(*plugin_acq_item);
992
- struct module *mod = get_module_by_name(plug, module_name);
993
- if (mod == NULL) {
994
- netdata_log_error("module %s not found", module_name);
995
- dictionary_acquired_item_release(plugins_dict, *plugin_acq_item);
996
- return NULL;
997
- }
998
- if (mod->type != MOD_TYPE_ARRAY) {
999
- netdata_log_error("module %s is not array", module_name);
1000
- dictionary_acquired_item_release(plugins_dict, *plugin_acq_item);
1001
- return NULL;
1002
- }
1003
- *job_dict = mod->jobs;
1004
- const DICTIONARY_ITEM *job_item = dictionary_get_and_acquire_item(mod->jobs, job_name);
1005
- if (job_item == NULL) {
1006
- netdata_log_error("job %s not found", job_name);
1007
- dictionary_acquired_item_release(plugins_dict, *plugin_acq_item);
1008
- return NULL;
1009
- }
1010
- struct job *job = dictionary_acquired_item_value(job_item);
1011
-
1012
- pthread_mutex_lock(&job->lock);
1013
- job->status = status;
1014
- job->state = status_code;
1015
- if (job->reason != NULL) {
1016
- freez(job->reason);
1017
- }
1018
- job->reason = reason != NULL ? strdupz(reason) : NULL; // reason is optional
1019
- job->last_state_update = now_realtime_usec();
1020
-
1021
- job->dirty = true;
1022
-
1023
- // no unlock and acquired_item_release on success on purpose
1024
- return job_item;
1025
-}
1026
-
1027
-int dyn_conf_init(void)
1028
-{
1029
- if (mkdir(DYN_CONF_DIR, 0755) == -1) {
1030
- if (errno != EEXIST) {
1031
- netdata_log_error("failed to create directory for dynamic configuration");
1032
- return 1;
1033
- }
1034
- }
1035
-
1036
- return 0;
1037
-}
1038
-
1039
-static void dyncfg_cleanup(void *ptr) {
1040
- struct netdata_static_thread *static_thread = (struct netdata_static_thread *) ptr;
1041
- static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
1042
-
1043
- netdata_log_info("cleaning up...");
1044
-
1045
- pthread_mutex_lock(&deferred_configs_lock);
1046
- dyncfg_shutdown = true;
1047
- while (deferred_configs != NULL) {
1048
- struct deferred_cfg_send *dcs = deferred_configs;
1049
- deferred_configs = dcs->next;
1050
- deferred_config_free(dcs);
1051
- }
1052
- pthread_mutex_unlock(&deferred_configs_lock);
1053
-
1054
- static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
1055
-}
1056
-
1057
-void *dyncfg_main(void *ptr)
1058
-{
1059
- netdata_thread_cleanup_push(dyncfg_cleanup, ptr);
1060
-
1061
- while (!netdata_exit) {
1062
- struct deferred_cfg_send *dcs = deferred_config_pop(ptr);
1063
- DICTIONARY *plugins_dict = dcs->plugins_dict;
1064
-#ifdef NETDATA_INTERNAL_CHECKS
1065
- if (plugins_dict == NULL) {
1066
- fatal("DYNCFG, plugins_dict is NULL");
1067
- deferred_config_free(dcs);
1068
- continue;
1069
- }
1070
-#endif
1071
-
1072
- const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, dcs->plugin_name);
1073
- if (plugin_item == NULL) {
1074
- error_report("DYNCFG, plugin %s not found", dcs->plugin_name);
1075
- deferred_config_free(dcs);
1076
- continue;
1077
- }
1078
- struct configurable_plugin *plugin = dictionary_acquired_item_value(plugin_item);
1079
- if (dcs->module_name == NULL) {
1080
- dyncfg_config_t cfg = load_config(dcs->plugin_name, NULL, NULL);
1081
- if (cfg.data != NULL) {
1082
- plugin->set_config_cb(plugin->cb_usr_ctx, plugin->name, &cfg);
1083
- freez(cfg.data);
1084
- }
1085
- } else if (dcs->job_name == NULL) {
1086
- dyncfg_config_t cfg = load_config(dcs->plugin_name, dcs->module_name, NULL);
1087
- if (cfg.data != NULL) {
1088
- struct module *mod = get_module_by_name(plugin, dcs->module_name);
1089
- mod->set_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name, &cfg);
1090
- freez(cfg.data);
1091
- }
1092
- } else {
1093
- dyncfg_config_t cfg = load_config(dcs->plugin_name, dcs->module_name, dcs->job_name);
1094
- if (cfg.data != NULL) {
1095
- struct module *mod = get_module_by_name(plugin, dcs->module_name);
1096
- mod->set_job_config_cb(mod->job_config_cb_usr_ctx, plugin->name, mod->name, dcs->job_name, &cfg);
1097
- freez(cfg.data);
1098
- }
1099
- }
1100
- deferred_config_free(dcs);
1101
- dictionary_acquired_item_release(plugins_dict, plugin_item);
1102
- }
1103
-
1104
- netdata_thread_cleanup_pop(1);
1105
- return NULL;
1106
-}
1107
-
1108
-bool is_dyncfg_function(const char *function_name, uint8_t type) {
1109
- // TODO add hash to speed things up
1110
- if (type & (DYNCFG_FUNCTION_TYPE_GET | DYNCFG_FUNCTION_TYPE_REGULAR)) {
1111
- if (strncmp(function_name, FUNCTION_NAME_GET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_GET_PLUGIN_CONFIG)) == 0)
1112
- return true;
1113
- if (strncmp(function_name, FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA)) == 0)
1114
- return true;
1115
- if (strncmp(function_name, FUNCTION_NAME_GET_MODULE_CONFIG, strlen(FUNCTION_NAME_GET_MODULE_CONFIG)) == 0)
1116
- return true;
1117
- if (strncmp(function_name, FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA)) == 0)
1118
- return true;
1119
- if (strncmp(function_name, FUNCTION_NAME_GET_JOB_CONFIG, strlen(FUNCTION_NAME_GET_JOB_CONFIG)) == 0)
1120
- return true;
1121
- if (strncmp(function_name, FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA)) == 0)
1122
- return true;
1123
- }
1124
-
1125
- if (type & (DYNCFG_FUNCTION_TYPE_SET | DYNCFG_FUNCTION_TYPE_PAYLOAD)) {
1126
- if (strncmp(function_name, FUNCTION_NAME_SET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_SET_PLUGIN_CONFIG)) == 0)
1127
- return true;
1128
- if (strncmp(function_name, FUNCTION_NAME_SET_MODULE_CONFIG, strlen(FUNCTION_NAME_SET_MODULE_CONFIG)) == 0)
1129
- return true;
1130
- if (strncmp(function_name, FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0)
1131
- return true;
1132
- }
1133
-
1134
- if (type & (DYNCFG_FUNCTION_TYPE_DELETE | DYNCFG_FUNCTION_TYPE_REGULAR)) {
1135
- if (strncmp(function_name, FUNCTION_NAME_DELETE_JOB, strlen(FUNCTION_NAME_DELETE_JOB)) == 0)
1136
- return true;
1137
- }
1138
-
1139
- return false;
1140
-}
libnetdata/dyn_conf/dyn_conf.h
deleted
-237
@@ -1,237 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#ifndef DYN_CONF_H
4
-#define DYN_CONF_H
5
-
6
-#include "../libnetdata.h"
7
-
8
-#define FUNCTION_NAME_GET_PLUGIN_CONFIG "get_plugin_config"
9
-#define FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA "get_plugin_config_schema"
10
-#define FUNCTION_NAME_GET_MODULE_CONFIG "get_module_config"
11
-#define FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA "get_module_config_schema"
12
-#define FUNCTION_NAME_GET_JOB_CONFIG "get_job_config"
13
-#define FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA "get_job_config_schema"
14
-#define FUNCTION_NAME_SET_PLUGIN_CONFIG "set_plugin_config"
15
-#define FUNCTION_NAME_SET_MODULE_CONFIG "set_module_config"
16
-#define FUNCTION_NAME_SET_JOB_CONFIG "set_job_config"
17
-#define FUNCTION_NAME_DELETE_JOB "delete_job"
18
-
19
-#define DYNCFG_MAX_WORDS 5
20
-
21
-#define DYNCFG_VFNC_RET_CFG_ACCEPTED (1)
22
-
23
-enum module_type {
24
- MOD_TYPE_UNKNOWN = 0,
25
- MOD_TYPE_ARRAY,
26
- MOD_TYPE_SINGLE
27
-};
28
-
29
-static inline enum module_type str2_module_type(const char *type_name)
30
-{
31
- if (strcmp(type_name, "job_array") == 0)
32
- return MOD_TYPE_ARRAY;
33
- else if (strcmp(type_name, "single") == 0)
34
- return MOD_TYPE_SINGLE;
35
- return MOD_TYPE_UNKNOWN;
36
-}
37
-
38
-static inline const char *module_type2str(enum module_type type)
39
-{
40
- switch (type) {
41
- case MOD_TYPE_ARRAY:
42
- return "job_array";
43
- case MOD_TYPE_SINGLE:
44
- return "single";
45
- default:
46
- return "unknown";
47
- }
48
-}
49
-
50
-struct dyncfg_config {
51
- void *data;
52
- size_t data_size;
53
-};
54
-
55
-typedef struct dyncfg_config dyncfg_config_t;
56
-
57
-struct configurable_plugin;
58
-struct module;
59
-
60
-enum job_status {
61
- JOB_STATUS_UNKNOWN = 0, // State used until plugin reports first status
62
- JOB_STATUS_STOPPED,
63
- JOB_STATUS_RUNNING,
64
- JOB_STATUS_ERROR
65
-};
66
-
67
-static inline enum job_status str2job_state(const char *state_name) {
68
- if (strcmp(state_name, "stopped") == 0)
69
- return JOB_STATUS_STOPPED;
70
- else if (strcmp(state_name, "running") == 0)
71
- return JOB_STATUS_RUNNING;
72
- else if (strcmp(state_name, "error") == 0)
73
- return JOB_STATUS_ERROR;
74
- return JOB_STATUS_UNKNOWN;
75
-}
76
-
77
-const char *job_status2str(enum job_status status);
78
-
79
-enum set_config_result {
80
- SET_CONFIG_ACCEPTED = 0,
81
- SET_CONFIG_REJECTED,
82
- SET_CONFIG_DEFFER
83
-};
84
-
85
-typedef uint32_t dyncfg_job_flg_t;
86
-enum job_flags {
87
- JOB_FLG_PS_LOADED = 1 << 0, // PS abbr. Persistent Storage
88
- JOB_FLG_PLUGIN_PUSHED = 1 << 1, // got it from plugin (e.g. autodiscovered job)
89
- JOB_FLG_STREAMING_PUSHED = 1 << 2, // got it through streaming
90
- JOB_FLG_USER_CREATED = 1 << 3, // user created this job during agent runtime
91
-};
92
-
93
-enum job_type {
94
- JOB_TYPE_UNKNOWN = 0,
95
- JOB_TYPE_STOCK = 1,
96
- JOB_TYPE_USER = 2,
97
- JOB_TYPE_AUTODISCOVERED = 3,
98
-};
99
-
100
-static inline const char* job_type2str(enum job_type type)
101
-{
102
- switch (type) {
103
- case JOB_TYPE_STOCK:
104
- return "stock";
105
- case JOB_TYPE_USER:
106
- return "user";
107
- case JOB_TYPE_AUTODISCOVERED:
108
- return "autodiscovered";
109
- case JOB_TYPE_UNKNOWN:
110
- default:
111
- return "unknown";
112
- }
113
-}
114
-
115
-static inline enum job_type dyncfg_str2job_type(const char *type_name)
116
-{
117
- if (strcmp(type_name, "stock") == 0)
118
- return JOB_TYPE_STOCK;
119
- else if (strcmp(type_name, "user") == 0)
120
- return JOB_TYPE_USER;
121
- else if (strcmp(type_name, "autodiscovered") == 0)
122
- return JOB_TYPE_AUTODISCOVERED;
123
- error_report("Unknown job type: %s", type_name);
124
- return JOB_TYPE_UNKNOWN;
125
-}
126
-
127
-struct job
128
-{
129
- const char *name;
130
- enum job_type type;
131
- struct module *module;
132
-
133
- pthread_mutex_t lock;
134
- // lock protexts only fields below (which are modified during job existence)
135
- // others are static during lifetime of job
136
-
137
- int dirty; // this relates to rrdpush, true if parent has different data than us
138
-
139
- // state reported by plugin
140
- usec_t last_state_update;
141
- enum job_status status; // reported by plugin, enum as this has to be interpreted by UI
142
- int state; // code reported by plugin which can mean anything plugin wants
143
- char *reason; // reported by plugin, can be NULL (optional)
144
-
145
- dyncfg_job_flg_t flags;
146
-};
147
-
148
-struct module
149
-{
150
- pthread_mutex_t lock;
151
- char *name;
152
- enum module_type type;
153
-
154
- struct configurable_plugin *plugin;
155
-
156
- // module config
157
- enum set_config_result (*set_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, dyncfg_config_t *cfg);
158
- dyncfg_config_t (*get_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name);
159
- dyncfg_config_t (*get_config_schema_cb)(void *usr_ctx, const char *plugin_name, const char *module_name);
160
- void *config_cb_usr_ctx;
161
-
162
- DICTIONARY *jobs;
163
-
164
- // jobs config
165
- dyncfg_config_t (*get_job_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name);
166
- dyncfg_config_t (*get_job_config_schema_cb)(void *usr_ctx, const char *plugin_name, const char *module_name);
167
- enum set_config_result (*set_job_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name, dyncfg_config_t *cfg);
168
- enum set_config_result (*delete_job_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name);
169
- void *job_config_cb_usr_ctx;
170
-};
171
-
172
-struct configurable_plugin {
173
- pthread_mutex_t lock;
174
- char *name;
175
- DICTIONARY *modules;
176
- const char *schema;
177
-
178
- dyncfg_config_t (*get_config_cb)(void *usr_ctx, const char *plugin_name);
179
- dyncfg_config_t (*get_config_schema_cb)(void *usr_ctx, const char *plugin_name);
180
- enum set_config_result (*set_config_cb)(void *usr_ctx, const char *plugin_name, dyncfg_config_t *cfg);
181
- void *cb_usr_ctx; // context for all callbacks (split if needed in future)
182
-};
183
-
184
-// API to be used by plugins
185
-const DICTIONARY_ITEM *register_plugin(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, bool localhost);
186
-void unregister_plugin(DICTIONARY *plugins_dict, const DICTIONARY_ITEM *plugin);
187
-int register_module(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, struct module *module, bool localhost);
188
-int register_job(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_type job_type, dyncfg_job_flg_t flags, int ignore_existing);
189
-
190
-const DICTIONARY_ITEM *report_job_status_acq_lock(DICTIONARY *plugins_dict, const DICTIONARY_ITEM **plugin_acq_item, DICTIONARY **job_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_status status, int status_code, char *reason);
191
-
192
-void dyn_conf_store_config(const char *function, const char *payload, struct configurable_plugin *plugin);
193
-void unlink_job(const char *plugin_name, const char *module_name, const char *job_name);
194
-void delete_job(struct configurable_plugin *plugin, const char *module_name, const char *job_name);
195
-void delete_job_pname(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name);
196
-
197
-// API to be used by the web server(s)
198
-json_object *get_list_of_plugins_json(DICTIONARY *plugins_dict);
199
-struct configurable_plugin *get_plugin_by_name(DICTIONARY *plugins_dict, const char *name);
200
-
201
-json_object *get_list_of_modules_json(struct configurable_plugin *plugin);
202
-struct module *get_module_by_name(struct configurable_plugin *plugin, const char *module_name);
203
-
204
-json_object *job2json(struct job *job);
205
-
206
-// helper struct to make interface between internal webserver and h2o same
207
-struct uni_http_response {
208
- int status;
209
- char *content;
210
- size_t content_length;
211
- HTTP_CONTENT_TYPE content_type;
212
- void (*content_free)(void *);
213
-};
214
-
215
-struct uni_http_response dyn_conf_process_http_request(DICTIONARY *plugins_dict, int method, const char *plugin, const char *module, const char *job_id, void *payload, size_t payload_size);
216
-
217
-// API to be used by main netdata process, initialization and destruction etc.
218
-int dyn_conf_init(void);
219
-void freez_dyncfg(void *ptr);
220
-
221
-#define dyncfg_dictionary_create() dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE)
222
-
223
-void plugin_del_cb(const DICTIONARY_ITEM *item, void *value, void *data);
224
-
225
-void *dyncfg_main(void *in);
226
-
227
-#define DYNCFG_FUNCTION_TYPE_REGULAR (1 << 0)
228
-#define DYNCFG_FUNCTION_TYPE_PAYLOAD (1 << 1)
229
-#define DYNCFG_FUNCTION_TYPE_GET (1 << 2)
230
-#define DYNCFG_FUNCTION_TYPE_SET (1 << 3)
231
-#define DYNCFG_FUNCTION_TYPE_DELETE (1 << 4)
232
-#define DYNCFG_FUNCTION_TYPE_ALL \
233
- (DYNCFG_FUNCTION_TYPE_REGULAR | DYNCFG_FUNCTION_TYPE_PAYLOAD | DYNCFG_FUNCTION_TYPE_GET | DYNCFG_FUNCTION_TYPE_SET | DYNCFG_FUNCTION_TYPE_DELETE)
234
-
235
-bool is_dyncfg_function(const char *function_name, uint8_t type);
236
-
237
-#endif //DYN_CONF_H
libnetdata/dyn_conf/tests/sample_test_config.json
deleted
-22
@@ -1,22 +0,0 @@
1
-{
2
- "http_endpoints": {
3
- "parent": {
4
- "host": "127.0.0.1",
5
- "mguid": null,
6
- "port": 20001,
7
- "ssl": false
8
- },
9
- "child": {
10
- "host": "127.0.0.1",
11
- "mguid": "3bc2f7de-1445-11ee-9ed7-3c7c3f21784c",
12
- "port": 19999,
13
- "ssl": false
14
- }
15
- },
16
- "global": {
17
- "test_plugin_name": "external_plugin",
18
- "test_array_module_name": "module_of_the_future",
19
- "test_single_module_name": "module_of_the_future_single_type",
20
- "test_job_name": "fixed_job"
21
- }
22
-}
libnetdata/dyn_conf/tests/sub_tests/test_parent_child.rb
deleted
-192
@@ -1,192 +0,0 @@
1
-class ParentChildTest
2
- @@plugin_cfg = <<~HEREDOC
3
-{ "test" : "true" }
4
-HEREDOC
5
- @@plugin_cfg2 = <<~HEREDOC
6
-{ "asdfgh" : "asdfgh" }
7
-HEREDOC
8
-
9
- @@job_cfg = <<~HEREDOC
10
-{ "i am newly created job" : "true" }
11
-HEREDOC
12
-
13
- def initialize
14
- @parent = $config[:http_endpoints][:parent]
15
- @child = $config[:http_endpoints][:child]
16
- @plugin = $config[:global][:test_plugin_name]
17
- @arry_mod = $config[:global][:test_array_module_name]
18
- @single_mod = $config[:global][:test_single_module_name]
19
- @test_job = $config[:global][:test_job_name]
20
- end
21
- def check_test_plugin_modules_list(host, child = nil)
22
- rc = DynCfgHttpClient.get_plugin_module_list(host, @plugin, child)
23
- assert_eq(rc.code, 200, "as HTTP code for get_module_list request on plugin \"#{@plugin}\"")
24
- modules = nil
25
- assert_nothing_raised do
26
- modules = JSON.parse(rc.parsed_response, symbolize_names: true)
27
- end
28
- assert_has_key?(modules, :modules)
29
- assert_eq(modules[:modules].count, 2, "as number of modules in plugin \"#{@plugin}\"")
30
- modules[:modules].each do |m|
31
- assert_has_key?(m, :name)
32
- assert_has_key?(m, :type)
33
- assert_is_one_of(m[:type], "job_array", "single")
34
- end
35
- assert_eq_str(modules[:modules][0][:name], @arry_mod, "name of first module in plugin \"#{@plugin}\"")
36
- assert_eq_str(modules[:modules][1][:name], @single_mod, "name of second module in plugin \"#{@plugin}\"")
37
- end
38
- def run
39
- TEST_SUITE("Parent/Child plugin config")
40
-
41
- TEST("parent/child/get_plugin_list", "Get child (hops:1) plugin list trough parent")
42
- plugins = DynCfgHttpClient.get_plugin_list(@parent, @child)
43
- assert_eq(plugins.code, 200, "as HTTP code for get_plugin_list request")
44
- assert_nothing_raised do
45
- plugins = JSON.parse(plugins.parsed_response, symbolize_names: true)
46
- end
47
- assert_has_key?(plugins, :configurable_plugins)
48
- assert_array_include?(plugins[:configurable_plugins], @plugin)
49
- PASS()
50
-
51
- TEST("parent/child/(set/get)plugin_config", "Set then get and compare child (hops:1) plugin config trough parent")
52
- rc = DynCfgHttpClient.set_plugin_config(@parent, @plugin, @@plugin_cfg, @child)
53
- assert_eq(rc.code, 200, "as HTTP code for set_plugin_config request")
54
-
55
- rc = DynCfgHttpClient.get_plugin_config(@parent, @plugin, @child)
56
- assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request")
57
- assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg, "as plugin config")
58
-
59
- # We do this twice with different configs to ensure first config was not loaded from persistent storage (from previous tests)
60
- rc = DynCfgHttpClient.set_plugin_config(@parent, @plugin, @@plugin_cfg2, @child)
61
- assert_eq(rc.code, 200, "as HTTP code for set_plugin_config request 2")
62
-
63
- rc = DynCfgHttpClient.get_plugin_config(@parent, @plugin, @child)
64
- assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request 2")
65
- assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg2, "set/get plugin config 2")
66
- PASS()
67
-
68
- TEST("child/get_plugin_config", "Get child (hops:0) plugin config and compare with what we got trough parent (set_plugin_config from previous test)")
69
- rc = DynCfgHttpClient.get_plugin_config(@child, @plugin, nil)
70
- assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request")
71
- assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg2.chomp, "as plugin config")
72
- PASS()
73
-
74
- TEST("parent/child/plugin_module_list", "Get child (hops:1) plugin module list trough parent and check its contents")
75
- check_test_plugin_modules_list(@parent, @child)
76
- PASS()
77
-
78
- TEST("child/plugin_module_list", "Get child (hops:0) plugin module list directly and check its contents")
79
- check_test_plugin_modules_list(@child, nil)
80
- PASS()
81
-
82
- TEST("parent/child/module/jobs", "Get list of jobs from child (hops:1) trough parent and check its contents, check job updates")
83
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child)
84
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
85
- jobs = nil
86
- assert_nothing_raised do
87
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
88
- end
89
- assert_has_key?(jobs, :jobs)
90
- new_job = jobs[:jobs].find {|i| i[:name] == @test_job}
91
- assert_not_nil(new_job)
92
- assert_has_key?(new_job, :status)
93
- assert_not_eq_str(new_job[:status], "unknown", "job status is other than unknown")
94
- assert_has_key?(new_job, :flags)
95
- assert_array_include?(new_job[:flags], "JOB_FLG_STREAMING_PUSHED")
96
- PASS()
97
-
98
- TEST("child/module/jobs", "Get list of jobs direct from child (hops:0) and check its contents, check job updates")
99
- rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil)
100
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
101
- jobs = nil
102
- assert_nothing_raised do
103
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
104
- end
105
- assert_has_key?(jobs, :jobs)
106
- new_job = jobs[:jobs].find {|i| i[:name] == @test_job}
107
- assert_not_nil(new_job)
108
- assert_has_key?(new_job, :status)
109
- assert_not_eq_str(new_job[:status], "unknown", "job status is other than unknown")
110
- assert_has_key?(new_job, :flags)
111
-
112
- assert_array_not_include?(new_job[:flags], "JOB_FLG_STREAMING_PUSHED") # this is plugin directly at child so it should not show this flag
113
- PASS()
114
-
115
- TEST("parent/child/single_module/jobs", "Attempt getting list of jobs from child (hops:1) trough parent on single module. Check it fails properly")
116
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @single_mod, @child)
117
- assert_eq(rc.code, 400, "as HTTP code for get_jobs request")
118
- assert_eq_str(rc.parsed_response, '400 - this module is not array type', "as HTTP code for get_jobs request on single module")
119
- PASS()
120
-
121
- created_job = SecureRandom.uuid
122
- TEST("parent/child/module/cr_del_job", "Create and delete job on child (hops:1) trough parent")
123
- # create new job
124
- rc = DynCfgHttpClient.create_job(@parent, @plugin, @arry_mod, created_job, @@job_cfg, @child)
125
- assert_eq_http_code(rc, 200, "as HTTP code for create_job request")
126
- # check this job is in job list @parent
127
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child)
128
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
129
- jobs = nil
130
- assert_nothing_raised do
131
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
132
- end
133
- assert_has_key?(jobs, :jobs)
134
- new_job = jobs[:jobs].find {|i| i[:name] == created_job}
135
- assert_not_nil(new_job)
136
- # check this job is in job list @child
137
- rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil)
138
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
139
- jobs = nil
140
- assert_nothing_raised do
141
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
142
- end
143
- assert_has_key?(jobs, :jobs)
144
- new_job = jobs[:jobs].find {|i| i[:name] == created_job}
145
- assert_not_nil(new_job)
146
- # check we can get job config back
147
- rc = DynCfgHttpClient.get_job_config(@parent, @plugin, @arry_mod, created_job, @child)
148
- assert_eq(rc.code, 200, "as HTTP code for get_job_config request")
149
- assert_eq_str(rc.parsed_response.chomp!, @@job_cfg, "as job config")
150
- # delete job
151
- rc = DynCfgHttpClient.delete_job(@parent, @plugin, @arry_mod, created_job, @child)
152
- assert_eq(rc.code, 200, "as HTTP code for delete_job request")
153
- # Check it is not in parents job list anymore
154
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child)
155
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
156
- jobs = nil
157
- assert_nothing_raised do
158
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
159
- end
160
- assert_has_key?(jobs, :jobs)
161
- new_job = jobs[:jobs].find {|i| i[:name] == created_job}
162
- assert_nil(new_job)
163
- # Check it is not in childs job list anymore
164
- rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil)
165
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
166
- jobs = nil
167
- assert_nothing_raised do
168
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
169
- end
170
- assert_has_key?(jobs, :jobs)
171
- new_job = jobs[:jobs].find {|i| i[:name] == created_job}
172
- assert_nil(new_job)
173
- PASS()
174
-
175
- TEST("parent/child/module/del_undeletable_job", "Try delete job on child (child rejects), check failure case works (hops:1)")
176
- # test if plugin rejects job deletion the job still remains in list as it should
177
- rc = DynCfgHttpClient.delete_job(@parent, @plugin, @arry_mod, @test_job, @child)
178
- assert_eq(rc.code, 500, "as HTTP code for delete_job request")
179
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child)
180
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
181
- jobs = nil
182
- assert_nothing_raised do
183
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
184
- end
185
- assert_has_key?(jobs, :jobs)
186
- job = jobs[:jobs].find {|i| i[:name] == @test_job}
187
- assert_not_nil(job)
188
- PASS()
189
- end
190
-end
191
-
192
-ParentChildTest.new.run()
libnetdata/dyn_conf/tests/test_dyncfg.rb
deleted
-266
@@ -1,266 +0,0 @@
1
-#!/usr/bin/env ruby
2
-
3
-require 'json'
4
-require 'httparty'
5
-require 'pastel'
6
-require 'securerandom'
7
-
8
-ARGV.length == 1 or raise "Usage: #{$0} <config file>"
9
-config_file = ARGV[0]
10
-
11
-File.exist?(config_file) or raise "File not found: #{config_file}"
12
-
13
-$config = JSON.parse(File.read(config_file), symbolize_names: true)
14
-
15
-$plugin_name = $config[:global][:test_plugin_name]
16
-$pastel = Pastel.new
17
-
18
-class TestRunner
19
- attr_reader :stats
20
- def initialize
21
- @stats = {
22
- :suites => 0,
23
- :tests => 0,
24
- :assertions => 0
25
- }
26
- @test = nil
27
- end
28
- def add_assertion()
29
- @stats[:assertions] += 1
30
- end
31
- def FAIL(msg, exception = nil, loc = nil)
32
- puts $pastel.red.bold(" ✕ FAIL")
33
- STDERR.print " "
34
- if loc
35
- STDERR.print $pastel.yellow("@#{loc.path}:#{loc.lineno}: ")
36
- else
37
- STDERR.print $pastel.yellow("@#{caller_locations(1, 1).first.path}:#{caller_locations(1, 1).first.lineno}: ")
38
- end
39
- STDERR.puts msg
40
- STDERR.puts exception.full_message(:highlight => true) if exception
41
- STDERR.puts $pastel.yellow(" Backtrace:")
42
- caller.each do |line|
43
- STDERR.puts " #{line}"
44
- end
45
- exit 1
46
- end
47
- def PASS()
48
- STDERR.puts $pastel.green.bold(" ✓ PASS")
49
- @stats[:tests] += 1
50
- @test = nil
51
- end
52
- def TEST_SUITE(name)
53
- puts $pastel.bold("• TEST SUITE: \"#{name}\"")
54
- @stats[:suites] += 1
55
- end
56
- def assert_no_test_running()
57
- unless @test.nil?
58
- STDERR.puts $pastel.red("\nFATAL: Test \"#{@test}\" did not call PASS() or FAIL()!")
59
- exit 1
60
- end
61
- end
62
- def TEST(name, description = nil)
63
- assert_no_test_running()
64
- @test = name
65
- col = 0
66
- txt = " ├─ T: #{name} "
67
- col += txt.length
68
- print $pastel.bold(txt)
69
-
70
- tab = 50
71
- rem = tab - (col % tab)
72
- rem.times do putc ' ' end
73
- col += rem
74
-
75
- if (description)
76
- txt = " - #{description} "
77
- col += txt.length
78
- print txt
79
-
80
- tab = 180
81
- rem = tab - (col % tab)
82
- rem.times do putc '.' end
83
- end
84
- end
85
- def FINALIZE()
86
- assert_no_test_running()
87
- end
88
-end
89
-
90
-$test_runner = TestRunner.new
91
-def FAIL(msg, exception = nil, loc = nil)
92
- $test_runner.FAIL(msg, exception, loc)
93
-end
94
-def PASS()
95
- $test_runner.PASS()
96
-end
97
-def TEST_SUITE(name)
98
- $test_runner.TEST_SUITE(name)
99
-end
100
-def TEST(name, description = nil)
101
- $test_runner.TEST(name, description)
102
-end
103
-
104
-def assert_eq(got, expected, msg = nil)
105
- unless got == expected
106
- FAIL("Expected #{expected}, got #{got} #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first)
107
- end
108
- $test_runner.add_assertion()
109
-end
110
-def assert_eq_http_code(got, expected, msg = nil)
111
- unless got.code == expected
112
- FAIL("Expected #{expected}, got #{got}. Server \"#{got.parsed_response}\" #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first)
113
- end
114
- $test_runner.add_assertion()
115
-end
116
-def assert_eq_str(got, expected, msg = nil)
117
- unless got == expected
118
- FAIL("Strings do not match #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first)
119
- end
120
- $test_runner.add_assertion()
121
-end
122
-def assert_not_eq_str(got, expected, msg = nil)
123
- unless got != expected
124
- FAIL("Strings shoud not match #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first)
125
- end
126
- $test_runner.add_assertion()
127
-end
128
-def assert_nothing_raised()
129
- begin
130
- yield
131
- rescue Exception => e
132
- FAIL("Unexpected exception of type #{e.class} raised. Msg: \"#{e.message}\"", e, caller_locations(1, 1).first)
133
- end
134
- $test_runner.add_assertion()
135
-end
136
-def assert_has_key?(hash, key)
137
- unless hash.has_key?(key)
138
- FAIL("Expected key \"#{key}\" in hash", nil, caller_locations(1, 1).first)
139
- end
140
- $test_runner.add_assertion()
141
-end
142
-def assert_array_include?(array, value)
143
- unless array.include?(value)
144
- FAIL("Expected array to include \"#{value}\"", nil, caller_locations(1, 1).first)
145
- end
146
- $test_runner.add_assertion()
147
-end
148
-def assert_array_not_include?(array, value)
149
- if array.include?(value)
150
- FAIL("Expected array to not include \"#{value}\"", nil, caller_locations(1, 1).first)
151
- end
152
- $test_runner.add_assertion()
153
-end
154
-def assert_is_one_of(value, *values)
155
- unless values.include?(value)
156
- FAIL("Expected value to be one of #{values.join(", ")}", nil, caller_locations(1, 1).first)
157
- end
158
- $test_runner.add_assertion()
159
-end
160
-def assert_not_nil(value)
161
- if value.nil?
162
- FAIL("Expected value to not be nil", nil, caller_locations(1, 1).first)
163
- end
164
- $test_runner.add_assertion()
165
-end
166
-def assert_nil(value)
167
- unless value.nil?
168
- FAIL("Expected value to be nil", nil, caller_locations(1, 1).first)
169
- end
170
- $test_runner.add_assertion()
171
-end
172
-
173
-
174
-class DynCfgHttpClient
175
- def self.protocol(cfg)
176
- return cfg[:ssl] ? 'https://' : 'http://'
177
- end
178
- def self.url_base(host)
179
- return "#{protocol(host)}#{host[:host]}:#{host[:port]}"
180
- end
181
- def self.get_url_cfg_base(host, child = nil)
182
- url = url_base(host)
183
- url += "/host/#{child[:mguid]}" if child
184
- url += "/api/v2/config"
185
- return url
186
- end
187
- def self.get_url_cfg_plugin(host, plugin, child = nil)
188
- return get_url_cfg_base(host, child) + '/' + plugin
189
- end
190
- def self.get_url_cfg_module(host, plugin, mod, child = nil)
191
- return get_url_cfg_plugin(host, plugin, child) + '/' + mod
192
- end
193
- def self.get_url_cfg_job(host, plugin, mod, job_id, child = nil)
194
- return get_url_cfg_module(host, plugin, mod, child) + "/#{job_id}"
195
- end
196
- def self.get_plugin_list(host, child = nil)
197
- begin
198
- return HTTParty.get(get_url_cfg_base(host, child), verify: false, format: :plain)
199
- rescue => e
200
- FAIL(e.message, e)
201
- end
202
- end
203
- def self.get_plugin_config(host, plugin, child = nil)
204
- begin
205
- return HTTParty.get(get_url_cfg_plugin(host, plugin, child), verify: false)
206
- rescue => e
207
- FAIL(e.message, e)
208
- end
209
- end
210
- def self.set_plugin_config(host, plugin, cfg, child = nil)
211
- begin
212
- return HTTParty.put(get_url_cfg_plugin(host, plugin, child), verify: false, body: cfg)
213
- rescue => e
214
- FAIL(e.message, e)
215
- end
216
- end
217
- def self.get_plugin_module_list(host, plugin, child = nil)
218
- begin
219
- return HTTParty.get(get_url_cfg_plugin(host, plugin, child) + "/modules", verify: false, format: :plain)
220
- rescue => e
221
- FAIL(e.message, e)
222
- end
223
- end
224
- def self.get_job_list(host, plugin, mod, child = nil)
225
- begin
226
- return HTTParty.get(get_url_cfg_module(host, plugin, mod, child) + "/jobs", verify: false, format: :plain)
227
- rescue => e
228
- FAIL(e.message, e)
229
- end
230
- end
231
- def self.create_job(host, plugin, mod, job_id, job_cfg, child = nil)
232
- begin
233
- return HTTParty.post(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, body: job_cfg)
234
- rescue => e
235
- FAIL(e.message, e)
236
- end
237
- end
238
- def self.delete_job(host, plugin, mod, job_id, child = nil)
239
- begin
240
- return HTTParty.delete(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false)
241
- rescue => e
242
- FAIL(e.message, e)
243
- end
244
- end
245
- def self.get_job_config(host, plugin, mod, job_id, child = nil)
246
- begin
247
- return HTTParty.get(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, format: :plain)
248
- rescue => e
249
- FAIL(e.message, e)
250
- end
251
- end
252
- def self.set_job_config(host, plugin, mod, job_id, job_cfg, child = nil)
253
- begin
254
- return HTTParty.put(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, body: job_cfg)
255
- rescue => e
256
- FAIL(e.message, e)
257
- end
258
- end
259
-end
260
-
261
-require_relative 'sub_tests/test_parent_child.rb'
262
-
263
-$test_runner.FINALIZE()
264
-puts $pastel.green.bold("All tests passed!")
265
-puts ("Total #{$test_runner.stats[:assertions]} assertions, #{$test_runner.stats[:tests]} tests in #{$test_runner.stats[:suites]} suites")
266
-exit 0
libnetdata/dyn_conf/tests/test_plugin/test.plugin
deleted
-250
@@ -1,250 +0,0 @@
1
-#!/usr/bin/env ruby
2
-
3
-# bogus chart that we create just so there is at least one chart
4
-CHART_TYPE = 'lines'
5
-UPDATE_EVERY = 1
6
-PRIORITY = 100000
7
-CHART_NAME = 'number_of_processes'
8
-DIMENSION_NAME = 'running'
9
-
10
-$plugin_name = "external_plugin"
11
-$plugin_version = "0.0.1"
12
-$plugin_config = <<-HEREDOC
13
-test_plugin_config
14
-hableba hableba hableba
15
-HEREDOC
16
-
17
-$array_module_name = 'module_of_the_future'
18
-$fixed_job_name = 'fixed_job'
19
-
20
-$modules = {
21
- $array_module_name => {
22
- :type => :job_array,
23
- :jobs => {
24
- $fixed_job_name => {
25
- :type => :fixed,
26
- :config => <<-HEREDOC
27
-fixed_job_config
28
-HEREDOC
29
- },
30
- },
31
- :config => <<-HEREDOC
32
-module_of_the_future_config
33
-HEREDOC
34
- },
35
- "module_of_the_future_single_type" => {
36
- :type => :single,
37
- :jobs => {},
38
- :config => <<-HEREDOC
39
-module_of_the_future_single_type_config
40
-HEREDOC
41
- }
42
-}
43
-
44
-def out(str)
45
- $log.puts "2 NETDATA> #{str}"
46
- $stdout.puts str
47
- $stdout.flush
48
- $log.flush
49
-end
50
-
51
-def log(str)
52
- $log.puts "LOG > #{str}"
53
- $log.flush
54
-end
55
-
56
-#TODO this is AI code, verify
57
-def split_with_quotes(str)
58
- result = []
59
- current_word = ""
60
- in_quotes = false
61
- escaped = false
62
-
63
- str.each_char do |char|
64
- if char == '\\' && !escaped
65
- escaped = true
66
- next
67
- end
68
-
69
- if char == '"' && !escaped
70
- in_quotes = !in_quotes
71
- current_word << char
72
- elsif char == ' ' && !in_quotes
73
- result << current_word unless current_word.empty?
74
- current_word = ""
75
- else
76
- current_word << char
77
- end
78
-
79
- escaped = false
80
- end
81
-
82
- result << current_word unless current_word.empty?
83
-
84
- result
85
-end
86
-
87
-
88
-def print_startup_messages
89
- out "DYNCFG_ENABLE #{$plugin_name}"
90
- $modules.each do |name, module_config|
91
- out "DYNCFG_REGISTER_MODULE #{name} #{module_config[:type]}"
92
- end
93
- out "CHART system.#{CHART_NAME} '' 'Number of running processes' 'processes' processes processes.#{CHART_NAME} #{CHART_TYPE} #{PRIORITY} #{UPDATE_EVERY}"
94
- out "DIMENSION #{DIMENSION_NAME} '' absolute 1 1"
95
-
96
- $modules.each do |mod_name, mod|
97
- next unless mod[:type] == :job_array
98
- mod[:jobs].each do |job_name, job|
99
- next unless job[:type] == :fixed
100
- out "DYNCFG_REGISTER_JOB #{mod_name} #{job_name} stock 0"
101
- out "REPORT_JOB_STATUS #{$array_module_name} #{$fixed_job_name} running 0"
102
- end
103
- end
104
-end
105
-
106
-def function_result(txid, msg, result)
107
- out "FUNCTION_RESULT_BEGIN #{txid} #{result} text/plain 5"
108
- out msg
109
- out "FUNCTION_RESULT_END"
110
-end
111
-
112
-def process_payload_function(params)
113
- log "payload function #{params[:fncname]}, #{params[:fncparams]}"
114
- fnc_name, mod_name, job_name = params[:fncparams]
115
- case fnc_name
116
- when 'set_plugin_config'
117
- $plugin_config = params[:payload]
118
- function_result(params[:txid], "plugin config set", 1)
119
- when 'set_module_config'
120
- mod = $modules[mod_name]
121
- return function_result(params[:txid], "no such module", 0) if mod.nil?
122
- mod[:config] = params[:payload]
123
- function_result(params[:txid], "module config set", 1)
124
- when 'set_job_config'
125
- mod = $modules[mod_name]
126
- return function_result(params[:txid], "no such module", 0) if mod.nil?
127
- job = mod[:jobs][job_name]
128
- if job.nil?
129
- job = Hash.new if job.nil?
130
- job[:type] = :dynamic
131
- mod[:jobs][job_name] = job
132
- end
133
- job[:config] = params[:payload]
134
- function_result(params[:txid], "job config set", 1)
135
- end
136
-end
137
-
138
-def process_function(params)
139
- log "normal function #{params[:fncname]}, #{params[:fncparams]}"
140
- fnc_name, mod_name, job_name = params[:fncparams]
141
- case fnc_name
142
- when 'get_plugin_config'
143
- function_result(params[:txid], $plugin_config, 1)
144
- when 'get_module_config'
145
- return function_result(params[:txid], "no such module", 0) unless $modules.has_key?(mod_name)
146
- function_result(params[:txid], $modules[mod_name][:config], 1)
147
- when 'get_job_config'
148
- mod = $modules[mod_name]
149
- return function_result(params[:txid], "no such module", 0) if mod.nil?
150
- job = mod[:jobs][job_name]
151
- return function_result(params[:txid], "no such job", 0) if job.nil?
152
- function_result(params[:txid], job[:config], 1)
153
- when 'delete_job'
154
- mod = $modules[mod_name]
155
- return function_result(params[:txid], "no such module", 0) if mod.nil?
156
- job = mod[:jobs][job_name]
157
- return function_result(params[:txid], "no such job", 0) if job.nil?
158
- if job[:type] == :fixed
159
- return function_result(params[:txid], "this job can't be deleted", 0)
160
- else
161
- mod[:jobs].delete(job_name)
162
- function_result(params[:txid], "job deleted", 1)
163
- end
164
- end
165
-end
166
-
167
-$inflight_incoming = nil
168
-def process_input(input)
169
- words = split_with_quotes(input)
170
-
171
- unless $inflight_incoming.nil?
172
- if input == "FUNCTION_PAYLOAD_END"
173
- log $inflight_incoming[:payload]
174
- process_payload_function($inflight_incoming)
175
- $inflight_incoming = nil
176
- else
177
- $inflight_incoming[:payload] << input
178
- $inflight_incoming[:payload] << "\n"
179
- end
180
- return
181
- end
182
-
183
- case words[0]
184
- when "FUNCTION", "FUNCTION_PAYLOAD"
185
- params = {}
186
- params[:command] = words[0]
187
- params[:txid] = words[1]
188
- params[:timeout] = words[2].to_i
189
- params[:fncname] = words[3]
190
- params[:fncname] = params[:fncname][1..-2] if params[:fncname].start_with?('"') && params[:fncname].end_with?('"')
191
- if params[:command] == "FUNCTION_PAYLOAD"
192
- $inflight_incoming = Hash.new
193
- params[:fncparams] = split_with_quotes(params[:fncname])
194
- params[:fncname] = params[:fncparams][0]
195
- $inflight_incoming[:txid] = params[:txid]
196
- $inflight_incoming[:fncname] = params[:fncname]
197
- $inflight_incoming[:params] = params
198
- $inflight_incoming[:fncparams] = params[:fncparams]
199
- $inflight_incoming[:payload] = ""
200
- else
201
- params[:fncparams] = split_with_quotes(params[:fncname])
202
- params[:fncname] = params[:fncparams][0]
203
- process_function(params)
204
- end
205
- end
206
-end
207
-
208
-def read_and_output_metric
209
- processes = `ps -e | wc -l`.to_i - 1 # -1 to exclude the header line
210
- timestamp = Time.now.to_i
211
-
212
- puts "BEGIN system.#{CHART_NAME}"
213
- puts "SET #{DIMENSION_NAME} = #{processes}"
214
- puts "END"
215
-end
216
-
217
-def the_main
218
- $stderr.reopen("/tmp/test_plugin_err.log", "w")
219
- $log = File.open("/tmp/test_plugin.log", "w")
220
- $log.puts "Starting plugin"
221
- print_startup_messages
222
- $log.puts "init done"
223
- $log.flush
224
-
225
- last_metric_time = Time.now
226
-
227
- loop do
228
- time_since_last_metric = Time.now - last_metric_time
229
-
230
- # If it's been more than 1 second since we collected metrics, collect them now
231
- if time_since_last_metric >= 1
232
- read_and_output_metric
233
- last_metric_time = Time.now
234
- end
235
-
236
- # Use select to wait for input, but only wait up to the time remaining until we need to collect metrics again
237
- remaining_time = [1 - time_since_last_metric, 0].max
238
- if select([$stdin], nil, nil, remaining_time)
239
- input = $stdin.gets
240
- next if input.class != String
241
- input.chomp!
242
- $log.puts "RAW INPUT< #{input}"
243
- $log.flush
244
- process_input(input)
245
- end
246
- end
247
-end
248
-
249
-
250
-the_main if __FILE__ == $PROGRAM_NAME
libnetdata/functions_evloop/functions_evloop.c
+239
-64
@@ -2,7 +2,7 @@
2
3
#include "functions_evloop.h"
4
5
-#define MAX_FUNCTION_PARAMETERS 1024
5
+static void functions_evloop_config_cb(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled, BUFFER *payload, const char *source, void *data);
6
7
struct functions_evloop_worker_job {
8
bool used;
@@ -12,13 +12,26 @@ struct functions_evloop_worker_job {
12
char *cmd;
13
const char *transaction;
14
time_t timeout;
15
+
16
+ BUFFER *payload;
17
+ const char *source;
18
+
19
functions_evloop_worker_execute_t cb;
20
+ void *cb_data;
21
};
22
23
+static void worker_job_cleanup(struct functions_evloop_worker_job *j) {
24
+ freez((void *)j->cmd);
25
+ freez((void *)j->transaction);
26
+ freez((void *)j->source);
27
+ buffer_free(j->payload);
28
+}
29
+
30
struct rrd_functions_expectation {
31
const char *function;
32
size_t function_length;
33
functions_evloop_worker_execute_t cb;
34
+ void *cb_data;
35
time_t default_timeout;
36
struct rrd_functions_expectation *prev, *next;
37
};
@@ -37,6 +50,10 @@ struct functions_evloop_globals {
50
netdata_thread_t reader_thread;
51
netdata_thread_t *worker_threads;
52
53
+ struct {
54
+ DICTIONARY *nodes;
55
+ } dyncfg;
56
+
57
struct rrd_functions_expectation *expectations;
58
};
59
@@ -73,7 +90,7 @@ static void *rrd_functions_worker_globals_worker_main(void *arg) {
90
91
last_acquired = true;
92
j = dictionary_acquired_item_value(acquired);
76
- j->cb(j->transaction, j->cmd, &j->stop_monotonic_ut, &j->cancelled);
93
+ j->cb(j->transaction, j->cmd, &j->stop_monotonic_ut, &j->cancelled, j->payload, j->source, j->cb_data);
94
dictionary_del(wg->worker_queue, j->transaction);
95
dictionary_acquired_item_release(wg->worker_queue, acquired);
96
dictionary_garbage_collect(wg->worker_queue);
@@ -84,73 +101,145 @@ static void *rrd_functions_worker_globals_worker_main(void *arg) {
101
return NULL;
102
}
103
104
+static void worker_add_job(struct functions_evloop_globals *wg, const char *keyword, char *transaction, char *function, char *timeout_s, BUFFER *payload, const char *source) {
105
+ if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
106
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
107
+ keyword,
108
+ transaction?transaction:"(unset)",
109
+ timeout_s?timeout_s:"(unset)",
110
+ function?function:"(unset)");
111
+ }
112
+ else {
113
+ int timeout = str2i(timeout_s);
114
+
115
+ const char *msg = "No function with this name found";
116
+ bool found = false;
117
+ struct rrd_functions_expectation *we;
118
+ for(we = wg->expectations; we ;we = we->next) {
119
+ if(strncmp(function, we->function, we->function_length) == 0) {
120
+ if(timeout <= 0)
121
+ timeout = (int)we->default_timeout;
122
+
123
+ struct functions_evloop_worker_job t = {
124
+ .cmd = strdupz(function),
125
+ .transaction = strdupz(transaction),
126
+ .running = false,
127
+ .cancelled = false,
128
+ .timeout = timeout,
129
+ .stop_monotonic_ut = now_monotonic_usec() + (timeout * USEC_PER_SEC),
130
+ .used = false,
131
+ .payload = buffer_dup(payload),
132
+ .source = source ? strdupz(source) : NULL,
133
+ .cb = we->cb,
134
+ .cb_data = we->cb_data,
135
+ };
136
+ struct functions_evloop_worker_job *j = dictionary_set(wg->worker_queue, transaction, &t, sizeof(t));
137
+ if(j->used) {
138
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Received duplicate function transaction '%s'. Ignoring it.", transaction);
139
+ worker_job_cleanup(&t);
140
+ msg = "Duplicate function transaction. Ignoring it.";
141
+ }
142
+ else {
143
+ found = true;
144
+ j->used = true;
145
+ pthread_cond_signal(&wg->worker_cond_var);
146
+ }
147
+ }
148
+ }
149
+
150
+ if(!found) {
151
+ netdata_mutex_lock(wg->stdout_mutex);
152
+ pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, msg);
153
+ netdata_mutex_unlock(wg->stdout_mutex);
154
+ }
155
+ }
156
+}
157
+
158
static void *rrd_functions_worker_globals_reader_main(void *arg) {
159
struct functions_evloop_globals *wg = arg;
160
90
- char buffer[PLUGINSD_LINE_MAX + 1];
161
+ struct {
162
+ size_t last_len; // to remember the last pos - do not use a pointer, the buffer may realloc...
163
+ bool enabled;
164
+ char *transaction;
165
+ char *function;
166
+ char *timeout_s;
167
+ char *source;
168
+ char *content_type;
169
+ } deferred = { 0 };
170
+
171
+ struct buffered_reader reader = { 0 };
172
+ buffered_reader_init(&reader);
173
+ BUFFER *buffer = buffer_create(sizeof(reader.read_buffer) + 2, NULL);
174
+
175
+ while(!(*wg->plugin_should_exit)) {
176
+ if(unlikely(!buffered_reader_next_line(&reader, buffer))) {
177
+ buffered_reader_ret_t ret = buffered_reader_read_timeout(
178
+ &reader,
179
+ fileno((FILE *)stdin),
180
+ 2 * 60 * MSEC_PER_SEC,
181
+ false
182
+ );
183
+
184
+ if(unlikely(ret != BUFFERED_READER_READ_OK && ret != BUFFERED_READER_READ_POLL_TIMEOUT))
185
+ break;
186
+
187
+ continue;
188
+ }
189
+
190
+ if(deferred.enabled) {
191
+ char *s = (char *)buffer_tostring(buffer);
192
+
193
+ if(strstr(&s[deferred.last_len], PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END "\n") != NULL) {
194
+ if(deferred.last_len > 0)
195
+ // remove the trailing newline from the buffer
196
+ deferred.last_len--;
197
+
198
+ s[deferred.last_len] = '\0';
199
+ buffer->len = deferred.last_len;
200
+ buffer->content_type = content_type_string2id(deferred.content_type);
201
+ worker_add_job(wg, PLUGINSD_KEYWORD_FUNCTION_PAYLOAD, deferred.transaction, deferred.function, deferred.timeout_s, buffer, deferred.source);
202
+ buffer_flush(buffer);
203
+
204
+ freez(deferred.transaction);
205
+ freez(deferred.function);
206
+ freez(deferred.timeout_s);
207
+ freez(deferred.source);
208
+ freez(deferred.content_type);
209
+ memset(&deferred, 0, sizeof(deferred));
210
+ }
211
+ else
212
+ deferred.last_len = buffer->len;
213
92
- char *s = NULL;
93
- while(!(*wg->plugin_should_exit) && (s = fgets(buffer, PLUGINSD_LINE_MAX, stdin))) {
214
+ continue;
215
+ }
216
217
char *words[MAX_FUNCTION_PARAMETERS] = { NULL };
96
- size_t num_words = quoted_strings_splitter_pluginsd(buffer, words, MAX_FUNCTION_PARAMETERS);
218
+ size_t num_words = quoted_strings_splitter_pluginsd((char *)buffer_tostring(buffer), words, MAX_FUNCTION_PARAMETERS);
219
220
const char *keyword = get_word(words, num_words, 0);
221
100
- if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0) {
222
+ if(keyword && (strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0)) {
223
char *transaction = get_word(words, num_words, 1);
224
char *timeout_s = get_word(words, num_words, 2);
225
char *function = get_word(words, num_words, 3);
104
-
105
- if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
106
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
107
- keyword,
108
- transaction?transaction:"(unset)",
109
- timeout_s?timeout_s:"(unset)",
110
- function?function:"(unset)");
111
- }
112
- else {
113
- int timeout = str2i(timeout_s);
114
-
115
- const char *msg = "No function with this name found";
116
- bool found = false;
117
- struct rrd_functions_expectation *we;
118
- for(we = wg->expectations; we ;we = we->next) {
119
- if(strncmp(function, we->function, we->function_length) == 0) {
120
- if(timeout <= 0)
121
- timeout = (int)we->default_timeout;
122
-
123
- struct functions_evloop_worker_job t = {
124
- .cmd = strdupz(function),
125
- .transaction = strdupz(transaction),
126
- .running = false,
127
- .cancelled = false,
128
- .timeout = timeout,
129
- .stop_monotonic_ut = now_monotonic_usec() + (timeout * USEC_PER_SEC),
130
- .used = false,
131
- .cb = we->cb,
132
- };
133
- struct functions_evloop_worker_job *j = dictionary_set(wg->worker_queue, transaction, &t, sizeof(t));
134
- if(j->used) {
135
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Received duplicate function transaction '%s'. Ignoring it.", transaction);
136
- freez((void *)t.cmd);
137
- freez((void *)t.transaction);
138
- msg = "Duplicate function transaction. Ignoring it.";
139
- }
140
- else {
141
- found = true;
142
- j->used = true;
143
- pthread_cond_signal(&wg->worker_cond_var);
144
- }
145
- }
146
- }
147
-
148
- if(!found) {
149
- netdata_mutex_lock(wg->stdout_mutex);
150
- pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, msg);
151
- netdata_mutex_unlock(wg->stdout_mutex);
152
- }
153
- }
226
+ char *source = get_word(words, num_words, 4);
227
+ worker_add_job(wg, keyword, transaction, function, timeout_s, NULL, source);
228
+ }
229
+ else if(keyword && (strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION_PAYLOAD) == 0)) {
230
+ char *transaction = get_word(words, num_words, 1);
231
+ char *timeout_s = get_word(words, num_words, 2);
232
+ char *function = get_word(words, num_words, 3);
233
+ char *source = get_word(words, num_words, 4);
234
+ char *content_type = get_word(words, num_words, 5);
235
+
236
+ deferred.transaction = strdupz(transaction ? transaction : "");
237
+ deferred.timeout_s = strdupz(timeout_s ? timeout_s : "");
238
+ deferred.function = strdupz(function ? function : "");
239
+ deferred.source = strdupz(source ? source : "");
240
+ deferred.content_type = strdupz(content_type ? content_type : "");
241
+ deferred.last_len = 0;
242
+ deferred.enabled = true;
243
}
244
else if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION_CANCEL) == 0) {
245
char *transaction = get_word(words, num_words, 1);
@@ -180,20 +269,20 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {
269
}
270
else
271
nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword?keyword:"(unset)");
183
- }
272
185
- if(!s || feof(stdin) || ferror(stdin)) {
186
- *wg->plugin_should_exit = true;
187
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "Received error on stdin.");
273
+ buffer_flush(buffer);
274
}
275
276
+ if(!(*wg->plugin_should_exit))
277
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Read error on stdin");
278
+
279
+ *wg->plugin_should_exit = true;
280
exit(1);
281
}
282
283
void worker_queue_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
284
struct functions_evloop_worker_job *j = value;
195
- freez((void *)j->cmd);
196
- freez((void *)j->transaction);
285
+ worker_job_cleanup(j);
286
}
287
288
struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, const char *tag, netdata_mutex_t *stdout_mutex, bool *plugin_should_exit) {
@@ -202,6 +291,8 @@ struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, co
291
wg->worker_queue = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
292
dictionary_register_delete_callback(wg->worker_queue, worker_queue_delete_cb, NULL);
293
294
+ wg->dyncfg.nodes = dyncfg_nodes_dictionary_create();
295
+
296
pthread_mutex_init(&wg->worker_mutex, NULL);
297
pthread_cond_init(&wg->worker_cond_var, NULL);
298
@@ -222,14 +313,17 @@ struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, co
313
rrd_functions_worker_globals_worker_main, wg);
314
}
315
316
+ functions_evloop_add_function(wg, "config", functions_evloop_config_cb, 120, wg);
317
+
318
return wg;
319
}
320
228
-void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout) {
321
+void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout, void *data) {
322
struct rrd_functions_expectation *we = callocz(1, sizeof(*we));
323
we->function = function;
324
we->function_length = strlen(we->function);
325
we->cb = cb;
326
+ we->cb_data = data;
327
we->default_timeout = default_timeout;
328
DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wg->expectations, we, prev, next);
329
}
@@ -240,3 +334,84 @@ void functions_evloop_cancel_threads(struct functions_evloop_globals *wg){
334
335
netdata_thread_cancel(wg->reader_thread);
336
}
337
+
338
+// ----------------------------------------------------------------------------
339
+
340
+static void functions_evloop_config_cb(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled,
341
+ BUFFER *payload, const char *source, void *data) {
342
+ struct functions_evloop_globals *wg = data;
343
+
344
+ CLEAN_BUFFER *result = buffer_create(1024, NULL);
345
+ int code = dyncfg_node_find_and_call(wg->dyncfg.nodes, transaction, function, stop_monotonic_ut, cancelled, payload, source, result);
346
+
347
+ netdata_mutex_lock(wg->stdout_mutex);
348
+ pluginsd_function_result_begin_to_stdout(transaction, code, content_type_id2string(result->content_type), result->expires);
349
+ printf("%s", buffer_tostring(result));
350
+ pluginsd_function_result_end_to_stdout();
351
+ fflush(stdout);
352
+ netdata_mutex_unlock(wg->stdout_mutex);
353
+}
354
+
355
+void functions_evloop_dyncfg_add(struct functions_evloop_globals *wg, const char *id, const char *path, DYNCFG_STATUS status, DYNCFG_TYPE type, DYNCFG_SOURCE_TYPE source_type, const char *source, DYNCFG_CMDS cmds, dyncfg_cb_t cb, void *data) {
356
+ if(!dyncfg_is_valid_id(id)) {
357
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "DYNCFG: id '%s' is invalid. Ignoring dynamic configuration for it.", id);
358
+ return;
359
+ }
360
+
361
+ struct dyncfg_node tmp = {
362
+ .cmds = cmds,
363
+ .type = type,
364
+ .cb = cb,
365
+ .data = data,
366
+ };
367
+ dictionary_set(wg->dyncfg.nodes, id, &tmp, sizeof(tmp));
368
+
369
+ CLEAN_BUFFER *c = buffer_create(100, NULL);
370
+ dyncfg_cmds2buffer(cmds, c);
371
+
372
+ netdata_mutex_lock(wg->stdout_mutex);
373
+
374
+ fprintf(stdout,
375
+ PLUGINSD_KEYWORD_CONFIG " '%s' " PLUGINSD_KEYWORD_CONFIG_ACTION_CREATE " '%s' '%s' '%s' '%s' '%s' '%s'\n",
376
+ id, dyncfg_id2status(status), dyncfg_id2type(type), path,
377
+ dyncfg_id2source_type(source_type), source, buffer_tostring(c)
378
+ );
379
+ fflush(stdout);
380
+
381
+ netdata_mutex_unlock(wg->stdout_mutex);
382
+}
383
+
384
+void functions_evloop_dyncfg_del(struct functions_evloop_globals *wg, const char *id) {
385
+ if(!dyncfg_is_valid_id(id)) {
386
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "DYNCFG: id '%s' is invalid. Ignoring dynamic configuration for it.", id);
387
+ return;
388
+ }
389
+
390
+ dictionary_del(wg->dyncfg.nodes, id);
391
+
392
+ netdata_mutex_lock(wg->stdout_mutex);
393
+
394
+ fprintf(stdout,
395
+ PLUGINSD_KEYWORD_CONFIG " %s " PLUGINSD_KEYWORD_CONFIG_ACTION_DELETE "\n",
396
+ id);
397
+ fflush(stdout);
398
+
399
+ netdata_mutex_unlock(wg->stdout_mutex);
400
+}
401
+
402
+void functions_evloop_dyncfg_status(struct functions_evloop_globals *wg, const char *id, DYNCFG_STATUS status) {
403
+ if(!dyncfg_is_valid_id(id)) {
404
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "DYNCFG: id '%s' is invalid. Ignoring dynamic configuration for it.", id);
405
+ return;
406
+ }
407
+
408
+ netdata_mutex_lock(wg->stdout_mutex);
409
+
410
+ fprintf(stdout,
411
+ PLUGINSD_KEYWORD_CONFIG " %s " PLUGINSD_KEYWORD_CONFIG_ACTION_STATUS " %s\n",
412
+ id, dyncfg_id2status(status));
413
+
414
+ fflush(stdout);
415
+
416
+ netdata_mutex_unlock(wg->stdout_mutex);
417
+}
libnetdata/functions_evloop/functions_evloop.h
+15
-7
@@ -5,6 +5,8 @@
5
6
#include "../libnetdata.h"
7
8
+#define MAX_FUNCTION_PARAMETERS 1024
9
+
10
#define PLUGINSD_KEYWORD_CHART "CHART"
11
#define PLUGINSD_KEYWORD_CHART_DEFINITION_END "CHART_DEFINITION_END"
12
@@ -28,6 +30,13 @@
30
#define PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN "FUNCTION_RESULT_BEGIN"
31
#define PLUGINSD_KEYWORD_FUNCTION_RESULT_END "FUNCTION_RESULT_END"
32
33
+#define PLUGINSD_KEYWORD_CONFIG "CONFIG"
34
+#define PLUGINSD_KEYWORD_CONFIG_ACTION_CREATE "create"
35
+#define PLUGINSD_KEYWORD_CONFIG_ACTION_DELETE "delete"
36
+#define PLUGINSD_KEYWORD_CONFIG_ACTION_STATUS "status"
37
+
38
+#define PLUGINSD_FUNCTION_CONFIG "config"
39
+
40
#define PLUGINSD_KEYWORD_REPLAY_CHART "REPLAY_CHART"
41
#define PLUGINSD_KEYWORD_REPLAY_BEGIN "RBEGIN"
42
#define PLUGINSD_KEYWORD_REPLAY_SET "RSET"
@@ -44,21 +53,16 @@
53
#define PLUGINSD_KEYWORD_HOST_LABEL "HOST_LABEL"
54
#define PLUGINSD_KEYWORD_HOST "HOST"
55
47
-#define PLUGINSD_KEYWORD_DYNCFG_ENABLE "DYNCFG_ENABLE"
48
-#define PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE "DYNCFG_REGISTER_MODULE"
49
-
50
-#define PLUGINSD_KEYWORD_REPORT_JOB_STATUS "REPORT_JOB_STATUS"
51
-
56
#define PLUGINSD_KEYWORD_EXIT "EXIT"
57
58
#define PLUGINSD_KEYWORD_SLOT "SLOT" // to change the length of this, update pluginsd_extract_chart_slot() too
59
60
#define PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT 10 // seconds
61
58
-typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled);
62
+typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled, BUFFER *payload, const char *source, void *data);
63
struct functions_evloop_worker_job;
64
struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, const char *tag, netdata_mutex_t *stdout_mutex, bool *plugin_should_exit);
61
-void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout);
65
+void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout, void *data);
66
void functions_evloop_cancel_threads(struct functions_evloop_globals *wg);
67
68
#define FUNCTIONS_EXTENDED_TIME_ON_PROGRESS_UT (10 * USEC_PER_SEC)
@@ -119,4 +123,8 @@ static inline void pluginsd_function_progress_to_stdout(const char *transaction,
123
fflush(stdout);
124
}
125
126
+void functions_evloop_dyncfg_add(struct functions_evloop_globals *wg, const char *id, const char *path, DYNCFG_STATUS status, DYNCFG_TYPE type, DYNCFG_SOURCE_TYPE source_type, const char *source, DYNCFG_CMDS cmds, dyncfg_cb_t cb, void *data);
127
+void functions_evloop_dyncfg_del(struct functions_evloop_globals *wg, const char *id);
128
+void functions_evloop_dyncfg_status(struct functions_evloop_globals *wg, const char *id, DYNCFG_STATUS status);
129
+
130
#endif //NETDATA_FUNCTIONS_EVLOOP_H
libnetdata/http/content_type.c
new
+96
@@ -0,0 +1,96 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "content_type.h"
4
+
5
+
6
+static struct {
7
+ const char *format;
8
+ HTTP_CONTENT_TYPE content_type;
9
+ bool needs_charset;
10
+ const char *options;
11
+} content_types[] = {
12
+ // primary - preferred during id-to-string conversions
13
+ { .format = "text/html", CT_TEXT_HTML, true },
14
+ { .format = "text/plain", CT_TEXT_PLAIN, true },
15
+ { .format = "text/css", CT_TEXT_CSS, true },
16
+ { .format = "text/yaml", CT_TEXT_YAML, true },
17
+ { .format = "text/xml", CT_TEXT_XML, true },
18
+ { .format = "text/xsl", CT_TEXT_XSL, true },
19
+ { .format = "application/json", CT_APPLICATION_JSON, true },
20
+ { .format = "application/xml", CT_APPLICATION_XML, true },
21
+ { .format = "application/javascript", CT_APPLICATION_X_JAVASCRIPT, true },
22
+ { .format = "application/octet-stream", CT_APPLICATION_OCTET_STREAM, false },
23
+ { .format = "image/svg+xml", CT_IMAGE_SVG_XML, false },
24
+ { .format = "application/x-font-truetype", CT_APPLICATION_X_FONT_TRUETYPE, false },
25
+ { .format = "application/x-font-opentype", CT_APPLICATION_X_FONT_OPENTYPE, false },
26
+ { .format = "application/font-woff", CT_APPLICATION_FONT_WOFF, false },
27
+ { .format = "application/font-woff2", CT_APPLICATION_FONT_WOFF2, false },
28
+ { .format = "application/vnd.ms-fontobject",CT_APPLICATION_VND_MS_FONTOBJ, false },
29
+ { .format = "image/png", CT_IMAGE_PNG, false },
30
+ { .format = "image/jpeg", CT_IMAGE_JPG, false },
31
+ { .format = "image/gif", CT_IMAGE_GIF, false },
32
+ { .format = "image/x-icon", CT_IMAGE_XICON, false },
33
+ { .format = "image/bmp", CT_IMAGE_BMP, false },
34
+ { .format = "image/icns", CT_IMAGE_ICNS, false },
35
+ { .format = "audio/mpeg", CT_AUDIO_MPEG, false },
36
+ { .format = "audio/ogg", CT_AUDIO_OGG, false },
37
+ { .format = "video/mp4", CT_VIDEO_MP4, false },
38
+ { .format = "application/pdf", CT_APPLICATION_PDF, false },
39
+ { .format = "application/zip", CT_APPLICATION_ZIP, false },
40
+ { .format = "image/png", CT_IMAGE_PNG, false },
41
+
42
+ // secondary - overlapping with primary
43
+
44
+ { .format = "text/plain", CT_PROMETHEUS, false, "version=0.0.4" },
45
+ { .format = "prometheus", CT_PROMETHEUS },
46
+ { .format = "text", CT_TEXT_PLAIN },
47
+ { .format = "txt", CT_TEXT_PLAIN },
48
+ { .format = "json", CT_APPLICATION_JSON },
49
+ { .format = "html", CT_TEXT_HTML },
50
+ { .format = "xml", CT_APPLICATION_XML },
51
+
52
+ // terminator
53
+ { .format = NULL, CT_TEXT_PLAIN },
54
+};
55
+
56
+HTTP_CONTENT_TYPE content_type_string2id(const char *format) {
57
+ if(format && *format) {
58
+ for (int i = 0; content_types[i].format; i++)
59
+ if (strcmp(content_types[i].format, format) == 0)
60
+ return content_types[i].content_type;
61
+ }
62
+
63
+ return CT_TEXT_PLAIN;
64
+}
65
+
66
+const char *content_type_id2string(HTTP_CONTENT_TYPE content_type) {
67
+ for (int i = 0; content_types[i].format; i++)
68
+ if (content_types[i].content_type == content_type)
69
+ return content_types[i].format;
70
+
71
+ return "text/plain";
72
+}
73
+
74
+void http_header_content_type(BUFFER *wb, HTTP_CONTENT_TYPE content_type) {
75
+ buffer_strcat(wb, "Content-Type: ");
76
+
77
+ for (int i = 0; content_types[i].format; i++) {
78
+ if (content_types[i].content_type == content_type) {
79
+ buffer_strcat(wb, content_types[i].format);
80
+
81
+ if(content_types[i].needs_charset) {
82
+ buffer_strcat(wb, "; charset=utf-8");
83
+ }
84
+ if(content_types[i].options) {
85
+ buffer_strcat(wb, "; ");
86
+ buffer_strcat(wb, content_types[i].options);
87
+ }
88
+
89
+ buffer_strcat(wb, "\r\n");
90
+
91
+ return;
92
+ }
93
+ }
94
+
95
+ buffer_strcat(wb, "text/plain; charset=utf-8\r\n");
96
+}
libnetdata/http/content_type.h
new
+45
@@ -0,0 +1,45 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_CONTENT_TYPE_H
4
+#define NETDATA_CONTENT_TYPE_H
5
+
6
+typedef enum __attribute__ ((__packed__)) {
7
+ CT_NONE = 0,
8
+ CT_APPLICATION_JSON,
9
+ CT_TEXT_PLAIN,
10
+ CT_TEXT_HTML,
11
+ CT_APPLICATION_X_JAVASCRIPT,
12
+ CT_TEXT_CSS,
13
+ CT_TEXT_XML,
14
+ CT_APPLICATION_XML,
15
+ CT_TEXT_XSL,
16
+ CT_APPLICATION_OCTET_STREAM,
17
+ CT_APPLICATION_X_FONT_TRUETYPE,
18
+ CT_APPLICATION_X_FONT_OPENTYPE,
19
+ CT_APPLICATION_FONT_WOFF,
20
+ CT_APPLICATION_FONT_WOFF2,
21
+ CT_APPLICATION_VND_MS_FONTOBJ,
22
+ CT_IMAGE_SVG_XML,
23
+ CT_IMAGE_PNG,
24
+ CT_IMAGE_JPG,
25
+ CT_IMAGE_GIF,
26
+ CT_IMAGE_XICON,
27
+ CT_IMAGE_ICNS,
28
+ CT_IMAGE_BMP,
29
+ CT_PROMETHEUS,
30
+ CT_AUDIO_MPEG,
31
+ CT_AUDIO_OGG,
32
+ CT_VIDEO_MP4,
33
+ CT_APPLICATION_PDF,
34
+ CT_APPLICATION_ZIP,
35
+ CT_TEXT_YAML,
36
+} HTTP_CONTENT_TYPE;
37
+
38
+HTTP_CONTENT_TYPE content_type_string2id(const char *format);
39
+const char *content_type_id2string(HTTP_CONTENT_TYPE content_type);
40
+
41
+#include "../libnetdata.h"
42
+
43
+void http_header_content_type(struct web_buffer *wb, HTTP_CONTENT_TYPE type);
44
+
45
+#endif //NETDATA_CONTENT_TYPE_H
libnetdata/http/http_access.c
+21
-16
@@ -5,34 +5,39 @@
5
static struct {
6
HTTP_ACCESS access;
7
const char *name;
8
-} rrd_function_access_levels[] = {
9
- { .access = HTTP_ACCESS_NONE, .name = "none" },
10
- { .access = HTTP_ACCESS_MEMBERS, .name = "members" },
11
- { .access = HTTP_ACCESS_ADMINS, .name = "admins" },
12
- { .access = HTTP_ACCESS_ANY, .name = "any" },
8
+} access_levels[] = {
9
+ { .access = HTTP_ACCESS_NONE, .name = "none" },
10
+ { .access = HTTP_ACCESS_MEMBER, .name = "member" },
11
+ { .access = HTTP_ACCESS_ADMIN, .name = "admin" },
12
+ { .access = HTTP_ACCESS_ANY, .name = "any" },
13
+
14
+ { .access = HTTP_ACCESS_MEMBER, .name = "members" },
15
+ { .access = HTTP_ACCESS_ADMIN, .name = "admins" },
16
+ { .access = HTTP_ACCESS_ANY, .name = "all" },
17
+
18
+ // terminator
19
+ { .access = 0, .name = NULL },
20
};
21
22
HTTP_ACCESS http_access2id(const char *access) {
23
if(!access || !*access)
17
- return HTTP_ACCESS_MEMBERS;
24
+ return HTTP_ACCESS_MEMBER;
25
19
- size_t entries = sizeof(rrd_function_access_levels) / sizeof(rrd_function_access_levels[0]);
20
- for(size_t i = 0; i < entries ;i++) {
21
- if(strcmp(rrd_function_access_levels[i].name, access) == 0)
22
- return rrd_function_access_levels[i].access;
26
+ for(size_t i = 0; access_levels[i].name ;i++) {
27
+ if(strcmp(access_levels[i].name, access) == 0)
28
+ return access_levels[i].access;
29
}
30
31
nd_log(NDLS_DAEMON, NDLP_WARNING, "HTTP access level '%s' is not valid", access);
26
- return HTTP_ACCESS_MEMBERS;
32
+ return HTTP_ACCESS_NONE;
33
}
34
35
const char *http_id2access(HTTP_ACCESS access) {
30
- size_t entries = sizeof(rrd_function_access_levels) / sizeof(rrd_function_access_levels[0]);
31
- for(size_t i = 0; i < entries ;i++) {
32
- if(access == rrd_function_access_levels[i].access)
33
- return rrd_function_access_levels[i].name;
36
+ for(size_t i = 0; access_levels[i].name ;i++) {
37
+ if(access == access_levels[i].access)
38
+ return access_levels[i].name;
39
}
40
41
nd_log(NDLS_DAEMON, NDLP_WARNING, "HTTP access level %d is not valid", access);
37
- return "members";
42
+ return "none";
43
}
libnetdata/http/http_access.h
+2
-2
@@ -5,8 +5,8 @@
5
6
typedef enum __attribute__((packed)) {
7
HTTP_ACCESS_NONE = 0,
8
- HTTP_ACCESS_ADMINS = 1,
9
- HTTP_ACCESS_MEMBERS = 2,
8
+ HTTP_ACCESS_ADMIN = 1,
9
+ HTTP_ACCESS_MEMBER = 2,
10
HTTP_ACCESS_ANY = 3,
11
12
// keep this list so that lower numbers are more strict access levels
libnetdata/http/http_defs.c
+2
@@ -210,6 +210,8 @@ static struct {
210
, { "bmp" , 0 , CT_IMAGE_BMP }
211
, { "ico" , 0 , CT_IMAGE_XICON }
212
, { "icns" , 0 , CT_IMAGE_ICNS }
213
+
214
+ // terminator
215
, { NULL , 0 , 0 }
216
};
217
libnetdata/http/http_defs.h
+2
@@ -12,6 +12,7 @@
12
13
// HTTP_CODES 2XX Success
14
#define HTTP_RESP_OK 200
15
+#define HTTP_RESP_ACCEPTED 202
16
17
// HTTP_CODES 3XX Redirections
18
#define HTTP_RESP_MOVED_PERM 301
@@ -33,6 +34,7 @@
34
35
// HTTP_CODES 5XX Server Errors
36
#define HTTP_RESP_INTERNAL_SERVER_ERROR 500
37
+#define HTTP_RESP_NOT_IMPLEMENTED 501
38
#define HTTP_RESP_SERVICE_UNAVAILABLE 503
39
#define HTTP_RESP_GATEWAY_TIMEOUT 504
40
#define HTTP_RESP_BACKEND_RESPONSE_INVALID 591
libnetdata/inlined.h
+36
@@ -597,4 +597,40 @@ static inline char *trim_all(char *buffer) {
597
return buffer;
598
}
599
600
+static inline bool streq(const char *a, const char *b) {
601
+ if (a == b)
602
+ return true;
603
+
604
+ if (a == NULL || b == NULL)
605
+ return false;
606
+
607
+ return strcmp(a, b) == 0;
608
+}
609
+
610
+static inline bool strstartswith(const char *string, const char *prefix) {
611
+ if (string == NULL || prefix == NULL)
612
+ return false;
613
+
614
+ size_t string_len = strlen(string);
615
+ size_t prefix_len = strlen(prefix);
616
+
617
+ if (prefix_len > string_len)
618
+ return false;
619
+
620
+ return strncmp(string, prefix, prefix_len) == 0;
621
+}
622
+
623
+static inline bool strendswith(const char *string, const char *suffix) {
624
+ if (string == NULL || suffix == NULL)
625
+ return false;
626
+
627
+ size_t string_len = strlen(string);
628
+ size_t suffix_len = strlen(suffix);
629
+
630
+ if (suffix_len > string_len)
631
+ return false;
632
+
633
+ return strcmp(string + string_len - suffix_len, suffix) == 0;
634
+}
635
+
636
#endif //NETDATA_INLINED_H
libnetdata/json/json.h
+6
-3
@@ -1,7 +1,6 @@
1
#ifndef CHECKIN_JSON_H
2
#define CHECKIN_JSON_H 1
3
4
-
4
#if ENABLE_JSONC
5
#include <json-c/json.h>
6
// fix an older json-c bug
@@ -72,6 +71,10 @@ size_t json_walk_primitive(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e);
71
72
int json_callback_print(JSON_ENTRY *e);
73
74
+static inline void cleanup_json_object_pp(struct json_object **jobj) {
75
+ if(*jobj)
76
+ json_object_put(*jobj);
77
+}
78
+#define CLEAN_JSON_OBJECT _cleanup_(cleanup_json_object_pp) struct json_object
79
76
-
77
-#endif
\ No newline at end of file
80
+#endif // CHECKIN_JSON_H
libnetdata/libnetdata.c
+42
@@ -129,6 +129,9 @@ static void (*libc_free)(void *) = free_first_run;
129
static char *strdup_first_run(const char *s);
130
static char *(*libc_strdup)(const char *) = strdup_first_run;
131
132
+static char *strndup_first_run(const char *s, size_t len);
133
+static char *(*libc_strndup)(const char *, size_t) = strndup_first_run;
134
+
135
static size_t malloc_usable_size_first_run(void *ptr);
136
#ifdef HAVE_MALLOC_USABLE_SIZE
137
static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size_first_run;
@@ -169,6 +172,11 @@ static char *strdup_first_run(const char *s) {
172
return libc_strdup(s);
173
}
174
175
+static char *strndup_first_run(const char *s, size_t len) {
176
+ link_system_library_function((libc_function_t *) &libc_strndup, "strndup", true);
177
+ return libc_strndup(s, len);
178
+}
179
+
180
static size_t malloc_usable_size_first_run(void *ptr) {
181
link_system_library_function((libc_function_t *) &libc_malloc_usable_size, "malloc_usable_size", false);
182
@@ -202,6 +210,10 @@ char *strdup(const char *s) {
210
return strdupz(s);
211
}
212
213
+char *strndup(const char *s, size_t len) {
214
+ return strndupz(s, len);
215
+}
216
+
217
size_t malloc_usable_size(void *ptr) {
218
return mallocz_usable_size(ptr);
219
}
@@ -365,6 +377,30 @@ char *strdupz_int(const char *s, const char *file, const char *function, size_t
377
return (char *)&t->data;
378
}
379
380
+char *strndupz_int(const char *s, size_t len, const char *file, const char *function, size_t line) {
381
+ struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
382
+ size_t size = len + 1;
383
+
384
+ size_t_atomic_count(add, p->strdup_calls, 1);
385
+ size_t_atomic_count(add, p->allocations, 1);
386
+ size_t_atomic_bytes(add, p->bytes, size);
387
+
388
+ struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);
389
+ if (unlikely(!t)) fatal("strndupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
390
+ t->signature.magic = 0x0BADCAFE;
391
+ t->signature.trace = p;
392
+ t->signature.size = size;
393
+
394
+#ifdef NETDATA_INTERNAL_CHECKS
395
+ for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
396
+ t->padding[i] = 0xFF;
397
+#endif
398
+
399
+ memcpy(&t->data, s, size);
400
+ t->data[len] = '\0';
401
+ return (char *)&t->data;
402
+}
403
+
404
static struct malloc_header *malloc_get_header(void *ptr, const char *caller, const char *file, const char *function, size_t line) {
405
uint8_t *ret = (uint8_t *)ptr - malloc_header_size;
406
struct malloc_header *t = (struct malloc_header *)ret;
@@ -450,6 +486,12 @@ char *strdupz(const char *s) {
486
return t;
487
}
488
489
+char *strndupz(const char *s, size_t len) {
490
+ char *t = strndup(s, len);
491
+ if (unlikely(!t)) fatal("Cannot strndup() string '%s' of len %zu", s, len);
492
+ return t;
493
+}
494
+
495
// If ptr is NULL, no operation is performed.
496
void freez(void *ptr) {
497
free(ptr);
libnetdata/libnetdata.h
+21
-1
@@ -242,6 +242,11 @@ size_t judy_aral_structures(void);
242
#define ABS(x) (((x) < 0)? (-(x)) : (x))
243
#define MIN(a,b) (((a)<(b))?(a):(b))
244
#define MAX(a,b) (((a)>(b))?(a):(b))
245
+#define SWAP(a, b) do { \
246
+ typeof(a) _tmp = b; \
247
+ b = a; \
248
+ a = _tmp; \
249
+} while(0)
250
251
#define GUID_LEN 36
252
@@ -515,6 +520,7 @@ int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4);
520
int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data);
521
522
#define strdupz(s) strdupz_int(s, __FILE__, __FUNCTION__, __LINE__)
523
+#define strndupz(s, len) strndupz_int(s, len, __FILE__, __FUNCTION__, __LINE__)
524
#define callocz(nmemb, size) callocz_int(nmemb, size, __FILE__, __FUNCTION__, __LINE__)
525
#define mallocz(size) mallocz_int(size, __FILE__, __FUNCTION__, __LINE__)
526
#define reallocz(ptr, size) reallocz_int(ptr, size, __FILE__, __FUNCTION__, __LINE__)
@@ -522,6 +528,7 @@ int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data
528
#define mallocz_usable_size(ptr) mallocz_usable_size_int(ptr, __FILE__, __FUNCTION__, __LINE__)
529
530
char *strdupz_int(const char *s, const char *file, const char *function, size_t line);
531
+char *strndupz_int(const char *s, size_t len, const char *file, const char *function, size_t line);
532
void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line);
533
void *mallocz_int(size_t size, const char *file, const char *function, size_t line);
534
void *reallocz_int(void *ptr, size_t size, const char *file, const char *function, size_t line);
@@ -530,6 +537,7 @@ size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function
537
538
#else // NETDATA_TRACE_ALLOCATIONS
539
char *strdupz(const char *s) MALLOCLIKE NEVERNULL;
540
+char *strndupz(const char *s, size_t len) MALLOCLIKE NEVERNULL;
541
void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;
542
void *mallocz(size_t size) MALLOCLIKE NEVERNULL;
543
void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL;
@@ -702,6 +710,8 @@ extern char *netdata_configured_host_prefix;
710
711
#include "uuid/uuid.h"
712
#include "http/http_access.h"
713
+#include "http/content_type.h"
714
+#include "config/dyncfg.h"
715
#include "libjudy/src/Judy.h"
716
#include "july/july.h"
717
#include "os.h"
@@ -747,7 +757,6 @@ extern char *netdata_configured_host_prefix;
757
#include "http/http_defs.h"
758
#include "gorilla/gorilla.h"
759
#include "facets/facets.h"
750
-#include "dyn_conf/dyn_conf.h"
760
#include "functions_evloop/functions_evloop.h"
761
#include "query_progress/progress.h"
762
@@ -901,6 +910,17 @@ bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_
910
911
int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size);
912
913
+static inline void freez_charp(char **p) {
914
+ freez(*p);
915
+}
916
+
917
+static inline void freez_const_charp(const char **p) {
918
+ freez((void *)*p);
919
+}
920
+
921
+#define CLEAN_CONST_CHAR_P _cleanup_(freez_const_charp) const char
922
+#define CLEAN_CHAR_P _cleanup_(freez_charp) char
923
+
924
# ifdef __cplusplus
925
}
926
# endif
libnetdata/log/log.c
+44
-28
@@ -1116,9 +1116,21 @@ static __thread struct log_field thread_log_fields[_NDF_MAX] = {
1116
.journal = "ND_SRC_TRANSPORT",
1117
.logfmt = "src_transport",
1118
},
1119
+ [NDF_ACCOUNT_ID] = {
1120
+ .journal = "ND_ACCOUNT_ID",
1121
+ .logfmt = "account",
1122
+ },
1123
+ [NDF_USER_NAME] = {
1124
+ .journal = "ND_USER_NAME",
1125
+ .logfmt = "user",
1126
+ },
1127
+ [NDF_USER_ROLE] = {
1128
+ .journal = "ND_USER_ROLE",
1129
+ .logfmt = "role",
1130
+ },
1131
[NDF_SRC_IP] = {
1120
- .journal = "ND_SRC_IP",
1121
- .logfmt = "src_ip",
1132
+ .journal = "ND_SRC_IP",
1133
+ .logfmt = "src_ip",
1134
},
1135
[NDF_SRC_FORWARDED_HOST] = {
1136
.journal = "ND_SRC_FORWARDED_HOST",
@@ -1360,11 +1372,12 @@ static void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_m
1372
case NDFT_DBL:
1373
buffer_json_member_add_double(wb, key, fields[i].entry.dbl);
1374
break;
1363
- case NDFT_UUID:{
1364
- char u[UUID_COMPACT_STR_LEN];
1365
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1366
- buffer_json_member_add_string(wb, key, u);
1367
- }
1375
+ case NDFT_UUID:
1376
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
1377
+ char u[UUID_COMPACT_STR_LEN];
1378
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1379
+ buffer_json_member_add_string(wb, key, u);
1380
+ }
1381
break;
1382
case NDFT_CALLBACK: {
1383
if(!tmp)
@@ -1689,13 +1702,14 @@ static void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields
1702
buffer_fast_strcat(wb, "=", 1);
1703
buffer_print_netdata_double(wb, fields[i].entry.dbl);
1704
break;
1692
- case NDFT_UUID: {
1693
- char u[UUID_COMPACT_STR_LEN];
1694
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1695
- buffer_strcat(wb, key);
1696
- buffer_fast_strcat(wb, "=", 1);
1697
- buffer_fast_strcat(wb, u, sizeof(u) - 1);
1698
- }
1705
+ case NDFT_UUID:
1706
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
1707
+ char u[UUID_COMPACT_STR_LEN];
1708
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1709
+ buffer_strcat(wb, key);
1710
+ buffer_fast_strcat(wb, "=", 1);
1711
+ buffer_fast_strcat(wb, u, sizeof(u) - 1);
1712
+ }
1713
break;
1714
case NDFT_CALLBACK: {
1715
if(!tmp)
@@ -1786,11 +1800,12 @@ static bool nd_logger_journal_libsystemd(struct log_field *fields, size_t fields
1800
case NDFT_DBL:
1801
rc = asprintf(&value, "%s=%f", key, fields[i].entry.dbl);
1802
break;
1789
- case NDFT_UUID: {
1790
- char u[UUID_COMPACT_STR_LEN];
1791
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1792
- rc = asprintf(&value, "%s=%s", key, u);
1793
- }
1803
+ case NDFT_UUID:
1804
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
1805
+ char u[UUID_COMPACT_STR_LEN];
1806
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1807
+ rc = asprintf(&value, "%s=%s", key, u);
1808
+ }
1809
break;
1810
case NDFT_CALLBACK: {
1811
if(!tmp)
@@ -1884,14 +1899,15 @@ static bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max
1899
buffer_print_netdata_double(wb, fields[i].entry.dbl);
1900
buffer_putc(wb, '\n');
1901
break;
1887
- case NDFT_UUID:{
1888
- char u[UUID_COMPACT_STR_LEN];
1889
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1890
- buffer_strcat(wb, key);
1891
- buffer_putc(wb, '=');
1892
- buffer_fast_strcat(wb, u, sizeof(u) - 1);
1893
- buffer_putc(wb, '\n');
1894
- }
1902
+ case NDFT_UUID:
1903
+ if(!uuid_is_null(*fields[i].entry.uuid)) {
1904
+ char u[UUID_COMPACT_STR_LEN];
1905
+ uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
1906
+ buffer_strcat(wb, key);
1907
+ buffer_putc(wb, '=');
1908
+ buffer_fast_strcat(wb, u, sizeof(u) - 1);
1909
+ buffer_putc(wb, '\n');
1910
+ }
1911
break;
1912
case NDFT_CALLBACK: {
1913
if(!tmp)
@@ -2089,7 +2105,7 @@ static void nd_logger_merge_log_stack_to_thread_fields(void) {
2105
if((type == NDFT_TXT && (!e->txt || !*e->txt)) ||
2106
(type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) ||
2107
(type == NDFT_STR && !e->str) ||
2092
- (type == NDFT_UUID && !e->uuid) ||
2108
+ (type == NDFT_UUID && (!e->uuid || uuid_is_null(*e->uuid))) ||
2109
(type == NDFT_CALLBACK && !e->cb.formatter) ||
2110
type == NDFT_UNSET)
2111
continue;
libnetdata/log/log.h
+5
@@ -63,6 +63,11 @@ typedef enum __attribute__((__packed__)) {
63
// web server, aclk and stream receiver
64
NDF_SRC_TRANSPORT, // the transport we received the request, one of: http, https, pluginsd
65
66
+ // Netdata Cloud Related
67
+ NDF_ACCOUNT_ID,
68
+ NDF_USER_NAME,
69
+ NDF_USER_ROLE,
70
+
71
// web server and stream receiver
72
NDF_SRC_IP, // the streaming / web server source IP
73
NDF_SRC_PORT, // the streaming / web server source Port
libnetdata/query_progress/progress.c
+4
-4
@@ -173,7 +173,7 @@ static void query_progress_cleanup_to_reuse(QUERY_PROGRESS *qp, uuid_t *transact
173
uuid_copy(qp->transaction, *transaction);
174
}
175
176
-static inline void query_progress_update(QUERY_PROGRESS *qp, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client) {
176
+static inline void query_progress_update(QUERY_PROGRESS *qp, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, BUFFER *payload, const char *client) {
177
qp->mode = mode;
178
qp->acl = acl;
179
qp->started_ut = started_ut ? started_ut : now_realtime_usec();
@@ -186,8 +186,8 @@ static inline void query_progress_update(QUERY_PROGRESS *qp, usec_t started_ut,
186
if(query && *query && !buffer_strlen(qp->query))
187
buffer_strcat(qp->query, query);
188
189
- if(payload && *payload && !buffer_strlen(qp->payload))
190
- buffer_strcat(qp->payload, payload);
189
+ if(payload && !buffer_strlen(qp->payload))
190
+ buffer_copy(qp->payload, payload);
191
192
if(client && *client && !buffer_strlen(qp->client))
193
buffer_strcat(qp->client, client);
@@ -210,7 +210,7 @@ static inline void query_progress_unlink_from_cache_unsafe(QUERY_PROGRESS *qp) {
210
// ----------------------------------------------------------------------------
211
// Progress API
212
213
-void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client) {
213
+void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, BUFFER *payload, const char *client) {
214
if(!transaction)
215
return;
216
libnetdata/query_progress/progress.h
+1
-1
@@ -5,7 +5,7 @@
5
6
#include "../libnetdata.h"
7
8
-void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client);
8
+void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, BUFFER *payload, const char *client);
9
void query_progress_done_step(uuid_t *transaction, size_t done);
10
void query_progress_set_finish_line(uuid_t *transaction, size_t all);
11
void query_progress_finished(uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size);
libnetdata/string/string.c
+19
@@ -307,6 +307,25 @@ STRING *string_strdupz(const char *str) {
307
return string;
308
}
309
310
+STRING *string_strndupz(const char *str, size_t len) {
311
+ if(unlikely(!str || !*str || !len)) return NULL;
312
+
313
+#ifdef NETDATA_INTERNAL_CHECKS
314
+ uint8_t partition = string_partition_str(str);
315
+#endif
316
+
317
+ char buf[len + 1];
318
+ memcpy(buf, str, len);
319
+ buf[len] = '\0';
320
+
321
+ STRING *string = string_index_search(buf, len + 1);
322
+ while(!string)
323
+ string = string_index_insert(buf, len + 1);
324
+
325
+ string_stats_atomic_increment(partition, active_references);
326
+ return string;
327
+}
328
+
329
void string_freez(STRING *string) {
330
if(unlikely(!string)) return;
331
libnetdata/string/string.h
+3
@@ -8,7 +8,10 @@
8
// STRING implementation
9
10
typedef struct netdata_string STRING;
11
+
12
STRING *string_strdupz(const char *str);
13
+STRING *string_strndupz(const char *str, size_t len);
14
+
15
STRING *string_dup(STRING *string);
16
void string_freez(STRING *string);
17
size_t string_strlen(STRING *string);
libnetdata/url/url.c
+24
-10
@@ -236,7 +236,8 @@ fail_cleanup:
236
return NULL;
237
}
238
239
-inline bool url_is_request_complete(char *begin, char *end, size_t length, char **post_payload, size_t *post_payload_size) {
239
+inline bool
240
+url_is_request_complete_and_extract_payload(const char *begin, const char *end, size_t length, BUFFER **post_payload) {
241
if (begin == end || length < 4)
242
return false;
243
@@ -244,29 +245,42 @@ inline bool url_is_request_complete(char *begin, char *end, size_t length, char
245
return strstr(end - 4, "\r\n\r\n");
246
}
247
else if(unlikely(strncmp(begin, "POST ", 5) == 0 || strncmp(begin, "PUT ", 4) == 0)) {
247
- char *cl = strstr(begin, "Content-Length: ");
248
+ const char *cl = strcasestr(begin, "Content-Length: ");
249
if(!cl) return false;
250
cl = &cl[16];
251
252
size_t content_length = str2ul(cl);
253
253
- char *payload = strstr(cl, "\r\n\r\n");
254
+ const char *payload = strstr(cl, "\r\n\r\n");
255
if(!payload) return false;
256
payload += 4;
257
258
size_t payload_length = length - (payload - begin);
259
260
if(payload_length == content_length) {
260
- if(post_payload && post_payload_size) {
261
- if (*post_payload)
262
- freez(*post_payload);
261
+ if(!*post_payload)
262
+ *post_payload = buffer_create(payload_length + 1, NULL);
263
264
- *post_payload = mallocz(payload_length + 1);
265
- memcpy(*post_payload, payload, payload_length);
266
- (*post_payload)[payload_length] = '\0';
264
+ buffer_contents_replace(*post_payload, payload, payload_length);
265
268
- *post_payload_size = payload_length;
266
+ // parse the content type
267
+ const char *ct = strcasestr(begin, "Content-Type: ");
268
+ if(ct) {
269
+ ct = &ct[14];
270
+ while (*ct && isspace(*ct)) ct++;
271
+ const char *space = ct;
272
+ while (*space && !isspace(*space) && *space != ';') space++;
273
+ size_t ct_len = space - ct;
274
+
275
+ char ct_copy[ct_len + 1];
276
+ memcpy(ct_copy, ct, ct_len);
277
+ ct_copy[ct_len] = '\0';
278
+
279
+ (*post_payload)->content_type = content_type_string2id(ct_copy);
280
}
281
+ else
282
+ (*post_payload)->content_type = CT_TEXT_PLAIN;
283
+
284
return true;
285
}
286
libnetdata/url/url.h
+1
-1
@@ -25,7 +25,7 @@ char *url_decode(char *str);
25
26
char *url_decode_r(char *to, const char *url, size_t size);
27
28
-bool url_is_request_complete(char *begin, char *end, size_t length, char **post_payload, size_t *post_payload_length);
28
+bool url_is_request_complete_and_extract_payload(const char *begin, const char *end, size_t length, BUFFER **post_payload);
29
char *url_find_protocol(char *s);
30
31
#endif /* NETDATA_URL_H */
logsmanagement/flb_plugin.c
+2
-2
@@ -854,7 +854,7 @@ static int flb_collect_logs_cb(void *record, size_t size, void *data){
854
memcpy(key, str, str_len);
855
key[str_len] = '\0';
856
metrics_dict_item_t item = {.dim_initialized = false, .num_new = 1};
857
- dictionary_set_advanced(dict, key, str_len + 1, &item, sizeof(item), NULL);
857
+ dictionary_set_advanced(dict, key, str_len, &item, sizeof(item), NULL);
858
}
859
c = &c[sz];
860
}
@@ -1161,7 +1161,7 @@ static int flb_collect_logs_cb(void *record, size_t size, void *data){
1161
memcpy(key, mqtt_topic, mqtt_topic_size);
1162
key[mqtt_topic_size] = '\0';
1163
metrics_dict_item_t item = {.dim_initialized = false, .num_new = 1};
1164
- dictionary_set_advanced(p_file_info->parser_metrics->mqtt->topic, key, mqtt_topic_size + 1, &item, sizeof(item), NULL);
1164
+ dictionary_set_advanced(p_file_info->parser_metrics->mqtt->topic, key, mqtt_topic_size, &item, sizeof(item), NULL);
1165
1166
// TODO: Fix: Metrics will still be collected if circ_buff_prepare_write() returns 0.
1167
if(unlikely(!circ_buff_prepare_write(buff, new_tmp_text_size)))
logsmanagement/functions.c
+6
-2
@@ -160,7 +160,10 @@ typedef struct function_query_status {
160
"|message" \
161
""
162
163
-static void logsmanagement_function_facets(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled){
163
+static void logsmanagement_function_facets(const char *transaction, char *function,
164
+ usec_t *stop_monotonic_ut, bool *cancelled,
165
+ BUFFER *payload __maybe_unused,
166
+ const char *src __maybe_unused, void *data __maybe_unused){
167
168
struct rusage start, end;
169
getrusage(RUSAGE_THREAD, &start);
@@ -711,7 +714,8 @@ struct functions_evloop_globals *logsmanagement_func_facets_init(bool *p_logsman
714
715
functions_evloop_add_function( wg, LOGS_MANAG_FUNC_NAME,
716
logsmanagement_function_facets,
714
- LOGS_MANAG_QUERY_TIMEOUT_DEFAULT);
717
+ LOGS_MANAG_QUERY_TIMEOUT_DEFAULT,
718
+ NULL);
719
720
return wg;
721
}
streaming/rrdpush.c
+5
-121
@@ -302,7 +302,7 @@ static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {
302
303
// send the chart functions
304
if(stream_has_capability(host->sender, STREAM_CAP_FUNCTIONS))
305
- rrd_functions_expose_rrdpush(st, wb);
305
+ rrd_chart_functions_expose_rrdpush(st, wb);
306
307
// send the chart local custom variables
308
rrdsetvar_print_to_streaming_custom_chart_variables(st, wb);
@@ -485,40 +485,6 @@ void rrdset_push_metrics_finished(RRDSET_STREAM_BUFFER *rsb, RRDSET *st) {
485
*rsb = (RRDSET_STREAM_BUFFER){ .wb = NULL, };
486
}
487
488
-#define dyncfg_can_push(host) (rrdhost_can_send_definitions_to_parent(host) && stream_has_capability((host)->sender, STREAM_CAP_DYNCFG))
489
-
490
-// assumes job is locked and acquired!!!
491
-void rrdpush_send_job_status_update(RRDHOST *host, const char *plugin_name, const char *module_name, struct job *job) {
492
- if(!dyncfg_can_push(host)) return;
493
-
494
- BUFFER *wb = sender_start(host->sender);
495
-
496
- buffer_sprintf(wb, PLUGINSD_KEYWORD_REPORT_JOB_STATUS " %s %s %s %s %d", plugin_name, module_name, job->name, job_status2str(job->status), job->state);
497
-
498
- if (job->reason && strlen(job->reason))
499
- buffer_sprintf(wb, " \"%s\"", job->reason);
500
-
501
- buffer_strcat(wb, "\n");
502
-
503
- sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
504
-
505
- sender_thread_buffer_free();
506
-
507
- job->dirty = 0;
508
-}
509
-
510
-void rrdpush_send_job_deleted(RRDHOST *host, const char *plugin_name, const char *module_name, const char *job_name) {
511
- if(!dyncfg_can_push(host)) return;
512
-
513
- BUFFER *wb = sender_start(host->sender);
514
-
515
- buffer_sprintf(wb, PLUGINSD_KEYWORD_DELETE_JOB " %s %s %s\n", plugin_name, module_name, job_name);
516
-
517
- sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
518
-
519
- sender_thread_buffer_free();
520
-}
521
-
488
RRDSET_STREAM_BUFFER rrdset_push_metric_initialize(RRDSET *st, time_t wall_clock_time) {
489
RRDHOST *host = st->rrdhost;
490
@@ -545,7 +511,7 @@ RRDSET_STREAM_BUFFER rrdset_push_metric_initialize(RRDSET *st, time_t wall_clock
511
512
if(unlikely(host_flags & RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED)) {
513
BUFFER *wb = sender_start(host->sender);
548
- rrd_functions_expose_global_rrdpush(host, wb);
514
+ rrd_global_functions_expose_rrdpush(host, wb, stream_has_capability(host->sender, STREAM_CAP_DYNCFG));
515
sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
516
}
517
@@ -605,94 +571,13 @@ void rrdpush_send_global_functions(RRDHOST *host) {
571
572
BUFFER *wb = sender_start(host->sender);
573
608
- rrd_functions_expose_global_rrdpush(host, wb);
574
+ rrd_global_functions_expose_rrdpush(host, wb, stream_has_capability(host->sender, STREAM_CAP_DYNCFG));
575
576
sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
577
578
sender_thread_buffer_free();
579
}
580
615
-void rrdpush_send_dyncfg(RRDHOST *host) {
616
- if(!dyncfg_can_push(host)) return;
617
-
618
- BUFFER *wb = sender_start(host->sender);
619
-
620
- DICTIONARY *plugins_dict = host->configurable_plugins;
621
-
622
- struct configurable_plugin *plug;
623
- dfe_start_read(plugins_dict, plug) {
624
- buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_ENABLE " %s\n", plug->name);
625
- struct module *mod;
626
- dfe_start_read(plug->modules, mod) {
627
- buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE " %s %s %s\n", plug->name, mod->name, module_type2str(mod->type));
628
- struct job *job;
629
- dfe_start_read(mod->jobs, job) {
630
- pthread_mutex_lock(&job->lock);
631
- buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB " %s %s %s %s %"PRIu32"\n", plug->name, mod->name, job->name, job_type2str(job->type), job->flags);
632
- buffer_sprintf(wb, PLUGINSD_KEYWORD_REPORT_JOB_STATUS " %s %s %s %s %d", plug->name, mod->name, job->name, job_status2str(job->status), job->state);
633
- if (job->reason)
634
- buffer_sprintf(wb, " \"%s\"", job->reason);
635
- buffer_sprintf(wb, "\n");
636
- job->dirty = 0;
637
- pthread_mutex_unlock(&job->lock);
638
- } dfe_done(job);
639
- } dfe_done(mod);
640
- }
641
- dfe_done(plug);
642
-
643
- sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
644
-
645
- sender_thread_buffer_free();
646
-}
647
-
648
-void rrdpush_send_dyncfg_enable(RRDHOST *host, const char *plugin_name) {
649
- if(!dyncfg_can_push(host)) return;
650
-
651
- BUFFER *wb = sender_start(host->sender);
652
-
653
- buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_ENABLE " %s\n", plugin_name);
654
-
655
- sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
656
-
657
- sender_thread_buffer_free();
658
-}
659
-
660
-void rrdpush_send_dyncfg_reg_module(RRDHOST *host, const char *plugin_name, const char *module_name, enum module_type type) {
661
- if(!dyncfg_can_push(host)) return;
662
-
663
- BUFFER *wb = sender_start(host->sender);
664
-
665
- buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE " %s %s %s\n", plugin_name, module_name, module_type2str(type));
666
-
667
- sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
668
-
669
- sender_thread_buffer_free();
670
-}
671
-
672
-void rrdpush_send_dyncfg_reg_job(RRDHOST *host, const char *plugin_name, const char *module_name, const char *job_name, enum job_type type, uint32_t flags) {
673
- if(!dyncfg_can_push(host)) return;
674
-
675
- BUFFER *wb = sender_start(host->sender);
676
-
677
- buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB " %s %s %s %s %"PRIu32"\n", plugin_name, module_name, job_name, job_type2str(type), flags);
678
-
679
- sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_DYNCFG);
680
-
681
- sender_thread_buffer_free();
682
-}
683
-
684
-void rrdpush_send_dyncfg_reset(RRDHOST *host, const char *plugin_name) {
685
- if(!dyncfg_can_push(host)) return;
686
-
687
- BUFFER *wb = sender_start(host->sender);
688
-
689
- buffer_sprintf(wb, PLUGINSD_KEYWORD_DYNCFG_RESET " %s\n", plugin_name);
690
-
691
- sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
692
-
693
- sender_thread_buffer_free();
694
-}
695
-
581
void rrdpush_send_claimed_id(RRDHOST *host) {
582
if(!stream_has_capability(host->sender, STREAM_CAP_CLAIM))
583
return;
@@ -1486,11 +1371,10 @@ STREAM_CAPABILITIES stream_our_capabilities(RRDHOST *host, bool sender) {
1371
STREAM_CAP_REPLICATION |
1372
STREAM_CAP_BINARY |
1373
STREAM_CAP_INTERPOLATED |
1489
- STREAM_CAP_SLOTS | STREAM_CAP_PROGRESS |
1374
+ STREAM_CAP_SLOTS |
1375
+ STREAM_CAP_PROGRESS |
1376
STREAM_CAP_COMPRESSIONS_AVAILABLE |
1491
- #ifdef NETDATA_TEST_DYNCFG
1377
STREAM_CAP_DYNCFG |
1493
- #endif
1378
STREAM_CAP_IEEE754 |
1379
STREAM_CAP_DATA_WITH_ML |
1380
0) & ~disabled_capabilities;
streaming/rrdpush.h
+31
-20
@@ -47,12 +47,13 @@ typedef enum {
47
STREAM_CAP_INTERPOLATED = (1 << 14), // streaming supports interpolated streaming of values
48
STREAM_CAP_IEEE754 = (1 << 15), // streaming supports binary/hex transfer of double values
49
STREAM_CAP_DATA_WITH_ML = (1 << 16), // streaming supports transferring anomaly bit
50
- STREAM_CAP_DYNCFG = (1 << 17), // dynamic configuration of plugins trough streaming
50
+ // STREAM_CAP_DYNCFG = (1 << 17), // leave this unused for as long as possible
51
STREAM_CAP_SLOTS = (1 << 18), // the sender can appoint a unique slot for each chart
52
STREAM_CAP_ZSTD = (1 << 19), // ZSTD compression supported
53
STREAM_CAP_GZIP = (1 << 20), // GZIP compression supported
54
STREAM_CAP_BROTLI = (1 << 21), // BROTLI compression supported
55
STREAM_CAP_PROGRESS = (1 << 22), // Functions PROGRESS support
56
+ STREAM_CAP_DYNCFG = (1 << 23), // support for DYNCFG
57
58
STREAM_CAP_INVALID = (1 << 30), // used as an invalid value for capabilities when this is set
59
// this must be signed int, so don't use the last bit
@@ -198,13 +199,6 @@ typedef enum __attribute__((packed)) {
199
SENDER_FLAG_OVERFLOW = (1 << 0), // The buffer has been overflown
200
} SENDER_FLAGS;
201
201
-struct function_payload_state {
202
- BUFFER *payload;
203
- char *txid;
204
- char *fn_name;
205
- char *timeout;
206
-};
207
-
202
struct sender_state {
203
RRDHOST *host;
204
pid_t tid; // the thread id of the sender, from gettid()
@@ -235,9 +229,6 @@ struct sender_state {
229
int rrdpush_sender_pipe[2]; // collector to sender thread signaling
230
int rrdpush_sender_socket;
231
238
- int receiving_function_payload;
239
- struct function_payload_state function_payload; // state when receiving function with payload
240
-
232
uint16_t hops;
233
234
struct line_splitter line;
@@ -276,6 +267,15 @@ struct sender_state {
267
time_t last_buffer_recreate_s; // true when the sender buffer should be re-created
268
} atomic;
269
270
+ struct {
271
+ bool intercept_input;
272
+ const char *transaction;
273
+ const char *timeout_s;
274
+ const char *function;
275
+ const char *source;
276
+ BUFFER *payload;
277
+ } functions;
278
+
279
int parent_using_h2o;
280
};
281
@@ -457,7 +457,6 @@ void *rrdpush_sender_thread(void *ptr);
457
void rrdpush_send_host_labels(RRDHOST *host);
458
void rrdpush_send_claimed_id(RRDHOST *host);
459
void rrdpush_send_global_functions(RRDHOST *host);
460
-void rrdpush_send_dyncfg(RRDHOST *host);
460
461
int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string, void *h2o_ctx);
462
void rrdpush_sender_thread_stop(RRDHOST *host, STREAM_HANDSHAKE reason, bool wait);
@@ -659,10 +658,30 @@ static inline const char *rrdhost_health_status_to_string(RRDHOST_HEALTH_STATUS
658
}
659
}
660
661
+typedef enum __attribute__((packed)) {
662
+ RRDHOST_DYNCFG_STATUS_UNAVAILABLE = 0,
663
+ RRDHOST_DYNCFG_STATUS_AVAILABLE,
664
+} RRDHOST_DYNCFG_STATUS;
665
+
666
+static inline const char *rrdhost_dyncfg_status_to_string(RRDHOST_DYNCFG_STATUS status) {
667
+ switch(status) {
668
+ default:
669
+ case RRDHOST_DYNCFG_STATUS_UNAVAILABLE:
670
+ return "unavailable";
671
+
672
+ case RRDHOST_DYNCFG_STATUS_AVAILABLE:
673
+ return "online";
674
+ }
675
+}
676
+
677
typedef struct rrdhost_status {
678
RRDHOST *host;
679
time_t now;
680
681
+ struct {
682
+ RRDHOST_DYNCFG_STATUS status;
683
+ } dyncfg;
684
+
685
struct {
686
RRDHOST_DB_STATUS status;
687
RRDHOST_DB_LIVENESS liveness;
@@ -733,14 +752,6 @@ typedef struct rrdhost_status {
752
void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s);
753
bool rrdhost_state_cloud_emulation(RRDHOST *host);
754
736
-void rrdpush_send_job_status_update(RRDHOST *host, const char *plugin_name, const char *module_name, struct job *job);
737
-void rrdpush_send_job_deleted(RRDHOST *host, const char *plugin_name, const char *module_name, const char *job_name);
738
-
739
-void rrdpush_send_dyncfg_enable(RRDHOST *host, const char *plugin_name);
740
-void rrdpush_send_dyncfg_reg_module(RRDHOST *host, const char *plugin_name, const char *module_name, enum module_type type);
741
-void rrdpush_send_dyncfg_reg_job(RRDHOST *host, const char *plugin_name, const char *module_name, const char *job_name, enum job_type type, uint32_t flags);
742
-void rrdpush_send_dyncfg_reset(RRDHOST *host, const char *plugin_name);
743
-
755
bool rrdpush_compression_initialize(struct sender_state *s);
756
bool rrdpush_decompression_initialize(struct receiver_state *rpt);
757
void rrdpush_parse_compression_order(struct receiver_state *rpt, const char *order);
streaming/sender.c
+90
-91
@@ -1128,7 +1128,7 @@ static void stream_execute_function_callback(BUFFER *func_wb, int code, void *da
1128
pluginsd_function_result_begin_to_buffer(wb
1129
, string2str(tmp->transaction)
1130
, code
1131
- , functions_content_type_to_format(func_wb->content_type)
1131
+ , content_type_id2string(func_wb->content_type)
1132
, func_wb->expires);
1133
1134
buffer_fast_strcat(wb, buffer_tostring(func_wb), buffer_strlen(func_wb));
@@ -1163,6 +1163,60 @@ static void stream_execute_function_progress_callback(void *data, size_t done, s
1163
}
1164
}
1165
1166
+static void execute_commands_function(struct sender_state *s, const char *command, const char *transaction, const char *timeout_s, const char *function, BUFFER *payload, const char *source) {
1167
+ worker_is_busy(WORKER_SENDER_JOB_FUNCTION_REQUEST);
1168
+ nd_log(NDLS_ACCESS, NDLP_INFO, NULL);
1169
+
1170
+ if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
1171
+ netdata_log_error("STREAM %s [send to %s] %s execution command is incomplete (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
1172
+ rrdhost_hostname(s->host), s->connected_to,
1173
+ command,
1174
+ transaction?transaction:"(unset)",
1175
+ timeout_s?timeout_s:"(unset)",
1176
+ function?function:"(unset)");
1177
+ }
1178
+ else {
1179
+ int timeout = str2i(timeout_s);
1180
+ if(timeout <= 0) timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
1181
+
1182
+ struct inflight_stream_function *tmp = callocz(1, sizeof(struct inflight_stream_function));
1183
+ tmp->received_ut = now_realtime_usec();
1184
+ tmp->sender = s;
1185
+ tmp->transaction = string_strdupz(transaction);
1186
+ BUFFER *wb = buffer_create(1024, &netdata_buffers_statistics.buffers_functions);
1187
+
1188
+ int code = rrd_function_run(s->host, wb,
1189
+ timeout,HTTP_ACCESS_ADMIN, function, false, transaction,
1190
+ stream_execute_function_callback, tmp,
1191
+ stream_has_capability(s, STREAM_CAP_PROGRESS) ? stream_execute_function_progress_callback : NULL,
1192
+ stream_has_capability(s, STREAM_CAP_PROGRESS) ? tmp : NULL,
1193
+ NULL, NULL, payload, source);
1194
+
1195
+ if(code != HTTP_RESP_OK) {
1196
+ if (!buffer_strlen(wb))
1197
+ rrd_call_function_error(wb, "Failed to route request to collector", code);
1198
+ }
1199
+ }
1200
+}
1201
+
1202
+static void cleanup_intercepting_input(struct sender_state *s) {
1203
+ freez((void *)s->functions.transaction);
1204
+ freez((void *)s->functions.timeout_s);
1205
+ freez((void *)s->functions.function);
1206
+ freez((void *)s->functions.source);
1207
+ buffer_free(s->functions.payload);
1208
+
1209
+ s->functions.transaction = NULL;
1210
+ s->functions.timeout_s = NULL;
1211
+ s->functions.function = NULL;
1212
+ s->functions.payload = NULL;
1213
+ s->functions.intercept_input = false;
1214
+}
1215
+
1216
+static void execute_commands_cleanup(struct sender_state *s) {
1217
+ cleanup_intercepting_input(s);
1218
+}
1219
+
1220
// This is just a placeholder until the gap filling state machine is inserted
1221
void execute_commands(struct sender_state *s) {
1222
worker_is_busy(WORKER_SENDER_JOB_EXECUTE);
@@ -1176,105 +1230,49 @@ void execute_commands(struct sender_state *s) {
1230
char *start = s->read_buffer, *end = &s->read_buffer[s->read_len], *newline;
1231
*end = 0;
1232
while( start < end && (newline = strchr(start, '\n')) ) {
1179
- *newline = '\0';
1233
+ s->line.count++;
1234
+
1235
+ if(s->functions.intercept_input) {
1236
+ if(strcmp(start, PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END "\n") == 0) {
1237
+ execute_commands_function(s, PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END,
1238
+ s->functions.transaction, s->functions.timeout_s,
1239
+ s->functions.function, s->functions.payload, s->functions.source);
1240
+
1241
+ cleanup_intercepting_input(s);
1242
+ }
1243
+ else
1244
+ buffer_strcat(s->functions.payload, start);
1245
1181
- if (s->receiving_function_payload && unlikely(strcmp(start, PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END) != 0)) {
1182
- if (buffer_strlen(s->function_payload.payload) != 0)
1183
- buffer_strcat(s->function_payload.payload, "\n");
1184
- buffer_strcat(s->function_payload.payload, start);
1246
start = newline + 1;
1247
continue;
1248
}
1249
1189
- s->line.count++;
1250
+ *newline = '\0';
1251
s->line.num_words = quoted_strings_splitter_pluginsd(start, s->line.words, PLUGINSD_MAX_WORDS);
1252
const char *command = get_word(s->line.words, s->line.num_words, 0);
1253
1193
- if(command && (strcmp(command, PLUGINSD_KEYWORD_FUNCTION) == 0 || strcmp(command, PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END) == 0)) {
1194
- worker_is_busy(WORKER_SENDER_JOB_FUNCTION_REQUEST);
1195
- nd_log(NDLS_ACCESS, NDLP_INFO, NULL);
1196
-
1197
- char *transaction = s->receiving_function_payload ? s->function_payload.txid : get_word(s->line.words, s->line.num_words, 1);
1198
- char *timeout_s = s->receiving_function_payload ? s->function_payload.timeout : get_word(s->line.words, s->line.num_words, 2);
1199
- char *function = s->receiving_function_payload ? s->function_payload.fn_name : get_word(s->line.words, s->line.num_words, 3);
1200
-
1201
- if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
1202
- netdata_log_error("STREAM %s [send to %s] %s execution command is incomplete (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
1203
- rrdhost_hostname(s->host), s->connected_to,
1204
- command,
1205
- transaction?transaction:"(unset)",
1206
- timeout_s?timeout_s:"(unset)",
1207
- function?function:"(unset)");
1208
- }
1209
- else {
1210
- int timeout = str2i(timeout_s);
1211
- if(timeout <= 0) timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
1212
-
1213
- struct inflight_stream_function *tmp = callocz(1, sizeof(struct inflight_stream_function));
1214
- tmp->received_ut = now_realtime_usec();
1215
- tmp->sender = s;
1216
- tmp->transaction = string_strdupz(transaction);
1217
- BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX + 1, &netdata_buffers_statistics.buffers_functions);
1218
-
1219
- char *payload = s->receiving_function_payload ? (char *)buffer_tostring(s->function_payload.payload) : NULL;
1220
- int code = rrd_function_run(s->host, wb,
1221
- timeout, HTTP_ACCESS_ADMINS, function, false, transaction,
1222
- stream_execute_function_callback, tmp,
1223
- stream_has_capability(s, STREAM_CAP_PROGRESS) ? stream_execute_function_progress_callback : NULL,
1224
- stream_has_capability(s, STREAM_CAP_PROGRESS) ? tmp : NULL,
1225
- NULL, NULL, payload);
1226
-
1227
- if(code != HTTP_RESP_OK) {
1228
- if (!buffer_strlen(wb))
1229
- rrd_call_function_error(wb, "Failed to route request to collector", code);
1230
-
1231
- stream_execute_function_callback(wb, code, tmp);
1232
- }
1233
- }
1234
-
1235
- if (s->receiving_function_payload) {
1236
- s->receiving_function_payload = false;
1237
-
1238
- buffer_free(s->function_payload.payload);
1239
- freez(s->function_payload.txid);
1240
- freez(s->function_payload.timeout);
1241
- freez(s->function_payload.fn_name);
1254
+ if(command && strcmp(command, PLUGINSD_KEYWORD_FUNCTION) == 0) {
1255
+ char *transaction = get_word(s->line.words, s->line.num_words, 1);
1256
+ char *timeout_s = get_word(s->line.words, s->line.num_words, 2);
1257
+ char *function = get_word(s->line.words, s->line.num_words, 3);
1258
+ char *source = get_word(s->line.words, s->line.num_words, 4);
1259
1243
- memset(&s->function_payload, 0, sizeof(struct function_payload_state));
1244
- }
1260
+ execute_commands_function(s, command, transaction, timeout_s, function, NULL, source);
1261
}
1246
- else if (command && strcmp(command, PLUGINSD_KEYWORD_FUNCTION_PAYLOAD) == 0) {
1247
- nd_log(NDLS_ACCESS, NDLP_INFO, NULL);
1248
-
1249
- if (s->receiving_function_payload) {
1250
- netdata_log_error("STREAM %s [send to %s] received %s command while already receiving function payload",
1251
- rrdhost_hostname(s->host), s->connected_to, command);
1252
- s->receiving_function_payload = false;
1253
- buffer_free(s->function_payload.payload);
1254
- s->function_payload.payload = NULL;
1255
-
1256
- // TODO send error response
1257
- }
1258
-
1259
- char *transaction = get_word(s->line.words, s->line.num_words, 1);
1260
- char *timeout_s = get_word(s->line.words, s->line.num_words, 2);
1261
- char *function = get_word(s->line.words, s->line.num_words, 3);
1262
-
1263
- if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
1264
- netdata_log_error("STREAM %s [send to %s] %s execution command is incomplete (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
1265
- rrdhost_hostname(s->host), s->connected_to,
1266
- command,
1267
- transaction?transaction:"(unset)",
1268
- timeout_s?timeout_s:"(unset)",
1269
- function?function:"(unset)");
1270
- }
1271
-
1272
- s->receiving_function_payload = true;
1273
- s->function_payload.payload = buffer_create(4096, &netdata_buffers_statistics.buffers_functions);
1274
-
1275
- s->function_payload.txid = strdupz(get_word(s->line.words, s->line.num_words, 1));
1276
- s->function_payload.timeout = strdupz(get_word(s->line.words, s->line.num_words, 2));
1277
- s->function_payload.fn_name = strdupz(get_word(s->line.words, s->line.num_words, 3));
1262
+ else if(command && strcmp(command, PLUGINSD_KEYWORD_FUNCTION_PAYLOAD) == 0) {
1263
+ char *transaction = get_word(s->line.words, s->line.num_words, 1);
1264
+ char *timeout_s = get_word(s->line.words, s->line.num_words, 2);
1265
+ char *function = get_word(s->line.words, s->line.num_words, 3);
1266
+ char *source = get_word(s->line.words, s->line.num_words, 4);
1267
+ char *content_type = get_word(s->line.words, s->line.num_words, 5);
1268
+
1269
+ s->functions.transaction = strdupz(transaction ? transaction : "");
1270
+ s->functions.timeout_s = strdupz(timeout_s ? timeout_s : "");
1271
+ s->functions.function = strdupz(function ? function : "");
1272
+ s->functions.source = strdupz(source ? source : "");
1273
+ s->functions.payload = buffer_create(0, NULL);
1274
+ s->functions.payload->content_type = content_type_string2id(content_type);
1275
+ s->functions.intercept_input = true;
1276
}
1277
else if(command && strcmp(command, PLUGINSD_KEYWORD_FUNCTION_CANCEL) == 0) {
1278
worker_is_busy(WORKER_SENDER_JOB_FUNCTION_REQUEST);
@@ -1480,6 +1478,7 @@ static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
1478
1479
rrdpush_sender_thread_close_socket(host);
1480
rrdpush_sender_pipe_close(host, host->sender->rrdpush_sender_pipe, false);
1481
+ execute_commands_cleanup(host->sender);
1482
1483
rrdhost_clear_sender___while_having_sender_mutex(host);
1484
@@ -1680,6 +1679,7 @@ void *rrdpush_sender_thread(void *ptr) {
1679
1680
now_s = now_monotonic_sec();
1681
rrdpush_sender_cbuffer_recreate_timed(s, now_s, false, true);
1682
+ execute_commands_cleanup(s);
1683
1684
rrdhost_flag_clear(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);
1685
s->flags &= ~SENDER_FLAG_OVERFLOW;
@@ -1697,7 +1697,6 @@ void *rrdpush_sender_thread(void *ptr) {
1697
rrdpush_send_claimed_id(s->host);
1698
rrdpush_send_host_labels(s->host);
1699
rrdpush_send_global_functions(s->host);
1700
- rrdpush_send_dyncfg(s->host);
1700
s->replication.oldest_request_after_t = 0;
1701
1702
rrdhost_flag_set(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);
web/api/http_auth.c
new
+91
@@ -0,0 +1,91 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "http_auth.h"
4
+
5
+#define BEARER_TOKEN_EXPIRATION 86400
6
+
7
+bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at the point the agent logs in - this should also be saved to /var/lib/netdata
8
+static DICTIONARY *netdata_authorized_bearers = NULL;
9
+
10
+struct bearer_token {
11
+ uuid_t cloud_account_id;
12
+ char cloud_user_name[CLOUD_USER_NAME_LENGTH];
13
+ HTTP_ACCESS access;
14
+ time_t created_s;
15
+ time_t expires_s;
16
+};
17
+
18
+bool web_client_bearer_token_auth(struct web_client *w, const char *v) {
19
+ if(!uuid_parse_flexi(v, w->auth.bearer_token)) {
20
+ char uuid_str[UUID_COMPACT_STR_LEN];
21
+ uuid_unparse_lower_compact(w->auth.bearer_token, uuid_str);
22
+
23
+ struct bearer_token *z = dictionary_get(netdata_authorized_bearers, uuid_str);
24
+ if (z && z->expires_s > now_monotonic_sec()) {
25
+ w->access = z->access;
26
+ strncpyz(w->auth.client_name, z->cloud_user_name, sizeof(w->auth.client_name) - 1);
27
+ uuid_copy(w->auth.cloud_account_id, z->cloud_account_id);
28
+
29
+ web_client_flags_clear_auth(w);
30
+ web_client_flag_set(w, WEB_CLIENT_FLAG_AUTH_BEARER);
31
+ return true;
32
+ }
33
+ }
34
+ else
35
+ nd_log(NDLS_DAEMON, NDLP_NOTICE, "Invalid bearer token '%s' received.", v);
36
+
37
+ return false;
38
+}
39
+
40
+static void bearer_token_cleanup(void) {
41
+ static time_t attempts = 0;
42
+
43
+ if(++attempts % 1000 != 0)
44
+ return;
45
+
46
+ time_t now_s = now_monotonic_sec();
47
+
48
+ struct bearer_token *z;
49
+ dfe_start_read(netdata_authorized_bearers, z) {
50
+ if(z->expires_s < now_s)
51
+ dictionary_del(netdata_authorized_bearers, z_dfe.name);
52
+ }
53
+ dfe_done(z);
54
+
55
+ dictionary_garbage_collect(netdata_authorized_bearers);
56
+}
57
+
58
+void bearer_tokens_init(void) {
59
+ netdata_authorized_bearers = dictionary_create_advanced(
60
+ DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
61
+ NULL, sizeof(struct bearer_token));
62
+}
63
+
64
+time_t bearer_create_token(uuid_t *uuid, struct web_client *w) {
65
+ char uuid_str[UUID_COMPACT_STR_LEN];
66
+
67
+ uuid_generate_random(*uuid);
68
+ uuid_unparse_lower_compact(*uuid, uuid_str);
69
+
70
+ struct bearer_token t = { 0 }, *z;
71
+ z = dictionary_set(netdata_authorized_bearers, uuid_str, &t, sizeof(t));
72
+ if(!z->created_s) {
73
+ z->created_s = now_monotonic_sec();
74
+ z->expires_s = z->created_s + BEARER_TOKEN_EXPIRATION;
75
+ z->access = w->access;
76
+ uuid_copy(z->cloud_account_id, w->auth.cloud_account_id);
77
+ strncpyz(z->cloud_user_name, w->auth.client_name, sizeof(z->cloud_account_id) - 1);
78
+ }
79
+
80
+ bearer_token_cleanup();
81
+
82
+ return now_realtime_sec() + BEARER_TOKEN_EXPIRATION;
83
+}
84
+
85
+bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
86
+ if(!web_client_flag_check(w, WEB_CLIENT_FLAG_AUTH_BEARER) || dst_len != UUID_STR_LEN)
87
+ return false;
88
+
89
+ uuid_unparse_lower(w->auth.bearer_token, dst);
90
+ return true;
91
+}
web/api/http_auth.h
new
+21
@@ -0,0 +1,21 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_HTTP_AUTH_H
4
+#define NETDATA_HTTP_AUTH_H
5
+
6
+#include "web_api.h"
7
+
8
+struct web_client;
9
+
10
+extern bool netdata_is_protected_by_bearer;
11
+
12
+bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
13
+
14
+time_t bearer_create_token(uuid_t *uuid, struct web_client *w);
15
+bool web_client_bearer_token_auth(struct web_client *w, const char *v);
16
+
17
+static inline bool web_client_has_enough_access_level(HTTP_ACCESS user_level, HTTP_ACCESS endpoint_level) {
18
+ return user_level != HTTP_ACCESS_NONE && user_level <= endpoint_level;
19
+}
20
+
21
+#endif //NETDATA_HTTP_AUTH_H
web/api/http_header.c
new
+241
@@ -0,0 +1,241 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "http_header.h"
4
+
5
+static void web_client_enable_deflate(struct web_client *w, bool gzip) {
6
+ if(gzip)
7
+ web_client_flag_set(w, WEB_CLIENT_ENCODING_GZIP);
8
+ else
9
+ web_client_flag_set(w, WEB_CLIENT_ENCODING_DEFLATE);
10
+
11
+ if(!web_client_check_conn_unix(w) && !web_client_check_conn_tcp(w) && !web_client_check_conn_cloud(w))
12
+ return;
13
+
14
+ if(unlikely(w->response.zinitialized)) {
15
+ // compression has already been initialized for this client.
16
+ return;
17
+ }
18
+
19
+ if(unlikely(w->response.sent)) {
20
+ netdata_log_error("%llu: Cannot enable compression in the middle of a conversation.", w->id);
21
+ return;
22
+ }
23
+
24
+ w->response.zstream.zalloc = Z_NULL;
25
+ w->response.zstream.zfree = Z_NULL;
26
+ w->response.zstream.opaque = Z_NULL;
27
+
28
+ w->response.zstream.next_in = (Bytef *)w->response.data->buffer;
29
+ w->response.zstream.avail_in = 0;
30
+ w->response.zstream.total_in = 0;
31
+
32
+ w->response.zstream.next_out = w->response.zbuffer;
33
+ w->response.zstream.avail_out = 0;
34
+ w->response.zstream.total_out = 0;
35
+
36
+ w->response.zstream.zalloc = Z_NULL;
37
+ w->response.zstream.zfree = Z_NULL;
38
+ w->response.zstream.opaque = Z_NULL;
39
+
40
+ // Select GZIP compression: windowbits = 15 + 16 = 31
41
+ if(deflateInit2(&w->response.zstream, web_gzip_level, Z_DEFLATED, 15 + ((gzip)?16:0), 8, web_gzip_strategy) != Z_OK) {
42
+ netdata_log_error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
43
+ return;
44
+ }
45
+
46
+ w->response.zsent = 0;
47
+ w->response.zoutput = true;
48
+ w->response.zinitialized = true;
49
+
50
+ if(!web_client_check_conn_cloud(w))
51
+ // cloud sends the entire response at once, not in chunks
52
+ web_client_flag_set(w, WEB_CLIENT_CHUNKED_TRANSFER);
53
+
54
+ netdata_log_debug(D_DEFLATE, "%llu: Initialized compression.", w->id);
55
+}
56
+
57
+static void http_header_origin(struct web_client *w, const char *v, size_t len __maybe_unused) {
58
+ freez(w->origin);
59
+ w->origin = strdupz(v);
60
+}
61
+
62
+static void http_header_connection(struct web_client *w, const char *v, size_t len __maybe_unused) {
63
+ if(strcasestr(v, "keep-alive"))
64
+ web_client_enable_keepalive(w);
65
+}
66
+
67
+static void http_header_dnt(struct web_client *w, const char *v, size_t len __maybe_unused) {
68
+ if(respect_web_browser_do_not_track_policy) {
69
+ if (*v == '0') web_client_disable_donottrack(w);
70
+ else if (*v == '1') web_client_enable_donottrack(w);
71
+ }
72
+}
73
+
74
+static void http_header_user_agent(struct web_client *w, const char *v, size_t len __maybe_unused) {
75
+ if(w->mode == HTTP_REQUEST_MODE_STREAM) {
76
+ freez(w->user_agent);
77
+ w->user_agent = strdupz(v);
78
+ }
79
+}
80
+
81
+static void http_header_x_auth_token(struct web_client *w, const char *v, size_t len __maybe_unused) {
82
+ freez(w->auth_bearer_token);
83
+ w->auth_bearer_token = strdupz(v);
84
+}
85
+
86
+static void http_header_host(struct web_client *w, const char *v, size_t len) {
87
+ char buffer[NI_MAXHOST];
88
+ strncpyz(buffer, v, (len < sizeof(buffer) - 1 ? len : sizeof(buffer) - 1));
89
+ freez(w->server_host);
90
+ w->server_host = strdupz(buffer);
91
+}
92
+
93
+static void http_header_accept_encoding(struct web_client *w, const char *v, size_t len __maybe_unused) {
94
+ if(web_enable_gzip) {
95
+ if(strcasestr(v, "gzip"))
96
+ web_client_enable_deflate(w, true);
97
+
98
+ // does not seem to work
99
+ // else if(strcasestr(v, "deflate"))
100
+ // web_client_enable_deflate(w, 0);
101
+ }
102
+}
103
+
104
+static void http_header_x_forwarded_host(struct web_client *w, const char *v, size_t len) {
105
+ char buffer[NI_MAXHOST];
106
+ strncpyz(buffer, v, (len < sizeof(buffer) - 1 ? len : sizeof(buffer) - 1));
107
+ freez(w->forwarded_host);
108
+ w->forwarded_host = strdupz(buffer);
109
+}
110
+
111
+static void http_header_x_forwarded_for(struct web_client *w, const char *v, size_t len) {
112
+ char buffer[NI_MAXHOST];
113
+ strncpyz(buffer, v, (len < sizeof(buffer) - 1 ? len : sizeof(buffer) - 1));
114
+ freez(w->forwarded_for);
115
+ w->forwarded_for = strdupz(buffer);
116
+}
117
+
118
+static void http_header_x_transaction_id(struct web_client *w, const char *v, size_t len) {
119
+ char buffer[UUID_STR_LEN * 2];
120
+ strncpyz(buffer, v, (len < sizeof(buffer) - 1 ? len : sizeof(buffer) - 1));
121
+ uuid_parse_flexi(buffer, w->transaction); // will not alter w->transaction if it fails
122
+}
123
+
124
+static void http_header_x_netdata_account_id(struct web_client *w, const char *v, size_t len) {
125
+ if(web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_CLOUD) && w->acl == HTTP_ACL_ACLK) {
126
+ char buffer[UUID_STR_LEN * 2];
127
+ strncpyz(buffer, v, (len < sizeof(buffer) - 1 ? len : sizeof(buffer) - 1));
128
+ uuid_parse_flexi(buffer, w->auth.cloud_account_id); // will not alter w->cloud_account_id if it fails
129
+ }
130
+}
131
+
132
+static void http_header_x_netdata_role(struct web_client *w, const char *v, size_t len) {
133
+ if(web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_CLOUD) && w->acl == HTTP_ACL_ACLK) {
134
+ char buffer[100];
135
+ strncpyz(buffer, v, (len < sizeof(buffer) - 1 ? len : sizeof(buffer) - 1));
136
+ if (strcasecmp(buffer, "admin") == 0)
137
+ w->access = HTTP_ACCESS_ADMIN;
138
+ else if(strcasecmp(buffer, "member") == 0)
139
+ w->access = HTTP_ACCESS_MEMBER;
140
+ else
141
+ w->access = HTTP_ACCESS_ANY;
142
+
143
+ web_client_flags_clear_auth(w);
144
+ web_client_flag_set(w, WEB_CLIENT_FLAG_AUTH_CLOUD);
145
+ }
146
+}
147
+
148
+static void http_header_x_netdata_user_name(struct web_client *w, const char *v, size_t len) {
149
+ if(web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_CLOUD) && w->acl == HTTP_ACL_ACLK) {
150
+ strncpyz(w->auth.client_name, v, (len < sizeof(w->auth.client_name) - 1 ? len : sizeof(w->auth.client_name) - 1));
151
+ }
152
+}
153
+
154
+static void http_header_x_netdata_auth(struct web_client *w, const char *v, size_t len __maybe_unused) {
155
+ if(web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_CLOUD) && w->acl == HTTP_ACL_ACLK)
156
+ // we don't need authorization bearer when the request comes from netdata cloud
157
+ return;
158
+
159
+ if(strncasecmp(v, "Bearer ", 7) == 0) {
160
+ v = &v[7];
161
+ while(*v && isspace(*v)) v++;
162
+ web_client_bearer_token_auth(w, v);
163
+ }
164
+}
165
+
166
+struct {
167
+ uint32_t hash;
168
+ const char *key;
169
+ void (*cb)(struct web_client *w, const char *value, size_t value_len);
170
+} supported_headers[] = {
171
+ { .hash = 0, .key = "Origin", .cb = http_header_origin },
172
+ { .hash = 0, .key = "Connection", .cb = http_header_connection },
173
+ { .hash = 0, .key = "DNT", .cb = http_header_dnt },
174
+ { .hash = 0, .key = "User-Agent", .cb = http_header_user_agent},
175
+ { .hash = 0, .key = "X-Auth-Token", .cb = http_header_x_auth_token },
176
+ { .hash = 0, .key = "Host", .cb = http_header_host },
177
+ { .hash = 0, .key = "Accept-Encoding", .cb = http_header_accept_encoding },
178
+ { .hash = 0, .key = "X-Forwarded-Host", .cb = http_header_x_forwarded_host },
179
+ { .hash = 0, .key = "X-Forwarded-For", .cb = http_header_x_forwarded_for },
180
+ { .hash = 0, .key = "X-Transaction-Id", .cb = http_header_x_transaction_id },
181
+ { .hash = 0, .key = "X-Netdata-Account-Id", .cb = http_header_x_netdata_account_id },
182
+ { .hash = 0, .key = "X-Netdata-Role", .cb = http_header_x_netdata_role },
183
+ { .hash = 0, .key = "X-Netdata-User-Name", .cb = http_header_x_netdata_user_name },
184
+ { .hash = 0, .key = "X-Netdata-Auth", .cb = http_header_x_netdata_auth },
185
+
186
+ // for historical reasons.
187
+ // there are a few nightly versions of netdata UI that incorrectly use this instead of X-Netdata-Auth
188
+ { .hash = 0, .key = "Authorization", .cb = http_header_x_netdata_auth },
189
+
190
+ // terminator
191
+ { .hash = 0, .key = NULL, .cb = NULL }
192
+};
193
+
194
+char *http_header_parse_line(struct web_client *w, char *s) {
195
+ if(unlikely(!supported_headers[0].hash)) {
196
+ // initialize the hashes, the first time it runs
197
+
198
+ for(size_t i = 0; supported_headers[i].key ;i++)
199
+ supported_headers[i].hash = simple_uhash(supported_headers[i].key);
200
+ }
201
+
202
+ char *e = s;
203
+
204
+ // find the colon
205
+ while(*e && *e != ':') e++;
206
+ if(!*e) return e;
207
+
208
+ // get the name
209
+ *e = '\0';
210
+
211
+ // find the value
212
+ char *v = e + 1, *ve;
213
+
214
+ // skip leading spaces from value
215
+ while(*v == ' ') v++;
216
+ ve = v;
217
+
218
+ // find the \r
219
+ while(*ve && *ve != '\r') ve++;
220
+ if(!*ve || ve[1] != '\n') {
221
+ *e = ':';
222
+ return ve;
223
+ }
224
+
225
+ // terminate the value
226
+ *ve = '\0';
227
+
228
+ uint32_t hash = simple_uhash(s);
229
+
230
+ for(size_t i = 0; supported_headers[i].key ;i++) {
231
+ if(likely(hash != supported_headers[i].hash || strcasecmp(s, supported_headers[i].key) != 0))
232
+ continue;
233
+
234
+ supported_headers[i].cb(w, v, ve - v);
235
+ break;
236
+ }
237
+
238
+ *e = ':';
239
+ *ve = '\r';
240
+ return ve;
241
+}
web/api/http_header.h
new
+11
@@ -0,0 +1,11 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_HTTP_HEADER_H
4
+#define NETDATA_HTTP_HEADER_H
5
+
6
+#include "web_api.h"
7
+
8
+struct web_client;
9
+char *http_header_parse_line(struct web_client *w, char *s);
10
+
11
+#endif //NETDATA_HTTP_HEADER_H
web/api/queries/weights.c
+1
-1
@@ -113,7 +113,7 @@ static void register_result(DICTIONARY *results, RRDHOST *host, RRDCONTEXT_ACQUI
113
// we can use the pointer address or RMA as a unique key for each metric
114
char buf[20 + 1];
115
ssize_t len = snprintfz(buf, sizeof(buf) - 1, "%p", rma);
116
- dictionary_set_advanced(results, buf, len + 1, &t, sizeof(struct register_result), NULL);
116
+ dictionary_set_advanced(results, buf, len, &t, sizeof(struct register_result), NULL);
117
}
118
119
// ----------------------------------------------------------------------------
web/api/web_api.c
+9
-66
@@ -2,62 +2,15 @@
2
3
#include "web_api.h"
4
5
-bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at the point the agent logs in - this should also be saved to /var/lib/netdata
6
-DICTIONARY *netdata_authorized_bearers = NULL;
7
-
8
-static short int web_client_check_acl_and_bearer(struct web_client *w, HTTP_ACL endpoint_acl) {
9
- if(endpoint_acl == HTTP_ACL_NONE || (endpoint_acl & HTTP_ACL_NOCHECK)) {
10
- // the endpoint is totally public
11
- w->access = HTTP_ACCESS_ADMINS;
12
- return HTTP_RESP_OK;
13
- }
14
-
15
- bool acl_allows = w->acl & endpoint_acl;
16
- if(!acl_allows)
17
- // the channel we received the request from (w->acl) is not compatible with the endpoint
18
- return HTTP_RESP_FORBIDDEN;
19
-
20
- if(!netdata_is_protected_by_bearer && !(endpoint_acl & (HTTP_ACL_BEARER_REQUIRED | HTTP_ACL_BEARER_OPTIONAL))) {
21
- // bearer protection is not enabled and is not required by the endpoint
22
- w->access = HTTP_ACCESS_ANY;
23
- return HTTP_RESP_OK;
24
- }
25
-
26
- if(!(endpoint_acl & (HTTP_ACL_BEARER_REQUIRED | HTTP_ACL_BEARER_OPTIONAL | HTTP_ACL_BEARER_IF_PROTECTED))) {
27
- // endpoint does not require a bearer
28
- w->access = HTTP_ACCESS_ANY;
29
- return HTTP_RESP_OK;
30
- }
31
-
32
- if((w->acl & (HTTP_ACL_ACLK | HTTP_ACL_WEBRTC))) {
33
- // the request is coming from ACLK or WEBRTC (authorized already),
34
- w->access = HTTP_ACCESS_MEMBERS;
35
- return HTTP_RESP_OK;
36
- }
37
-
38
- // we now need a bearer to serve the request,
39
- // because:
40
- // 1. HTTP_ACL_BEARER_REQUIRED, or
41
- // 2. netdata_is_protected_by_bearer == true
42
-
43
- BEARER_STATUS t = api_check_bearer_token(w);
44
- if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED) {
45
- // we have a valid bearer on the request
46
- w->access = HTTP_ACCESS_MEMBERS;
47
- return HTTP_RESP_OK;
48
- }
49
-
50
- if(endpoint_acl & HTTP_ACL_BEARER_OPTIONAL) {
5
+int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_path_endpoint, struct web_api_command *api_commands) {
6
+ if(!web_client_flags_check_auth(w))
7
w->access = HTTP_ACCESS_ANY;
52
- return HTTP_RESP_OK;
53
- }
54
-
55
- netdata_log_info("BEARER: bearer is required for request: code %d", t);
8
57
- return HTTP_RESP_PRECOND_FAIL;
58
-}
9
+#ifdef NETDATA_GOD_MODE
10
+ web_client_flag_set(w, WEB_CLIENT_FLAG_AUTH_GOD);
11
+ w->access = HTTP_ACCESS_ADMIN;
12
+#endif
13
60
-int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_path_endpoint, struct web_api_command *api_commands) {
14
buffer_no_cacheable(w->response.data);
15
16
if(unlikely(!url_path_endpoint || !*url_path_endpoint)) {
@@ -89,19 +42,9 @@ int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_pat
42
if (api_command != url_path_endpoint)
43
freez(api_command);
44
92
- short int code = web_client_check_acl_and_bearer(w, api_commands[i].acl);
93
- if(code != HTTP_RESP_OK) {
94
- if(code == HTTP_RESP_FORBIDDEN)
95
- return web_client_permission_denied(w);
96
-
97
- if(code == HTTP_RESP_PRECOND_FAIL)
98
- return web_client_bearer_required(w);
99
-
100
- buffer_flush(w->response.data);
101
- buffer_sprintf(w->response.data, "Failed with code %d", code);
102
- w->response.code = code;
103
- return code;
104
- }
45
+ bool acl_allows = (w->acl & api_commands[i].acl) || (api_commands[i].acl & HTTP_ACL_NOCHECK);
46
+ if(!acl_allows)
47
+ return web_client_permission_denied(w);
48
49
char *query_string = (char *)buffer_tostring(w->url_query_string_decoded);
50
web/api/web_api.h
+2
-16
@@ -4,28 +4,14 @@
4
#define NETDATA_WEB_API_H 1
5
6
#include "daemon/common.h"
7
+#include "web/api/http_header.h"
8
+#include "web/api/http_auth.h"
9
#include "web/api/badges/web_buffer_svg.h"
10
#include "web/api/ilove/ilove.h"
11
#include "web/api/formatters/rrd2json.h"
12
#include "web/api/health/health_cmdapi.h"
13
#include "web/api/queries/weights.h"
14
13
-extern bool netdata_is_protected_by_bearer;
14
-extern DICTIONARY *netdata_authorized_bearers;
15
-typedef enum __attribute__((packed)) {
16
- BEARER_STATUS_NO_BEARER_IN_HEADERS,
17
- BEARER_STATUS_BEARER_DOES_NOT_FIT,
18
- BEARER_STATUS_NOT_PARSABLE,
19
- BEARER_STATUS_EXTRACTED_FROM_HEADER,
20
- BEARER_STATUS_NO_BEARERS_DICTIONARY,
21
- BEARER_STATUS_NOT_FOUND_IN_DICTIONARY,
22
- BEARER_STATUS_EXPIRED,
23
- BEARER_STATUS_AVAILABLE_AND_VALIDATED,
24
-} BEARER_STATUS;
25
-
26
-BEARER_STATUS api_check_bearer_token(struct web_client *w);
27
-BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len);
28
-
15
struct web_api_command {
16
const char *command;
17
uint32_t hash;
web/api/web_api_v1.c
+126
-10
@@ -108,13 +108,15 @@ static struct {
108
uint32_t hash;
109
DATASOURCE_FORMAT value;
110
} api_v1_data_google_formats[] = {
111
- // this is not error - when google requests json, it expects javascript
112
- // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source#responseformat
113
- { "json" , 0 , DATASOURCE_DATATABLE_JSONP}
114
- , {"html" , 0 , DATASOURCE_HTML}
115
- , {"csv" , 0 , DATASOURCE_CSV}
116
- , {"tsv-excel", 0 , DATASOURCE_TSV}
117
- , { NULL, 0, 0}
111
+ // this is not an error - when Google requests json, it expects javascript
112
+ // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source#responseformat
113
+ {"json", 0, DATASOURCE_DATATABLE_JSONP}
114
+ , {"html", 0, DATASOURCE_HTML}
115
+ , {"csv", 0, DATASOURCE_CSV}
116
+ , {"tsv-excel", 0, DATASOURCE_TSV}
117
+
118
+ // terminator
119
+ , {NULL, 0, 0}
120
};
121
122
void web_client_api_v1_init(void) {
@@ -857,7 +859,7 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
859
responseHandler,
860
google_version,
861
google_reqId,
860
- (int64_t)st->last_updated.tv_sec);
862
+ (int64_t)(st ? st->last_updated.tv_sec : 0));
863
}
864
else if(format == DATASOURCE_JSONP) {
865
if(responseHandler == NULL)
@@ -942,7 +944,7 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *
944
char *cookie = strstr(w->response.data->buffer, NETDATA_REGISTRY_COOKIE_NAME "=");
945
if(cookie)
946
strncpyz(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME)], UUID_STR_LEN - 1);
945
- else if(extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)) != BEARER_STATUS_EXTRACTED_FROM_HEADER)
947
+ else if(!extract_bearer_token_from_request(w, person_guid, sizeof(person_guid)))
948
person_guid[0] = '\0';
949
950
char action = '\0';
@@ -1418,10 +1420,14 @@ int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char
1420
char transaction[UUID_COMPACT_STR_LEN];
1421
uuid_unparse_lower_compact(w->transaction, transaction);
1422
1423
+ CLEAN_BUFFER *source = buffer_create(100, NULL);
1424
+ web_client_source2buffer(w, source);
1425
+
1426
return rrd_function_run(host, wb, timeout, w->access, function, true, transaction,
1427
NULL, NULL,
1428
web_client_progress_functions_update, w,
1424
- web_client_interrupt_callback, w, NULL);
1429
+ web_client_interrupt_callback, w, NULL,
1430
+ buffer_tostring(source));
1431
}
1432
1433
int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, char *url __maybe_unused) {
@@ -1440,6 +1446,114 @@ int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, cha
1446
return HTTP_RESP_OK;
1447
}
1448
1449
+void web_client_source2buffer(struct web_client *w, BUFFER *source) {
1450
+ if(web_client_flag_check(w, WEB_CLIENT_FLAG_AUTH_CLOUD))
1451
+ buffer_sprintf(source, "method=NC");
1452
+ else if(web_client_flag_check(w, WEB_CLIENT_FLAG_AUTH_BEARER))
1453
+ buffer_sprintf(source, "method=api-bearer");
1454
+ else
1455
+ buffer_sprintf(source, "method=api");
1456
+
1457
+ if(web_client_flag_check(w, WEB_CLIENT_FLAG_AUTH_CLOUD) || web_client_flag_check(w, WEB_CLIENT_FLAG_AUTH_BEARER)) {
1458
+ buffer_sprintf(source, ",role=%s", http_id2access(w->access));
1459
+
1460
+ char uuid_str[UUID_COMPACT_STR_LEN];
1461
+ uuid_unparse_lower_compact(w->auth.cloud_account_id, uuid_str);
1462
+ buffer_sprintf(source, ",user=%s,account=%s", w->auth.client_name, uuid_str);
1463
+ }
1464
+ else if(web_client_flag_check(w, WEB_CLIENT_FLAG_AUTH_GOD))
1465
+ buffer_strcat(source, ",role=god");
1466
+ else
1467
+ buffer_sprintf(source, ",role=%s", http_id2access(w->access));
1468
+
1469
+ if(w->client_ip[0])
1470
+ buffer_sprintf(source, ",ip=%s", w->client_ip);
1471
+
1472
+ if(w->forwarded_for)
1473
+ buffer_sprintf(source, ",forwarded_for=%s", w->forwarded_for);
1474
+}
1475
+
1476
+static int web_client_api_request_v1_config(RRDHOST *host, struct web_client *w, char *url __maybe_unused) {
1477
+ char *action = "tree";
1478
+ char *path = "/";
1479
+ char *id = NULL;
1480
+ char *add_name = NULL;
1481
+ int timeout = 120;
1482
+
1483
+ while(url) {
1484
+ char *value = strsep_skip_consecutive_separators(&url, "&");
1485
+ if(!value || !*value) continue;
1486
+
1487
+ char *name = strsep_skip_consecutive_separators(&value, "=");
1488
+ if(!name || !*name) continue;
1489
+ if(!value || !*value) continue;
1490
+
1491
+ // name and value are now the parameters
1492
+ // they are not null and not empty
1493
+
1494
+ if(!strcmp(name, "action"))
1495
+ action = value;
1496
+ else if(!strcmp(name, "path"))
1497
+ path = value;
1498
+ else if(!strcmp(name, "id"))
1499
+ id = value;
1500
+ else if(!strcmp(name, "name"))
1501
+ add_name = value;
1502
+ else if(!strcmp(name, "timeout")) {
1503
+ timeout = (int)strtol(value, NULL, 10);
1504
+ if(timeout < 10)
1505
+ timeout = 10;
1506
+ }
1507
+ }
1508
+
1509
+ char transaction[UUID_COMPACT_STR_LEN];
1510
+ uuid_unparse_lower_compact(w->transaction, transaction);
1511
+
1512
+ size_t len = (action ? strlen(action) : 0)
1513
+ + (id ? strlen(id) : 0)
1514
+ + (path ? strlen(path) : 0)
1515
+ + (add_name ? strlen(add_name) : 0)
1516
+ + 100;
1517
+
1518
+ char cmd[len];
1519
+ if(strcmp(action, "tree") == 0)
1520
+ snprintfz(cmd, sizeof(cmd), PLUGINSD_FUNCTION_CONFIG " tree '%s' '%s'", path, id?id:"");
1521
+ else {
1522
+ DYNCFG_CMDS c = dyncfg_cmds2id(action);
1523
+ if(!id || !*id || !dyncfg_is_valid_id(id)) {
1524
+ rrd_call_function_error(w->response.data, "invalid id given", HTTP_RESP_BAD_REQUEST);
1525
+ return HTTP_RESP_BAD_REQUEST;
1526
+ }
1527
+ if(c == DYNCFG_CMD_NONE) {
1528
+ rrd_call_function_error(w->response.data, "invalid action given", HTTP_RESP_BAD_REQUEST);
1529
+ return HTTP_RESP_BAD_REQUEST;
1530
+ }
1531
+ else if(c == DYNCFG_CMD_ADD) {
1532
+ if(!add_name || !*add_name || !dyncfg_is_valid_id(add_name)) {
1533
+ rrd_call_function_error(w->response.data, "invalid name given", HTTP_RESP_BAD_REQUEST);
1534
+ return HTTP_RESP_BAD_REQUEST;
1535
+ }
1536
+ snprintfz(cmd, sizeof(cmd), PLUGINSD_FUNCTION_CONFIG " %s %s %s", id, dyncfg_id2cmd_one(c), add_name);
1537
+ }
1538
+ else
1539
+ snprintfz(cmd, sizeof(cmd), PLUGINSD_FUNCTION_CONFIG " %s %s", id, dyncfg_id2cmd_one(c));
1540
+ }
1541
+
1542
+ CLEAN_BUFFER *source = buffer_create(100, NULL);
1543
+ web_client_source2buffer(w, source);
1544
+
1545
+ buffer_flush(w->response.data);
1546
+ int code = rrd_function_run(host, w->response.data, timeout, w->access, cmd,
1547
+ true, transaction,
1548
+ NULL, NULL,
1549
+ web_client_progress_functions_update, w,
1550
+ web_client_interrupt_callback, w,
1551
+ w->payload,
1552
+ buffer_tostring(source));
1553
+
1554
+ return code;
1555
+}
1556
+
1557
#ifndef ENABLE_DBENGINE
1558
int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struct web_client *w __maybe_unused, char *url __maybe_unused) {
1559
return HTTP_RESP_NOT_FOUND;
@@ -1586,6 +1700,8 @@ static struct web_api_command api_commands_v1[] = {
1700
{"function", 0, HTTP_ACL_ACLK_WEBRTC_DASHBOARD_WITH_OPTIONAL_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function, 0 },
1701
{"functions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions, 0 },
1702
1703
+ {"config", 0, HTTP_ACL_ACLK_WEBRTC_DASHBOARD_WITH_OPTIONAL_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_config, 0 },
1704
+
1705
{"dbengine_stats", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats, 0 },
1706
1707
// terminator
web/api/web_api_v1.h
+2
@@ -38,6 +38,8 @@ void web_client_api_v1_management_init(void);
38
void host_labels2json(RRDHOST *host, BUFFER *wb, const char *key);
39
void web_client_api_request_v1_info_summary_alarm_statuses(RRDHOST *host, BUFFER *wb, const char *key);
40
41
+void web_client_source2buffer(struct web_client *w, BUFFER *source);
42
+
43
extern char *api_secret;
44
45
#endif //NETDATA_WEB_API_V1_H
web/api/web_api_v2.c
+2
-281
@@ -3,112 +3,6 @@
3
#include "web_api_v2.h"
4
#include "../rtc/webrtc.h"
5
6
-#define BEARER_TOKEN_EXPIRATION 86400
7
-
8
-struct bearer_token {
9
- time_t created_s;
10
- time_t expires_s;
11
-};
12
-
13
-static void bearer_token_cleanup(void) {
14
- static time_t attempts = 0;
15
-
16
- if(++attempts % 1000 != 0)
17
- return;
18
-
19
- time_t now_s = now_monotonic_sec();
20
-
21
- struct bearer_token *z;
22
- dfe_start_read(netdata_authorized_bearers, z) {
23
- if(z->expires_s < now_s)
24
- dictionary_del(netdata_authorized_bearers, z_dfe.name);
25
- }
26
- dfe_done(z);
27
-
28
- dictionary_garbage_collect(netdata_authorized_bearers);
29
-}
30
-
31
-void bearer_tokens_init(void) {
32
- netdata_authorized_bearers = dictionary_create_advanced(
33
- DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
34
- NULL, sizeof(struct bearer_token));
35
-}
36
-
37
-static time_t bearer_get_token(uuid_t *uuid) {
38
- char uuid_str[UUID_STR_LEN];
39
-
40
- uuid_generate_random(*uuid);
41
- uuid_unparse_lower(*uuid, uuid_str);
42
-
43
- struct bearer_token t = { 0 }, *z;
44
- z = dictionary_set(netdata_authorized_bearers, uuid_str, &t, sizeof(t));
45
- if(!z->created_s) {
46
- z->created_s = now_monotonic_sec();
47
- z->expires_s = z->created_s + BEARER_TOKEN_EXPIRATION;
48
- }
49
-
50
- bearer_token_cleanup();
51
-
52
- return now_realtime_sec() + BEARER_TOKEN_EXPIRATION;
53
-}
54
-
55
-#define HTTP_REQUEST_AUTHORIZATION_BEARER "\r\nAuthorization: Bearer "
56
-#define HTTP_REQUEST_X_NETDATA_AUTH_BEARER "\r\nX-Netdata-Auth: Bearer "
57
-
58
-BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
59
- const char *req = buffer_tostring(w->response.data);
60
- size_t req_len = buffer_strlen(w->response.data);
61
- const char *bearer = NULL;
62
- const char *bearer_end = NULL;
63
-
64
- bearer = strcasestr(req, HTTP_REQUEST_X_NETDATA_AUTH_BEARER);
65
- if(bearer)
66
- bearer_end = bearer + sizeof(HTTP_REQUEST_X_NETDATA_AUTH_BEARER) - 1;
67
- else {
68
- bearer = strcasestr(req, HTTP_REQUEST_AUTHORIZATION_BEARER);
69
- if(bearer)
70
- bearer_end = bearer + sizeof(HTTP_REQUEST_AUTHORIZATION_BEARER) - 1;
71
- }
72
-
73
- if(!bearer || !bearer_end)
74
- return BEARER_STATUS_NO_BEARER_IN_HEADERS;
75
-
76
- const char *token_start = bearer_end;
77
-
78
- while(isspace(*token_start))
79
- token_start++;
80
-
81
- const char *token_end = token_start + UUID_STR_LEN - 1 + 2;
82
- if (token_end > req + req_len)
83
- return BEARER_STATUS_BEARER_DOES_NOT_FIT;
84
-
85
- strncpyz(dst, token_start, dst_len - 1);
86
- uuid_t uuid;
87
- if (uuid_parse(dst, uuid) != 0)
88
- return BEARER_STATUS_NOT_PARSABLE;
89
-
90
- return BEARER_STATUS_EXTRACTED_FROM_HEADER;
91
-}
92
-
93
-BEARER_STATUS api_check_bearer_token(struct web_client *w) {
94
- if(!netdata_authorized_bearers)
95
- return BEARER_STATUS_NO_BEARERS_DICTIONARY;
96
-
97
- char token[UUID_STR_LEN];
98
- BEARER_STATUS t = extract_bearer_token_from_request(w, token, sizeof(token));
99
- if(t != BEARER_STATUS_EXTRACTED_FROM_HEADER)
100
- return t;
101
-
102
- struct bearer_token *z = dictionary_get(netdata_authorized_bearers, token);
103
- if(!z)
104
- return BEARER_STATUS_NOT_FOUND_IN_DICTIONARY;
105
-
106
- if(z->expires_s < now_monotonic_sec())
107
- return BEARER_STATUS_EXPIRED;
108
-
109
- return BEARER_STATUS_AVAILABLE_AND_VALIDATED;
110
-}
111
-
6
static bool verify_agent_uuids(const char *machine_guid, const char *node_id, const char *claim_id) {
7
if(!machine_guid || !node_id || !claim_id)
8
return false;
@@ -206,7 +100,7 @@ int api_v2_bearer_token(RRDHOST *host __maybe_unused, struct web_client *w __may
100
}
101
102
uuid_t uuid;
209
- time_t expires_s = bearer_get_token(&uuid);
103
+ time_t expires_s = bearer_create_token(&uuid, w);
104
105
BUFFER *wb = w->response.data;
106
buffer_flush(wb);
@@ -661,7 +555,7 @@ cleanup:
555
}
556
557
static int web_client_api_request_v2_webrtc(RRDHOST *host __maybe_unused, struct web_client *w, char *url __maybe_unused) {
664
- return webrtc_new_connection(w->post_payload, w->response.data);
558
+ return webrtc_new_connection(buffer_tostring(w->payload), w->response.data);
559
}
560
561
static int web_client_api_request_v2_progress(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
@@ -689,176 +583,6 @@ static int web_client_api_request_v2_progress(RRDHOST *host __maybe_unused, stru
583
return web_api_v2_report_progress(&tr, w->response.data);
584
}
585
692
-#define CONFIG_API_V2_URL "/api/v2/config"
693
-static int web_client_api_request_v2_config(RRDHOST *host __maybe_unused, struct web_client *w, char *query __maybe_unused) {
694
-
695
- char *url = strdupz(buffer_tostring(w->url_as_received));
696
- char *url_full = url;
697
-
698
- buffer_flush(w->response.data);
699
-
700
- if (strncmp(url, "/host/", strlen("/host/")) == 0) {
701
- url += strlen("/host/");
702
- char *host_id_end = strchr(url, '/');
703
- if (host_id_end == NULL) {
704
- buffer_sprintf(w->response.data, "Invalid URL");
705
- freez(url_full);
706
- return HTTP_RESP_BAD_REQUEST;
707
- }
708
- url += host_id_end - url;
709
- }
710
-
711
- if (strncmp(url, CONFIG_API_V2_URL, strlen(CONFIG_API_V2_URL)) != 0) {
712
- buffer_sprintf(w->response.data, "Invalid URL");
713
- freez(url_full);
714
- return HTTP_RESP_BAD_REQUEST;
715
- }
716
- url += strlen(CONFIG_API_V2_URL);
717
-
718
- char *save_ptr = NULL;
719
- char *plugin = strtok_r(url, "/", &save_ptr);
720
- char *module = strtok_r(NULL, "/", &save_ptr);
721
- char *job_id = strtok_r(NULL, "/", &save_ptr);
722
- char *extra = strtok_r(NULL, "/", &save_ptr);
723
-
724
- if (extra != NULL) {
725
- buffer_sprintf(w->response.data, "Invalid URL");
726
- freez(url_full);
727
- return HTTP_RESP_BAD_REQUEST;
728
- }
729
-
730
- int http_method;
731
- switch (w->mode)
732
- {
733
- case HTTP_REQUEST_MODE_GET:
734
- case HTTP_REQUEST_MODE_POST:
735
- case HTTP_REQUEST_MODE_PUT:
736
- case HTTP_REQUEST_MODE_DELETE:
737
- http_method = w->mode;
738
- break;
739
- default:
740
- buffer_sprintf(w->response.data, "Invalid HTTP method");
741
- freez(url_full);
742
- return HTTP_RESP_BAD_REQUEST;
743
- }
744
-
745
- struct uni_http_response resp = dyn_conf_process_http_request(host->configurable_plugins, http_method, plugin, module, job_id, w->post_payload, w->post_payload_size);
746
- if (resp.content[resp.content_length - 1] != '\0') {
747
- char *con = mallocz(resp.content_length + 1);
748
- memcpy(con, resp.content, resp.content_length);
749
- con[resp.content_length] = '\0';
750
- if (resp.content_free)
751
- resp.content_free(resp.content);
752
- resp.content = con;
753
- resp.content_free = freez_dyncfg;
754
- }
755
- buffer_strcat(w->response.data, resp.content);
756
- if (resp.content_free)
757
- resp.content_free(resp.content);
758
- w->response.data->content_type = resp.content_type;
759
- freez(url_full);
760
- return resp.status;
761
-}
762
-
763
-static json_object *job_statuses_grouped() {
764
- json_object *top_obj = json_object_new_object();
765
- json_object *host_vec = json_object_new_array();
766
-
767
-
768
- RRDHOST *host;
769
-
770
- dfe_start_reentrant(rrdhost_root_index, host) {
771
- json_object *host_obj = json_object_new_object();
772
- json_object *host_sub_obj = json_object_new_string(host->machine_guid);
773
- json_object_object_add(host_obj, "host_guid", host_sub_obj);
774
- host_sub_obj = json_object_new_array();
775
-
776
- DICTIONARY *plugins_dict = host->configurable_plugins;
777
-
778
- struct configurable_plugin *plugin;
779
- dfe_start_read(plugins_dict, plugin) {
780
- json_object *plugin_obj = json_object_new_object();
781
- json_object *plugin_sub_obj = json_object_new_string(plugin->name);
782
- json_object_object_add(plugin_obj, "name", plugin_sub_obj);
783
- plugin_sub_obj = json_object_new_array();
784
-
785
- struct module *module;
786
- dfe_start_read(plugin->modules, module) {
787
- json_object *module_obj = json_object_new_object();
788
- json_object *module_sub_obj = json_object_new_string(module->name);
789
- json_object_object_add(module_obj, "name", module_sub_obj);
790
- module_sub_obj = json_object_new_array();
791
-
792
- struct job *job;
793
- dfe_start_read(module->jobs, job) {
794
- json_object *job_obj = json_object_new_object();
795
- json_object *job_sub_obj = json_object_new_string(job->name);
796
- json_object_object_add(job_obj, "name", job_sub_obj);
797
- job_sub_obj = job2json(job);
798
- json_object_object_add(job_obj, "job", job_sub_obj);
799
- json_object_array_add(module_sub_obj, job_obj);
800
- } dfe_done(job);
801
- json_object_object_add(module_obj, "jobs", module_sub_obj);
802
- json_object_array_add(plugin_sub_obj, module_obj);
803
- } dfe_done(module);
804
- json_object_object_add(plugin_obj, "modules", plugin_sub_obj);
805
- json_object_array_add(host_sub_obj, plugin_obj);
806
- } dfe_done(plugin);
807
- json_object_object_add(host_obj, "plugins", host_sub_obj);
808
- json_object_array_add(host_vec, host_obj);
809
- }
810
- dfe_done(host);
811
-
812
- json_object_object_add(top_obj, "hosts", host_vec);
813
- return top_obj;
814
-}
815
-
816
-static json_object *job_statuses_flat() {
817
- RRDHOST *host;
818
-
819
- json_object *ret = json_object_new_array();
820
-
821
- dfe_start_reentrant(rrdhost_root_index, host) {
822
- DICTIONARY *plugins_dict = host->configurable_plugins;
823
-
824
- struct configurable_plugin *plugin;
825
- dfe_start_read(plugins_dict, plugin) {
826
- struct module *module;
827
- dfe_start_read(plugin->modules, module) {
828
- struct job *job;
829
- dfe_start_read(module->jobs, job) {
830
- json_object *job_rich = json_object_new_object();
831
- json_object *obj = json_object_new_string(host->machine_guid);
832
- json_object_object_add(job_rich, "host_guid", obj);
833
- obj = json_object_new_string(plugin->name);
834
- json_object_object_add(job_rich, "plugin_name", obj);
835
- obj = json_object_new_string(module->name);
836
- json_object_object_add(job_rich, "module_name", obj);
837
- obj = job2json(job);
838
- json_object_object_add(job_rich, "job", obj);
839
- json_object_array_add(ret, job_rich);
840
- } dfe_done(job);
841
- } dfe_done(module);
842
- } dfe_done(plugin);
843
- }
844
- dfe_done(host);
845
-
846
- return ret;
847
-}
848
-
849
-static int web_client_api_request_v2_job_statuses(RRDHOST *host __maybe_unused, struct web_client *w, char *query) {
850
- json_object *json;
851
- if (strstr(query, "grouped") != NULL)
852
- json = job_statuses_grouped();
853
- else
854
- json = job_statuses_flat();
855
-
856
- buffer_flush(w->response.data);
857
- buffer_strcat(w->response.data, json_object_to_json_string_ext(json, JSON_C_TO_STRING_PRETTY));
858
- w->response.data->content_type = CT_APPLICATION_JSON;
859
- return HTTP_RESP_OK;
860
-}
861
-
586
static struct web_api_command api_commands_v2[] = {
587
{"info", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info, 0},
588
@@ -882,9 +606,6 @@ static struct web_api_command api_commands_v2[] = {
606
{"bearer_protection", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection, 0},
607
{"bearer_get_token", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token, 0},
608
885
- {"config", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_config, 1},
886
- {"job_statuses", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_job_statuses, 0},
887
-
609
{ "ilove.svg", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_ilove, 0 },
610
{ "progress", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_progress, 0 },
611
web/rtc/webrtc.c
+1
@@ -292,6 +292,7 @@ static void webrtc_execute_api_request(WEBRTC_DC *chan, const char *request, siz
292
w->statistics.received_bytes = size;
293
w->interrupt.callback = web_client_stop_callback;
294
w->interrupt.callback_data = chan;
295
+ web_client_set_conn_webrtc(w);
296
297
w->acl = HTTP_ACL_WEBRTC;
298
web/server/static/static-threaded.c
+5
-4
@@ -40,7 +40,8 @@ static struct web_client *web_client_create_on_fd(POLLINFO *pi) {
40
w->port_acl = pi->port_acl;
41
42
int flag = 1;
43
- if(unlikely(web_client_check_tcp(w) && setsockopt(w->ifd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0))
43
+ if(unlikely(
44
+ web_client_check_conn_tcp(w) && setsockopt(w->ifd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0))
45
netdata_log_debug(D_WEB_CLIENT, "%llu: failed to enable TCP_NODELAY on socket fd %d.", w->id, w->ifd);
46
47
flag = 1;
@@ -205,13 +206,13 @@ static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data
206
struct web_client *w = web_client_create_on_fd(pi);
207
208
if (!strncmp(pi->client_port, "UNIX", 4)) {
208
- web_client_set_unix(w);
209
+ web_client_set_conn_unix(w);
210
} else {
210
- web_client_set_tcp(w);
211
+ web_client_set_conn_tcp(w);
212
}
213
214
#ifdef ENABLE_HTTPS
214
- if ((!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx)) {
215
+ if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) {
216
sock_delnonblock(w->ifd);
217
218
//Read the first 7 bytes from the message, but the message
web/server/web_client.c
+89
-268
@@ -10,6 +10,26 @@ char *web_x_frame_options = NULL;
10
11
int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRATEGY;
12
13
+void web_client_set_conn_tcp(struct web_client *w) {
14
+ web_client_flags_clear_conn(w);
15
+ web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_TCP);
16
+}
17
+
18
+void web_client_set_conn_unix(struct web_client *w) {
19
+ web_client_flags_clear_conn(w);
20
+ web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_UNIX);
21
+}
22
+
23
+void web_client_set_conn_cloud(struct web_client *w) {
24
+ web_client_flags_clear_conn(w);
25
+ web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_CLOUD);
26
+}
27
+
28
+void web_client_set_conn_webrtc(struct web_client *w) {
29
+ web_client_flags_clear_conn(w);
30
+ web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_WEBRTC);
31
+}
32
+
33
inline int web_client_permission_denied(struct web_client *w) {
34
w->response.data->content_type = CT_TEXT_PLAIN;
35
buffer_flush(w->response.data);
@@ -36,7 +56,7 @@ static inline int bad_request_multiple_dashboard_versions(struct web_client *w)
56
57
static inline int web_client_cork_socket(struct web_client *w __maybe_unused) {
58
#ifdef TCP_CORK
39
- if(likely(web_client_is_corkable(w) && !w->tcp_cork && w->ofd != -1)) {
59
+ if(likely(web_client_check_conn_tcp(w) && !w->tcp_cork && w->ofd != -1)) {
60
w->tcp_cork = true;
61
if(unlikely(setsockopt(w->ofd, IPPROTO_TCP, TCP_CORK, (char *) &w->tcp_cork, sizeof(int)) != 0)) {
62
netdata_log_error("%llu: failed to enable TCP_CORK on socket.", w->id);
@@ -111,9 +131,8 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) {
131
buffer_free(w->response.data);
132
w->response.data = NULL;
133
114
- freez(w->post_payload);
115
- w->post_payload = NULL;
116
- w->post_payload_size = 0;
134
+ buffer_free(w->payload);
135
+ w->payload = NULL;
136
}
137
else {
138
// the web client is to be re-used
@@ -126,7 +145,11 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) {
145
buffer_reset(w->response.header);
146
buffer_reset(w->response.data);
147
129
- // leave w->post_payload
148
+ if(w->payload)
149
+ buffer_reset(w->payload);
150
+
151
+ // to add more items here,
152
+ // web_client_reuse_from_cache() needs to be adjusted to maintain them
153
}
154
155
freez(w->server_host);
@@ -157,9 +180,11 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) {
180
w->response.zstream.total_in = 0;
181
w->response.zstream.total_out = 0;
182
w->response.zinitialized = false;
160
- w->flags &= ~WEB_CLIENT_CHUNKED_TRANSFER;
183
+ web_client_flag_clear(w, WEB_CLIENT_CHUNKED_TRANSFER);
184
}
185
186
+ web_client_flags_check_auth(w);
187
+ web_client_flag_clear(w, WEB_CLIENT_ENCODING_GZIP|WEB_CLIENT_ENCODING_DEFLATE);
188
web_client_reset_path_flags(w);
189
}
190
@@ -194,6 +219,12 @@ void web_client_log_completed_request(struct web_client *w, bool update_web_stat
219
ND_LOG_FIELD_U64(NDF_RESPONSE_PREPARATION_TIME_USEC, prep_ut),
220
ND_LOG_FIELD_U64(NDF_RESPONSE_SENT_TIME_USEC, sent_ut),
221
ND_LOG_FIELD_U64(NDF_RESPONSE_TOTAL_TIME_USEC, total_ut),
222
+ ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip),
223
+ ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
224
+ ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for),
225
+ ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->auth.cloud_account_id),
226
+ ND_LOG_FIELD_TXT(NDF_USER_NAME, w->auth.client_name),
227
+ ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2access(w->access)),
228
ND_LOG_FIELD_END(),
229
};
230
ND_LOG_STACK_PUSH(lgs);
@@ -218,14 +249,6 @@ void web_client_log_completed_request(struct web_client *w, bool update_web_stat
249
}
250
251
void web_client_request_done(struct web_client *w) {
221
- ND_LOG_STACK lgs[] = {
222
- ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip),
223
- ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for),
224
- ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
225
- ND_LOG_FIELD_END(),
226
- };
227
- ND_LOG_STACK_PUSH(lgs);
228
-
252
web_client_uncork_socket(w);
253
254
netdata_log_debug(D_WEB_CLIENT, "%llu: Resetting client.", w->id);
@@ -503,52 +526,6 @@ static int mysendfile(struct web_client *w, char *filename) {
526
}
527
#endif
528
506
-void web_client_enable_deflate(struct web_client *w, int gzip) {
507
- if(unlikely(w->response.zinitialized)) {
508
- netdata_log_debug(D_DEFLATE, "%llu: Compression has already be initialized for this client.", w->id);
509
- return;
510
- }
511
-
512
- if(unlikely(w->response.sent)) {
513
- netdata_log_error("%llu: Cannot enable compression in the middle of a conversation.", w->id);
514
- return;
515
- }
516
-
517
- w->response.zstream.zalloc = Z_NULL;
518
- w->response.zstream.zfree = Z_NULL;
519
- w->response.zstream.opaque = Z_NULL;
520
-
521
- w->response.zstream.next_in = (Bytef *)w->response.data->buffer;
522
- w->response.zstream.avail_in = 0;
523
- w->response.zstream.total_in = 0;
524
-
525
- w->response.zstream.next_out = w->response.zbuffer;
526
- w->response.zstream.avail_out = 0;
527
- w->response.zstream.total_out = 0;
528
-
529
- w->response.zstream.zalloc = Z_NULL;
530
- w->response.zstream.zfree = Z_NULL;
531
- w->response.zstream.opaque = Z_NULL;
532
-
533
-// if(deflateInit(&w->response.zstream, Z_DEFAULT_COMPRESSION) != Z_OK) {
534
-// netdata_log_error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
535
-// return;
536
-// }
537
-
538
- // Select GZIP compression: windowbits = 15 + 16 = 31
539
- if(deflateInit2(&w->response.zstream, web_gzip_level, Z_DEFLATED, 15 + ((gzip)?16:0), 8, web_gzip_strategy) != Z_OK) {
540
- netdata_log_error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
541
- return;
542
- }
543
-
544
- w->response.zsent = 0;
545
- w->response.zoutput = true;
546
- w->response.zinitialized = true;
547
- w->flags |= WEB_CLIENT_CHUNKED_TRANSFER;
548
-
549
- netdata_log_debug(D_DEFLATE, "%llu: Initialized compression.", w->id);
550
-}
551
-
529
void buffer_data_options2string(BUFFER *wb, uint32_t options) {
530
int count = 0;
531
@@ -653,6 +630,9 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_f
630
ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
631
ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
632
ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
633
+ ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->auth.cloud_account_id),
634
+ ND_LOG_FIELD_TXT(NDF_USER_NAME, w->auth.client_name),
635
+ ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2access(w->access)),
636
ND_LOG_FIELD_END(),
637
};
638
ND_LOG_STACK_PUSH(lgs);
@@ -661,7 +641,7 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_f
641
web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
642
query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl,
643
buffer_tostring(w->url_as_received),
664
- w->post_payload,
644
+ w->payload,
645
w->forwarded_for ? w->forwarded_for : w->client_ip);
646
}
647
@@ -687,191 +667,6 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_f
667
}
668
}
669
690
-const char *web_content_type_to_string(HTTP_CONTENT_TYPE content_type) {
691
- switch(content_type) {
692
- case CT_TEXT_HTML:
693
- return "text/html; charset=utf-8";
694
-
695
- case CT_APPLICATION_XML:
696
- return "application/xml; charset=utf-8";
697
-
698
- case CT_APPLICATION_JSON:
699
- return "application/json; charset=utf-8";
700
-
701
- case CT_APPLICATION_X_JAVASCRIPT:
702
- return "application/javascript; charset=utf-8";
703
-
704
- case CT_TEXT_CSS:
705
- return "text/css; charset=utf-8";
706
-
707
- case CT_TEXT_XML:
708
- return "text/xml; charset=utf-8";
709
-
710
- case CT_TEXT_XSL:
711
- return "text/xsl; charset=utf-8";
712
-
713
- case CT_APPLICATION_OCTET_STREAM:
714
- return "application/octet-stream";
715
-
716
- case CT_IMAGE_SVG_XML:
717
- return "image/svg+xml";
718
-
719
- case CT_APPLICATION_X_FONT_TRUETYPE:
720
- return "application/x-font-truetype";
721
-
722
- case CT_APPLICATION_X_FONT_OPENTYPE:
723
- return "application/x-font-opentype";
724
-
725
- case CT_APPLICATION_FONT_WOFF:
726
- return "application/font-woff";
727
-
728
- case CT_APPLICATION_FONT_WOFF2:
729
- return "application/font-woff2";
730
-
731
- case CT_APPLICATION_VND_MS_FONTOBJ:
732
- return "application/vnd.ms-fontobject";
733
-
734
- case CT_IMAGE_PNG:
735
- return "image/png";
736
-
737
- case CT_IMAGE_JPG:
738
- return "image/jpeg";
739
-
740
- case CT_IMAGE_GIF:
741
- return "image/gif";
742
-
743
- case CT_IMAGE_XICON:
744
- return "image/x-icon";
745
-
746
- case CT_IMAGE_BMP:
747
- return "image/bmp";
748
-
749
- case CT_IMAGE_ICNS:
750
- return "image/icns";
751
-
752
- case CT_PROMETHEUS:
753
- return "text/plain; version=0.0.4";
754
-
755
- case CT_AUDIO_MPEG:
756
- return "audio/mpeg";
757
-
758
- case CT_AUDIO_OGG:
759
- return "audio/ogg";
760
-
761
- case CT_VIDEO_MP4:
762
- return "video/mp4";
763
-
764
- case CT_APPLICATION_PDF:
765
- return "application/pdf";
766
-
767
- case CT_APPLICATION_ZIP:
768
- return "application/zip";
769
-
770
- default:
771
- case CT_TEXT_PLAIN:
772
- return "text/plain; charset=utf-8";
773
- }
774
-}
775
-
776
-static inline char *http_header_parse(struct web_client *w, char *s, int parse_useragent) {
777
- static uint32_t hash_origin = 0, hash_connection = 0, hash_donottrack = 0, hash_useragent = 0,
778
- hash_authorization = 0, hash_host = 0, hash_forwarded_host = 0, hash_forwarded_for = 0,
779
- hash_transaction_id = 0;
780
- static uint32_t hash_accept_encoding = 0;
781
-
782
- if(unlikely(!hash_origin)) {
783
- hash_origin = simple_uhash("Origin");
784
- hash_connection = simple_uhash("Connection");
785
- hash_accept_encoding = simple_uhash("Accept-Encoding");
786
- hash_donottrack = simple_uhash("DNT");
787
- hash_useragent = simple_uhash("User-Agent");
788
- hash_authorization = simple_uhash("X-Auth-Token");
789
- hash_host = simple_uhash("Host");
790
- hash_forwarded_host = simple_uhash("X-Forwarded-Host");
791
- hash_forwarded_for = simple_uhash("X-Forwarded-For");
792
- hash_transaction_id = simple_uhash("X-Transaction-ID");
793
- }
794
-
795
- char *e = s;
796
-
797
- // find the :
798
- while(*e && *e != ':') e++;
799
- if(!*e) return e;
800
-
801
- // get the name
802
- *e = '\0';
803
-
804
- // find the value
805
- char *v = e + 1, *ve;
806
-
807
- // skip leading spaces from value
808
- while(*v == ' ') v++;
809
- ve = v;
810
-
811
- // find the \r
812
- while(*ve && *ve != '\r') ve++;
813
- if(!*ve || ve[1] != '\n') {
814
- *e = ':';
815
- return ve;
816
- }
817
-
818
- // terminate the value
819
- *ve = '\0';
820
-
821
- uint32_t hash = simple_uhash(s);
822
-
823
- if(hash == hash_origin && !strcasecmp(s, "Origin"))
824
- w->origin = strdupz(v);
825
-
826
- else if(hash == hash_connection && !strcasecmp(s, "Connection")) {
827
- if(strcasestr(v, "keep-alive"))
828
- web_client_enable_keepalive(w);
829
- }
830
- else if(respect_web_browser_do_not_track_policy && hash == hash_donottrack && !strcasecmp(s, "DNT")) {
831
- if(*v == '0') web_client_disable_donottrack(w);
832
- else if(*v == '1') web_client_enable_donottrack(w);
833
- }
834
- else if(parse_useragent && hash == hash_useragent && !strcasecmp(s, "User-Agent")) {
835
- w->user_agent = strdupz(v);
836
- }
837
- else if(hash == hash_authorization&& !strcasecmp(s, "X-Auth-Token")) {
838
- w->auth_bearer_token = strdupz(v);
839
- }
840
- else if(hash == hash_host && !strcasecmp(s, "Host")) {
841
- char buffer[NI_MAXHOST];
842
- strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
843
- w->server_host = strdupz(buffer);
844
- }
845
- else if(hash == hash_accept_encoding && !strcasecmp(s, "Accept-Encoding")) {
846
- if(web_enable_gzip) {
847
- if(strcasestr(v, "gzip"))
848
- web_client_enable_deflate(w, 1);
849
- //
850
- // does not seem to work
851
- // else if(strcasestr(v, "deflate"))
852
- // web_client_enable_deflate(w, 0);
853
- }
854
- }
855
- else if(hash == hash_forwarded_host && !strcasecmp(s, "X-Forwarded-Host")) {
856
- char buffer[NI_MAXHOST];
857
- strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
858
- w->forwarded_host = strdupz(buffer);
859
- }
860
- else if(hash == hash_forwarded_for && !strcasecmp(s, "X-Forwarded-For")) {
861
- char buffer[NI_MAXHOST];
862
- strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
863
- w->forwarded_for = strdupz(buffer);
864
- }
865
- else if(hash == hash_transaction_id && !strcasecmp(s, "X-Transaction-ID")) {
866
- char buffer[UUID_STR_LEN * 2];
867
- strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
868
- uuid_parse_flexi(buffer, w->transaction); // will not alter w->transaction if it fails
869
- }
870
-
871
- *e = ':';
872
- *ve = '\r';
873
- return ve;
874
-}
670
671
/**
672
* Valid Method
@@ -955,7 +750,7 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
750
* @return It returns HTTP_VALIDATION_OK on success and another code present
751
* in the enum HTTP_VALIDATION otherwise.
752
*/
958
-static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
753
+HTTP_VALIDATION http_request_validate(struct web_client *w) {
754
char *s = (char *)buffer_tostring(w->response.data), *encoded_url = NULL;
755
756
size_t last_pos = w->header_parse_last_size;
@@ -971,7 +766,8 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
766
if(w->header_parse_last_size < last_pos)
767
last_pos = 0;
768
974
- is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size, &w->post_payload, &w->post_payload_size);
769
+ is_it_valid =
770
+ url_is_request_complete_and_extract_payload(s, &s[last_pos], w->header_parse_last_size, &w->payload);
771
if(!is_it_valid) {
772
if(w->header_parse_tries > HTTP_REQ_MAX_HEADER_FETCH_TRIES) {
773
netdata_log_info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));
@@ -987,7 +783,8 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
783
is_it_valid = 1;
784
} else {
785
last_pos = w->header_parse_last_size;
990
- is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size, &w->post_payload, &w->post_payload_size);
786
+ is_it_valid =
787
+ url_is_request_complete_and_extract_payload(s, &s[last_pos], w->header_parse_last_size, &w->payload);
788
}
789
790
s = web_client_valid_method(w, s);
@@ -1050,7 +847,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
847
*ue = c;
848
849
#ifdef ENABLE_HTTPS
1053
- if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) {
850
+ if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
851
if (!w->ssl.conn && (http_is_using_ssl_force(w) || http_is_using_ssl_default(w)) && (w->mode != HTTP_REQUEST_MODE_STREAM)) {
852
w->header_parse_tries = 0;
853
w->header_parse_last_size = 0;
@@ -1067,7 +864,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
864
}
865
866
// another header line
1070
- s = http_header_parse(w, s, (w->mode == HTTP_REQUEST_MODE_STREAM)); // parse user agent
867
+ s = http_header_parse_line(w, s);
868
}
869
}
870
@@ -1080,17 +877,23 @@ static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,
877
{
878
ssize_t bytes;
879
#ifdef ENABLE_HTTPS
1083
- if ((!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx)) {
880
+ if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) {
881
if (SSL_connection(&w->ssl)) {
882
bytes = netdata_ssl_write(&w->ssl, buf, len) ;
883
web_client_enable_wait_from_ssl(w);
884
}
885
else
886
bytes = send(w->ofd,buf, len , flags);
1090
- } else
887
+ }
888
+ else if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w))
889
bytes = send(w->ofd,buf, len , flags);
890
+ else
891
+ bytes = -999;
892
#else
1093
- bytes = send(w->ofd, buf, len, flags);
893
+ if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w))
894
+ bytes = send(w->ofd, buf, len, flags);
895
+ else
896
+ bytes = -999;
897
#endif
898
899
return bytes;
@@ -1111,7 +914,6 @@ void web_client_build_http_header(struct web_client *w) {
914
// prepare the HTTP response header
915
netdata_log_debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, w->response.code);
916
1114
- const char *content_type_string = web_content_type_to_string(w->response.data->content_type);
917
const char *code_msg = http_response_code2string(w->response.code);
918
919
// prepare the last modified and expiration dates
@@ -1135,15 +937,15 @@ void web_client_build_http_header(struct web_client *w) {
937
"Server: Netdata Embedded HTTP Server %s\r\n"
938
"Access-Control-Allow-Origin: %s\r\n"
939
"Access-Control-Allow-Credentials: true\r\n"
1138
- "Content-Type: %s\r\n"
940
"Date: %s\r\n",
941
w->response.code,
942
code_msg,
943
web_client_has_keepalive(w)?"keep-alive":"close",
944
VERSION,
945
w->origin ? w->origin : "*",
1145
- content_type_string,
946
rfc7231_date);
947
+
948
+ http_header_content_type(w->response.header_output, w->response.data->content_type);
949
}
950
951
if(unlikely(web_x_frame_options))
@@ -1225,7 +1027,7 @@ static inline void web_client_send_http_header(struct web_client *w) {
1027
size_t count = 0;
1028
ssize_t bytes;
1029
#ifdef ENABLE_HTTPS
1228
- if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) {
1030
+ if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
1031
if (SSL_connection(&w->ssl)) {
1032
bytes = netdata_ssl_write(&w->ssl, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output));
1033
web_client_enable_wait_from_ssl(w);
@@ -1241,7 +1043,7 @@ static inline void web_client_send_http_header(struct web_client *w) {
1043
}
1044
}
1045
}
1244
- else {
1046
+ else if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) {
1047
while((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
1048
count++;
1049
@@ -1251,15 +1053,21 @@ static inline void web_client_send_http_header(struct web_client *w) {
1053
}
1054
}
1055
}
1056
+ else
1057
+ bytes = -999;
1058
#else
1255
- while((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
1256
- count++;
1059
+ if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) {
1060
+ while ((bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
1061
+ count++;
1062
1258
- if(count > 100 || (errno != EAGAIN && errno != EWOULDBLOCK)) {
1259
- netdata_log_error("Cannot send HTTP headers to web client.");
1260
- break;
1063
+ if (count > 100 || (errno != EAGAIN && errno != EWOULDBLOCK)) {
1064
+ netdata_log_error("Cannot send HTTP headers to web client.");
1065
+ break;
1066
+ }
1067
}
1068
}
1069
+ else
1070
+ bytes = -999;
1071
#endif
1072
1073
if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) {
@@ -1356,6 +1164,9 @@ int web_client_api_request_with_node_selection(RRDHOST *host, struct web_client
1164
ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
1165
ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
1166
ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
1167
+ ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->auth.cloud_account_id),
1168
+ ND_LOG_FIELD_TXT(NDF_USER_NAME, w->auth.client_name),
1169
+ ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2access(w->access)),
1170
ND_LOG_FIELD_END(),
1171
};
1172
ND_LOG_STACK_PUSH(lgs);
@@ -1578,6 +1389,9 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1389
ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
1390
ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
1391
ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
1392
+ ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->auth.cloud_account_id),
1393
+ ND_LOG_FIELD_TXT(NDF_USER_NAME, w->auth.client_name),
1394
+ ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2access(w->access)),
1395
ND_LOG_FIELD_END(),
1396
};
1397
ND_LOG_STACK_PUSH(lgs);
@@ -1594,7 +1408,7 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1408
web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
1409
query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl,
1410
buffer_tostring(w->url_as_received),
1597
- w->post_payload,
1411
+ w->payload,
1412
w->forwarded_for ? w->forwarded_for : w->client_ip);
1413
}
1414
@@ -2099,7 +1913,7 @@ ssize_t web_client_receive(struct web_client *w)
1913
errno = 0;
1914
1915
#ifdef ENABLE_HTTPS
2102
- if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) {
1916
+ if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
1917
if (SSL_connection(&w->ssl)) {
1918
bytes = netdata_ssl_read(&w->ssl, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
1919
web_client_enable_wait_from_ssl(w);
@@ -2108,11 +1922,16 @@ ssize_t web_client_receive(struct web_client *w)
1922
bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1923
}
1924
}
2111
- else{
1925
+ else if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) {
1926
bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1927
}
1928
+ else // other connection methods
1929
+ bytes = -1;
1930
#else
2115
- bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1931
+ if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w))
1932
+ bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1933
+ else
1934
+ bytes = -1;
1935
#endif
1936
1937
if(likely(bytes > 0)) {
@@ -2194,6 +2013,7 @@ void web_client_reuse_from_cache(struct web_client *w) {
2013
BUFFER *b4 = w->url_path_decoded;
2014
BUFFER *b5 = w->url_as_received;
2015
BUFFER *b6 = w->url_query_string_decoded;
2016
+ BUFFER *b7 = w->payload;
2017
2018
#ifdef ENABLE_HTTPS
2019
NETDATA_SSL ssl = w->ssl;
@@ -2220,6 +2040,7 @@ void web_client_reuse_from_cache(struct web_client *w) {
2040
w->url_path_decoded = b4;
2041
w->url_as_received = b5;
2042
w->url_query_string_decoded = b6;
2043
+ w->payload = b7;
2044
}
2045
2046
struct web_client *web_client_create(size_t *statistics_memory_accounting) {
web/server/web_client.h
+68
-30
@@ -5,6 +5,8 @@
5
6
#include "libnetdata/libnetdata.h"
7
8
+struct web_client;
9
+
10
extern int web_enable_gzip, web_gzip_level, web_gzip_strategy;
11
12
#define HTTP_REQ_MAX_HEADER_FETCH_TRIES 100
@@ -12,7 +14,7 @@ extern int web_enable_gzip, web_gzip_level, web_gzip_strategy;
14
extern int respect_web_browser_do_not_track_policy;
15
extern char *web_x_frame_options;
16
15
-typedef enum {
17
+typedef enum __attribute__((packed)) {
18
HTTP_VALIDATION_OK,
19
HTTP_VALIDATION_NOT_SUPPORTED,
20
HTTP_VALIDATION_TOO_MANY_READ_RETRIES,
@@ -24,25 +26,48 @@ typedef enum {
26
#endif
27
} HTTP_VALIDATION;
28
27
-typedef enum web_client_flags {
28
- WEB_CLIENT_FLAG_DEAD = (1 << 1), // if set, this client is dead
29
- WEB_CLIENT_FLAG_KEEPALIVE = (1 << 2), // if set, the web client will be re-used
30
- WEB_CLIENT_FLAG_WAIT_RECEIVE = (1 << 3), // if set, we are waiting more input data
31
- WEB_CLIENT_FLAG_WAIT_SEND = (1 << 4), // if set, we have data to send to the client
32
- WEB_CLIENT_FLAG_DO_NOT_TRACK = (1 << 5), // if set, we should not set cookies on this client
33
- WEB_CLIENT_FLAG_TRACKING_REQUIRED = (1 << 6), // if set, we need to send cookies
34
- WEB_CLIENT_FLAG_TCP_CLIENT = (1 << 7), // if set, the client is using a TCP socket
35
- WEB_CLIENT_FLAG_UNIX_CLIENT = (1 << 8), // if set, the client is using a UNIX socket
36
- WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET = (1 << 9), // don't close the socket when cleaning up (static-threaded web server)
37
- WEB_CLIENT_CHUNKED_TRANSFER = (1 << 10), // chunked transfer (used with zlib compression)
38
- WEB_CLIENT_FLAG_SSL_WAIT_RECEIVE = (1 << 11), // if set, we are waiting more input data from an ssl conn
39
- WEB_CLIENT_FLAG_SSL_WAIT_SEND = (1 << 12), // if set, we have data to send to the client from an ssl conn
40
- WEB_CLIENT_FLAG_PATH_IS_V0 = (1 << 13), // v0 dashboard found on the path
41
- WEB_CLIENT_FLAG_PATH_IS_V1 = (1 << 14), // v1 dashboard found on the path
42
- WEB_CLIENT_FLAG_PATH_IS_V2 = (1 << 15), // v2 dashboard found on the path
43
- WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH = (1 << 16), // the path has a trailing hash
44
- WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION = (1 << 17), // the path ends with a filename extension
45
- WEB_CLIENT_FLAG_PROGRESS_TRACKING = (1 << 18), // when set we track the progress of this transaction
29
+typedef enum __attribute__((packed)) {
30
+ WEB_CLIENT_FLAG_DEAD = (1 << 0), // this client is dead
31
+
32
+ WEB_CLIENT_FLAG_KEEPALIVE = (1 << 1), // the web client will be re-used
33
+
34
+ // compression
35
+ WEB_CLIENT_ENCODING_GZIP = (1 << 2),
36
+ WEB_CLIENT_ENCODING_DEFLATE = (1 << 3),
37
+ WEB_CLIENT_CHUNKED_TRANSFER = (1 << 4), // chunked transfer (used with zlib compression)
38
+
39
+ WEB_CLIENT_FLAG_WAIT_RECEIVE = (1 << 5), // we are waiting more input data
40
+ WEB_CLIENT_FLAG_WAIT_SEND = (1 << 6), // we have data to send to the client
41
+ WEB_CLIENT_FLAG_SSL_WAIT_RECEIVE = (1 << 7), // we are waiting more input data from ssl connection
42
+ WEB_CLIENT_FLAG_SSL_WAIT_SEND = (1 << 8), // we have data to send to the client from ssl connection
43
+
44
+ // DNT
45
+ WEB_CLIENT_FLAG_DO_NOT_TRACK = (1 << 9), // we should not set cookies on this client
46
+ WEB_CLIENT_FLAG_TRACKING_REQUIRED = (1 << 10), // we need to send cookies
47
+
48
+ // connection type
49
+ WEB_CLIENT_FLAG_CONN_TCP = (1 << 11), // the client is using a TCP socket
50
+ WEB_CLIENT_FLAG_CONN_UNIX = (1 << 12), // the client is using a UNIX socket
51
+ WEB_CLIENT_FLAG_CONN_CLOUD = (1 << 13), // the client is using Netdata Cloud
52
+ WEB_CLIENT_FLAG_CONN_WEBRTC = (1 << 14), // the client is using WebRTC
53
+
54
+ // streaming
55
+ WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET = (1 << 15), // don't close the socket when cleaning up
56
+
57
+ // dashboard version
58
+ WEB_CLIENT_FLAG_PATH_IS_V0 = (1 << 16), // v0 dashboard found on the path
59
+ WEB_CLIENT_FLAG_PATH_IS_V1 = (1 << 17), // v1 dashboard found on the path
60
+ WEB_CLIENT_FLAG_PATH_IS_V2 = (1 << 18), // v2 dashboard found on the path
61
+ WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH = (1 << 19), // the path has a trailing hash
62
+ WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION = (1 << 20), // the path ends with a filename extension
63
+
64
+ // authorization
65
+ WEB_CLIENT_FLAG_AUTH_CLOUD = (1 << 21),
66
+ WEB_CLIENT_FLAG_AUTH_BEARER = (1 << 22),
67
+ WEB_CLIENT_FLAG_AUTH_GOD = (1 << 23),
68
+
69
+ // transient settings
70
+ WEB_CLIENT_FLAG_PROGRESS_TRACKING = (1 << 24), // flag to avoid redoing progress work
71
} WEB_CLIENT_FLAGS;
72
73
#define WEB_CLIENT_FLAG_PATH_WITH_VERSION (WEB_CLIENT_FLAG_PATH_IS_V0|WEB_CLIENT_FLAG_PATH_IS_V1|WEB_CLIENT_FLAG_PATH_IS_V2)
@@ -83,12 +108,19 @@ typedef enum web_client_flags {
108
#define web_client_enable_ssl_wait_send(w) web_client_flag_set(w, WEB_CLIENT_FLAG_SSL_WAIT_SEND)
109
#define web_client_disable_ssl_wait_send(w) web_client_flag_clear(w, WEB_CLIENT_FLAG_SSL_WAIT_SEND)
110
86
-#define web_client_set_tcp(w) web_client_flag_set(w, WEB_CLIENT_FLAG_TCP_CLIENT)
87
-#define web_client_set_unix(w) web_client_flag_set(w, WEB_CLIENT_FLAG_UNIX_CLIENT)
88
-#define web_client_check_unix(w) web_client_flag_check(w, WEB_CLIENT_FLAG_UNIX_CLIENT)
89
-#define web_client_check_tcp(w) web_client_flag_check(w, WEB_CLIENT_FLAG_TCP_CLIENT)
111
+#define web_client_check_conn_unix(w) web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_UNIX)
112
+#define web_client_check_conn_tcp(w) web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_TCP)
113
+#define web_client_check_conn_cloud(w) web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_CLOUD)
114
+#define web_client_check_conn_webrtc(w) web_client_flag_check(w, WEB_CLIENT_FLAG_CONN_WEBRTC)
115
+
116
+#define web_client_flags_clear_conn(w) web_client_flag_clear(w, WEB_CLIENT_FLAG_CONN_TCP | WEB_CLIENT_FLAG_CONN_UNIX | WEB_CLIENT_FLAG_CONN_CLOUD | WEB_CLIENT_FLAG_CONN_WEBRTC)
117
+#define web_client_flags_check_auth(w) web_client_flag_check(w, WEB_CLIENT_FLAG_AUTH_CLOUD | WEB_CLIENT_FLAG_AUTH_BEARER)
118
+#define web_client_flags_clear_auth(w) web_client_flag_clear(w, WEB_CLIENT_FLAG_AUTH_CLOUD | WEB_CLIENT_FLAG_AUTH_BEARER)
119
91
-#define web_client_is_corkable(w) web_client_flag_check(w, WEB_CLIENT_FLAG_TCP_CLIENT)
120
+void web_client_set_conn_tcp(struct web_client *w);
121
+void web_client_set_conn_unix(struct web_client *w);
122
+void web_client_set_conn_cloud(struct web_client *w);
123
+void web_client_set_conn_webrtc(struct web_client *w);
124
125
#define NETDATA_WEB_REQUEST_URL_SIZE 65536 // static allocation
126
@@ -100,6 +132,8 @@ typedef enum web_client_flags {
132
#define NETDATA_WEB_REQUEST_MAX_SIZE 65536
133
#define NETDATA_WEB_DECODED_URL_INITIAL_SIZE 512
134
135
+#define CLOUD_USER_NAME_LENGTH 64
136
+
137
struct response {
138
BUFFER *header; // our response header
139
BUFFER *header_output; // internal use
@@ -157,9 +191,7 @@ struct web_client {
191
char *origin; // the Origin: header
192
char *user_agent; // the User-Agent: header
193
160
- char *post_payload; // when this request is a POST, this has the payload
161
- size_t post_payload_size; // the size of the buffer allocated for the payload
162
- // the actual contents may be less than the size
194
+ BUFFER *payload; // when this request is a POST, this has the payload
195
196
// STATIC-THREADED WEB SERVER MEMBERS
197
size_t pollinfo_slot; // POLLINFO slot of the web client
@@ -169,6 +201,12 @@ struct web_client {
201
NETDATA_SSL ssl;
202
#endif
203
204
+ struct {
205
+ uuid_t bearer_token;
206
+ uuid_t cloud_account_id;
207
+ char client_name[CLOUD_USER_NAME_LENGTH];
208
+ } auth;
209
+
210
struct { // A callback to check if the query should be interrupted / stopped
211
web_client_interrupt_t callback;
212
void *callback_data;
@@ -219,8 +257,6 @@ void web_client_free(struct web_client *w);
257
258
void web_client_decode_path_and_query_string(struct web_client *w, const char *path_and_query_string);
259
int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_fragment);
222
-const char *web_content_type_to_string(HTTP_CONTENT_TYPE content_type);
223
-void web_client_enable_deflate(struct web_client *w, int gzip);
260
int web_client_api_request_with_node_selection(RRDHOST *host, struct web_client *w, char *decoded_url_path);
261
262
void web_client_timeout_checkpoint_init(struct web_client *w);
@@ -230,4 +266,6 @@ bool web_client_timeout_checkpoint_and_check(struct web_client *w, usec_t *usec_
266
usec_t web_client_timeout_checkpoint_response_ready(struct web_client *w, usec_t *usec_since_last_checkpoint);
267
void web_client_log_completed_request(struct web_client *w, bool update_web_stats);
268
269
+HTTP_VALIDATION http_request_validate(struct web_client *w);
270
+
271
#endif