@cryptotaxi247 / netdata-1 / commits / da32dd8be

Queries Progress (#16574)

* track the progress of queries * add query_progress in libnetdata Makefile.am * add acl, response size and response code to the tracking * define the required functions * fix the last commit * added /api/v2/progress?transaction=ID to report the progress of queries * added function to report netdata-queries * track hashtable additions * when resusing a transaction, maintain the counter * keep track of linked and indexing * added X-Forwarded-Host and X-Forwarded-For to logs. X-Forwarded-For is also added in progress tracking * report compact uuids to match logs; register the actual duration of the transaction * added rowOptions to function; now web_client keeps track if it tracks progress or not * add http request method to progress * add tags per function; /api/vX/functions is now not protected * compact the sanitization array * split pluginsd_parser into multiple files * cleanup keyword definitions * code cleanup * extracted rrd_collector to separate files * added http access level to functions * renamed access "all" to "any" * implemented optional protection on functions * add priority to functions, to allow the UI select the best function (lower priority) when the user has not selected a function * added progress report from the plugins to netdata and from children to parents - untested * added progress reporting in systemd-journal * query timeout is now handled by evloop for external plugins * propagate progress reports to children and plugins * fix codeql warning * adapt to cmake * minor changes * extend function timeout when progress is received; added streaming capability to propagate progress reports to parents and send progress requests to children * revert change in dictionary.h * add log when access level is invalid * update access level of functions * added logs when processing progress updates * log when the deferred response is too big * comment out sender progress to find the issue * added missing newline in streaming progress reports * propogate progress reports to functions * fix logs

Costa Tsaousis committed Dec 15, 2023 at 18:15 UTC da32dd8be88ec9e88921d134b02df8ab85856779
84 files changed +4645 -3307
CMakeLists.txt
+16 -1
@@ -389,6 +389,8 @@ set(LIBNETDATA_FILES ${CONFIG_H}
389 libnetdata/popen/popen.h
390 libnetdata/procfile/procfile.c
391 libnetdata/procfile/procfile.h
392 + libnetdata/query_progress/progress.c
393 + libnetdata/query_progress/progress.h
394 libnetdata/required_dummies.h
395 libnetdata/socket/security.c
396 libnetdata/socket/security.h
@@ -411,6 +413,9 @@ set(LIBNETDATA_FILES ${CONFIG_H}
413 libnetdata/string/utf8.h
414 libnetdata/worker_utilization/worker_utilization.c
415 libnetdata/worker_utilization/worker_utilization.h
416 + libnetdata/http/http_access.c
417 + libnetdata/http/http_access.h
418 + libnetdata/http/http_defs.c
419 libnetdata/http/http_defs.h
420 libnetdata/dyn_conf/dyn_conf.c
421 libnetdata/dyn_conf/dyn_conf.h)
@@ -644,8 +649,16 @@ endif()
649
650 set(PLUGINSD_PLUGIN_FILES collectors/plugins.d/plugins_d.c
651 collectors/plugins.d/plugins_d.h
652 + collectors/plugins.d/pluginsd_dyncfg.c
653 + collectors/plugins.d/pluginsd_dyncfg.h
654 + collectors/plugins.d/pluginsd_functions.c
655 + collectors/plugins.d/pluginsd_functions.h
656 + collectors/plugins.d/pluginsd_internals.c
657 + collectors/plugins.d/pluginsd_internals.h
658 collectors/plugins.d/pluginsd_parser.c
648 - collectors/plugins.d/pluginsd_parser.h)
659 + collectors/plugins.d/pluginsd_parser.h
660 + collectors/plugins.d/pluginsd_replication.c
661 + collectors/plugins.d/pluginsd_replication.h)
662
663 set(RRD_PLUGIN_FILES database/contexts/api_v1.c
664 database/contexts/api_v2.c
@@ -662,6 +675,8 @@ set(RRD_PLUGIN_FILES database/contexts/api_v1.c
675 database/rrdcalc.h
676 database/rrdcalctemplate.c
677 database/rrdcalctemplate.h
678 + database/rrdcollector.c
679 + database/rrdcollector.h
680 database/rrddim.c
681 database/rrddimvar.c
682 database/rrddimvar.h
aclk/aclk_query.c
+2 -2
@@ -106,8 +106,8 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
106 char *start, *end;
107
108 struct web_client *w = web_client_get_from_cache();
109 - w->acl = WEB_CLIENT_ACL_ACLK;
110 - w->mode = WEB_CLIENT_MODE_GET;
109 + w->acl = HTTP_ACL_ACLK;
110 + w->mode = HTTP_REQUEST_MODE_GET;
111 w->timings.tv_in = query->created_tv;
112
113 w->interrupt.callback = aclk_web_client_interrupt_cb;
collectors/apps.plugin/apps_plugin.c
+7 -3
@@ -13,11 +13,15 @@
13 #define APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION "Detailed information on the currently running processes."
14
15 #define APPS_PLUGIN_FUNCTIONS() do { \
16 - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" %d \"%s\"\n", PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \
16 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" %d \"%s\" \"top\" \"members\" %d\n", \
17 + PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION, \
18 + RRDFUNCTIONS_PRIORITY_DEFAULT / 10); \
19 } while(0)
20
21 #define APPS_PLUGIN_GLOBAL_FUNCTIONS() do { \
20 - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"processes\" %d \"%s\"\n", PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \
22 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"processes\" %d \"%s\" \"top\" \"members\" %d\n", \
23 + PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION, \
24 + RRDFUNCTIONS_PRIORITY_DEFAULT / 10); \
25 } while(0)
26
27 // ----------------------------------------------------------------------------
@@ -4393,7 +4397,7 @@ static void apps_plugin_function_processes_help(const char *transaction) {
4397 buffer_json_add_array_item_double(wb, _tmp); \
4398 } while(0)
4399
4396 -static void function_processes(const char *transaction, char *function __maybe_unused, int timeout __maybe_unused, bool *cancelled __maybe_unused) {
4400 +static void function_processes(const char *transaction, char *function __maybe_unused, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) {
4401 struct pid_stat *p;
4402
4403 char *words[PLUGINSD_MAX_WORDS] = { NULL };
collectors/cgroups.plugin/cgroup-internals.h
+19 -9
@@ -452,15 +452,25 @@ static inline char *cgroup_chart_type(char *buffer, struct cgroup *cg) {
452 }
453
454 #define RRDFUNCTIONS_CGTOP_HELP "View running containers"
455 -
456 -int cgroup_function_cgroup_top(BUFFER *wb, int timeout, const char *function, void *collector_data,
457 - rrd_function_result_callback_t result_cb, void *result_cb_data,
458 - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
459 - rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data);
460 -int cgroup_function_systemd_top(BUFFER *wb, int timeout, const char *function, void *collector_data,
461 - rrd_function_result_callback_t result_cb, void *result_cb_data,
462 - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
463 - rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data);
455 +#define RRDFUNCTIONS_SYSTEMD_SERVICES_HELP "View systemd services"
456 +
457 +int cgroup_function_cgroup_top(uuid_t *transaction, BUFFER *wb,
458 + usec_t *stop_monotonic_ut, const char *function, void *collector_data,
459 + rrd_function_result_callback_t result_cb, void *result_cb_data,
460 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
461 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
462 + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
463 + rrd_function_register_progresser_cb_t register_progresser_cb,
464 + void *register_progresser_cb_data);
465 +
466 +int cgroup_function_systemd_top(uuid_t *transaction, BUFFER *wb,
467 + usec_t *stop_monotonic_ut, const char *function, void *collector_data,
468 + rrd_function_result_callback_t result_cb, void *result_cb_data,
469 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
470 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
471 + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
472 + rrd_function_register_progresser_cb_t register_progresser_cb,
473 + void *register_progresser_cb_data);
474
475 void cgroup_netdev_link_init(void);
476 const DICTIONARY_ITEM *cgroup_netdev_get(struct cgroup *cg);
collectors/cgroups.plugin/cgroup-top.c
+20 -12
@@ -97,12 +97,16 @@ void cgroup_netdev_get_bandwidth(struct cgroup *cg, NETDATA_DOUBLE *received, NE
97 *sent = t->sent[slot];
98 }
99
100 -int cgroup_function_cgroup_top(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused,
101 - void *collector_data __maybe_unused,
102 - rrd_function_result_callback_t result_cb, void *result_cb_data,
103 - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
104 - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
105 - void *register_canceller_cb_data __maybe_unused) {
100 +int cgroup_function_cgroup_top(uuid_t *transaction __maybe_unused, BUFFER *wb,
101 + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
102 + void *collector_data __maybe_unused,
103 + rrd_function_result_callback_t result_cb, void *result_cb_data,
104 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
105 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
106 + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
107 + void *register_canceller_cb_data __maybe_unused,
108 + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
109 + void *register_progresser_cb_data __maybe_unused) {
110
111 buffer_flush(wb);
112 wb->content_type = CT_APPLICATION_JSON;
@@ -342,12 +346,16 @@ int cgroup_function_cgroup_top(BUFFER *wb, int timeout __maybe_unused, const cha
346 return response;
347 }
348
345 -int cgroup_function_systemd_top(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused,
346 - void *collector_data __maybe_unused,
347 - rrd_function_result_callback_t result_cb, void *result_cb_data,
348 - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
349 - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
350 - void *register_canceller_cb_data __maybe_unused) {
349 +int cgroup_function_systemd_top(uuid_t *transaction __maybe_unused, BUFFER *wb,
350 + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
351 + void *collector_data __maybe_unused,
352 + rrd_function_result_callback_t result_cb, void *result_cb_data,
353 + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
354 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
355 + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
356 + void *register_canceller_cb_data __maybe_unused,
357 + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
358 + void *register_progresser_cb_data __maybe_unused) {
359
360 buffer_flush(wb);
361 wb->content_type = CT_APPLICATION_JSON;
collectors/cgroups.plugin/sys_fs_cgroup.c
+8 -2
@@ -1671,8 +1671,14 @@ void *cgroups_main(void *ptr) {
1671 // for the other nodes, the origin server should register it
1672 rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
1673 cgroup_netdev_link_init();
1674 - rrd_function_add(localhost, NULL, "containers-vms", 10, RRDFUNCTIONS_CGTOP_HELP, true, cgroup_function_cgroup_top, NULL);
1675 - rrd_function_add(localhost, NULL, "systemd-services", 10, RRDFUNCTIONS_CGTOP_HELP, true, cgroup_function_systemd_top, NULL);
1674 +
1675 + rrd_function_add(localhost, NULL, "containers-vms", 10, RRDFUNCTIONS_PRIORITY_DEFAULT / 2,
1676 + RRDFUNCTIONS_CGTOP_HELP, "top", HTTP_ACCESS_ANY,
1677 + true, cgroup_function_cgroup_top, NULL);
1678 +
1679 + rrd_function_add(localhost, NULL, "systemd-services", 10, RRDFUNCTIONS_PRIORITY_DEFAULT / 3,
1680 + RRDFUNCTIONS_SYSTEMD_SERVICES_HELP, "top", HTTP_ACCESS_ANY,
1681 + true, cgroup_function_systemd_top, NULL);
1682
1683 heartbeat_t hb;
1684 heartbeat_init(&hb);
collectors/diskspace.plugin/plugin_diskspace.c
+13 -7
@@ -636,12 +636,16 @@ static void diskspace_main_cleanup(void *ptr) {
636 #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
637 #endif
638
639 -int diskspace_function_mount_points(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused,
640 - void *collector_data __maybe_unused,
641 - rrd_function_result_callback_t result_cb, void *result_cb_data,
642 - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
643 - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
644 - void *register_canceller_cb_data __maybe_unused) {
639 +int diskspace_function_mount_points(uuid_t *transaction __maybe_unused, BUFFER *wb,
640 + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
641 + void *collector_data __maybe_unused,
642 + rrd_function_result_callback_t result_cb, void *result_cb_data,
643 + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
644 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
645 + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
646 + void *register_canceller_cb_data __maybe_unused,
647 + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
648 + void *register_progresser_cb_data __maybe_unused) {
649
650 buffer_flush(wb);
651 wb->content_type = CT_APPLICATION_JSON;
@@ -865,7 +869,9 @@ void *diskspace_main(void *ptr) {
869 worker_register_job_name(WORKER_JOB_CLEANUP, "cleanup");
870
871 rrd_collector_started();
868 - rrd_function_add(localhost, NULL, "mount-points", 10, RRDFUNCTIONS_DISKSPACE_HELP, true, diskspace_function_mount_points, NULL);
872 + rrd_function_add(localhost, NULL, "mount-points", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSPACE_HELP,
873 + "top", HTTP_ACCESS_ANY,
874 + true, diskspace_function_mount_points, NULL);
875
876 netdata_thread_cleanup_push(diskspace_main_cleanup, ptr);
877
collectors/ebpf.plugin/ebpf_functions.c
+1 -2
@@ -639,10 +639,9 @@ void ebpf_socket_read_open_connections(BUFFER *buf, struct ebpf_module *em)
639 */
640 static void ebpf_function_socket_manipulation(const char *transaction,
641 char *function __maybe_unused,
642 - int timeout __maybe_unused,
642 + usec_t *stop_monotonic_ut __maybe_unused,
643 bool *cancelled __maybe_unused)
644 {
645 - UNUSED(timeout);
645 ebpf_module_t *em = &ebpf_modules[EBPF_MODULE_SOCKET_IDX];
646
647 char *words[PLUGINSD_MAX_WORDS] = {NULL};
collectors/ebpf.plugin/ebpf_functions.h
+2 -1
@@ -6,7 +6,8 @@
6 #ifdef NETDATA_DEV_MODE
7 // Common
8 static inline void EBPF_PLUGIN_FUNCTIONS(const char *NAME, const char *DESC) {
9 - fprintf(stdout, "%s \"%s\" 10 \"%s\"\n", PLUGINSD_KEYWORD_FUNCTION, NAME, DESC);
9 + fprintf(stdout, "%s \"%s\" 10 \"%s\" \"top\" \"any\" %d\n",
10 + PLUGINSD_KEYWORD_FUNCTION, NAME, DESC, RRDFUNCTIONS_PRIORITY_DEFAULT);
11 }
12 #endif
13
collectors/freeipmi.plugin/freeipmi_plugin.c
+3 -2
@@ -23,7 +23,8 @@
23 #include "libnetdata/required_dummies.h"
24
25 #define FREEIPMI_GLOBAL_FUNCTION_SENSORS() do { \
26 - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"ipmi-sensors\" %d \"%s\"\n", 5, "Displays current sensor state and readings"); \
26 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"ipmi-sensors\" %d \"%s\" \"top\" \"any\" %d\n", \
27 + 5, "Displays current sensor state and readings", 100); \
28 } while(0)
29
30 // component names, based on our patterns
@@ -1470,7 +1471,7 @@ static const char *get_sensor_function_priority(struct sensor *sn) {
1471 }
1472 }
1473
1473 -static void freeimi_function_sensors(const char *transaction, char *function __maybe_unused, int timeout __maybe_unused, bool *cancelled __maybe_unused) {
1474 +static void freeimi_function_sensors(const char *transaction, char *function __maybe_unused, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) {
1475 time_t expires = now_realtime_sec() + update_every;
1476
1477 BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
collectors/log2journal/log2journal.c
+1 -1
@@ -67,7 +67,7 @@ static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, HASHED_KEY *k) {
67
68 if(!k->hashtable_ptr) {
69 HASHED_KEY *ht_key;
70 - SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&jb->hashtable, k->hash, true);
70 + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&jb->hashtable, k->hash, NULL, true);
71 if((ht_key = SIMPLE_HASHTABLE_SLOT_DATA(slot))) {
72 if(!(ht_key->flags & HK_COLLISION_CHECKED)) {
73 ht_key->flags |= HK_COLLISION_CHECKED;
collectors/log2journal/log2journal.h
+1 -4
@@ -99,10 +99,7 @@ static inline void freez(void *ptr) {
99 // hashtable for HASHED_KEY
100
101 // cleanup hashtable defines
102 -#undef SIMPLE_HASHTABLE_SORT_FUNCTION
103 -#undef SIMPLE_HASHTABLE_VALUE_TYPE
104 -#undef SIMPLE_HASHTABLE_NAME
105 -#undef NETDATA_SIMPLE_HASHTABLE_H
102 +#include "../../libnetdata/simple_hashtable_undef.h"
103
104 struct hashed_key;
105 static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2);
collectors/plugins.d/README.md
+6 -1
@@ -134,6 +134,7 @@ Netdata parses lines starting with:
134 - `FLUSH` - ignore the last collected values
135 - `DISABLE` - disable this plugin
136 - `FUNCTION` - define functions
137 +- `FUNCTION_PROGRESS` - report the progress of a function execution
138 - `FUNCTION_RESULT_BEGIN` - to initiate the transmission of function results
139 - `FUNCTION_RESULT_END` - to end the transmission of function results
140
@@ -146,6 +147,7 @@ Netdata may send the following commands to the plugin's `stdin`:
147 - `FUNCTION` - to call a specific function, with all parameters inline
148 - `FUNCTION_PAYLOAD` - to call a specific function, with a payload of parameters
149 - `FUNCTION_PAYLOAD_END` - to end the payload of parameters
150 +- `FUNCTION_CANCEL` - cancel a running function transaction
151
152 ### Command line parameters
153
@@ -466,7 +468,10 @@ The `source` is an integer field that can have the following values:
468
469 The plugin can register functions to Netdata, like this:
470
469 -> FUNCTION [GLOBAL] "name and parameters of the function" timeout "help string for users"
471 +> FUNCTION [GLOBAL] "name and parameters of the function" timeout "help string for users" "tags" "access"
472 +
473 +- Tags currently recognized are either `top` or `logs` (or both, space separated).
474 +- Access is one of `any`, `members`, or `admins`.
475
476 A function can be used by users to ask for more information from the collector. Netdata maintains a registry of functions in 2 levels:
477
collectors/plugins.d/gperf-config.txt
+22 -20
@@ -30,29 +30,31 @@ DIMENSION, 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_R
30 END, 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13
31 FUNCTION, 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14
32 FUNCTION_RESULT_BEGIN, 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15
33 -LABEL, 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 16
34 -OVERWRITE, 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17
35 -SET, 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18
36 -VARIABLE, 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 19
37 -DYNCFG_ENABLE, 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20
38 -DYNCFG_REGISTER_MODULE, 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21
39 -DYNCFG_REGISTER_JOB, 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22
40 -DYNCFG_RESET, 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23
41 -REPORT_JOB_STATUS, 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24
42 -DELETE_JOB, 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25
33 +FUNCTION_PROGRESS, 43, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16
34 +#
35 +LABEL, 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17
36 +OVERWRITE, 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 18
37 +SET, 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19
38 +VARIABLE, 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 20
39 +DYNCFG_ENABLE, 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21
40 +DYNCFG_REGISTER_MODULE, 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22
41 +DYNCFG_REGISTER_JOB, 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23
42 +DYNCFG_RESET, 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24
43 +REPORT_JOB_STATUS, 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25
44 +DELETE_JOB, 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26
45 #
46 # Streaming only keywords
47 #
46 -CLAIMED_ID, 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26
47 -BEGIN2, 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27
48 -SET2, 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28
49 -END2, 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29
48 +CLAIMED_ID, 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 27
49 +BEGIN2, 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28
50 +SET2, 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29
51 +END2, 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30
52 #
53 # Streaming Replication keywords
54 #
53 -CHART_DEFINITION_END, 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 30
54 -RBEGIN, 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31
55 -RDSTATE, 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32
56 -REND, 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33
57 -RSET, 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34
58 -RSSTATE, 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35
55 +CHART_DEFINITION_END, 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31
56 +RBEGIN, 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32
57 +RDSTATE, 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33
58 +REND, 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34
59 +RSET, 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35
60 +RSSTATE, 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36
collectors/plugins.d/gperf-hashtable.h
+80 -77
@@ -30,12 +30,12 @@
30 #endif
31
32
33 -#define GPERF_PARSER_TOTAL_KEYWORDS 35
33 +#define GPERF_PARSER_TOTAL_KEYWORDS 36
34 #define GPERF_PARSER_MIN_WORD_LENGTH 3
35 #define GPERF_PARSER_MAX_WORD_LENGTH 22
36 #define GPERF_PARSER_MIN_HASH_VALUE 3
37 -#define GPERF_PARSER_MAX_HASH_VALUE 47
38 -/* maximum key range = 45, duplicates = 0 */
37 +#define GPERF_PARSER_MAX_HASH_VALUE 48
38 +/* maximum key range = 46, duplicates = 0 */
39
40 #ifdef __GNUC__
41 __inline
@@ -49,32 +49,32 @@ gperf_keyword_hash_function (register const char *str, register size_t len)
49 {
50 static unsigned char asso_values[] =
51 {
52 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
53 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
54 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
55 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
56 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
57 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
58 - 48, 48, 48, 48, 48, 11, 18, 0, 0, 0,
59 - 6, 48, 9, 0, 48, 48, 20, 48, 0, 8,
60 - 48, 48, 1, 12, 48, 20, 18, 48, 2, 0,
61 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
62 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
63 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
64 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
65 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
66 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
67 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
68 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
69 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
70 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
71 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
72 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
73 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
74 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
75 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
76 - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
77 - 48, 48, 48, 48, 48, 48
52 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
53 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
54 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
55 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
56 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
57 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
58 + 49, 49, 49, 49, 49, 23, 29, 0, 0, 0,
59 + 0, 49, 9, 0, 49, 49, 20, 49, 0, 8,
60 + 49, 49, 1, 12, 49, 23, 6, 49, 2, 0,
61 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
62 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
63 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
64 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
65 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
66 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
67 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
68 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
69 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
70 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
71 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
72 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
73 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
74 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
75 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
76 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
77 + 49, 49, 49, 49, 49, 49
78 };
79 return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];
80 }
@@ -84,78 +84,81 @@ static PARSER_KEYWORD gperf_keywords[] =
84 {(char*)0}, {(char*)0}, {(char*)0},
85 #line 30 "gperf-config.txt"
86 {"END", 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13},
87 -#line 49 "gperf-config.txt"
88 - {"END2", 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29},
89 -#line 56 "gperf-config.txt"
90 - {"REND", 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33},
87 +#line 51 "gperf-config.txt"
88 + {"END2", 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30},
89 +#line 58 "gperf-config.txt"
90 + {"REND", 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34},
91 #line 17 "gperf-config.txt"
92 {"EXIT", 99, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3},
93 #line 16 "gperf-config.txt"
94 {"DISABLE", 98, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2},
95 -#line 55 "gperf-config.txt"
96 - {"RDSTATE", 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32},
95 +#line 57 "gperf-config.txt"
96 + {"RDSTATE", 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33},
97 #line 29 "gperf-config.txt"
98 {"DIMENSION", 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 12},
99 -#line 42 "gperf-config.txt"
100 - {"DELETE_JOB", 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25},
99 +#line 44 "gperf-config.txt"
100 + {"DELETE_JOB", 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26},
101 {(char*)0},
102 -#line 40 "gperf-config.txt"
103 - {"DYNCFG_RESET", 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23},
104 -#line 37 "gperf-config.txt"
105 - {"DYNCFG_ENABLE", 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20},
102 +#line 42 "gperf-config.txt"
103 + {"DYNCFG_RESET", 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24},
104 +#line 39 "gperf-config.txt"
105 + {"DYNCFG_ENABLE", 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21},
106 #line 26 "gperf-config.txt"
107 {"CHART", 32, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 9},
108 -#line 35 "gperf-config.txt"
109 - {"SET", 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18},
110 -#line 48 "gperf-config.txt"
111 - {"SET2", 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28},
112 -#line 57 "gperf-config.txt"
113 - {"RSET", 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34},
108 +#line 37 "gperf-config.txt"
109 + {"SET", 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19},
110 +#line 50 "gperf-config.txt"
111 + {"SET2", 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29},
112 +#line 59 "gperf-config.txt"
113 + {"RSET", 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35},
114 +#line 43 "gperf-config.txt"
115 + {"REPORT_JOB_STATUS", 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25},
116 #line 41 "gperf-config.txt"
115 - {"REPORT_JOB_STATUS", 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24},
116 -#line 39 "gperf-config.txt"
117 - {"DYNCFG_REGISTER_JOB", 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22},
118 -#line 58 "gperf-config.txt"
119 - {"RSSTATE", 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35},
117 + {"DYNCFG_REGISTER_JOB", 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23},
118 +#line 60 "gperf-config.txt"
119 + {"RSSTATE", 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36},
120 #line 18 "gperf-config.txt"
121 {"HOST", 71, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 4},
122 -#line 38 "gperf-config.txt"
123 - {"DYNCFG_REGISTER_MODULE", 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21},
124 -#line 25 "gperf-config.txt"
125 - {"BEGIN", 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8},
126 -#line 47 "gperf-config.txt"
127 - {"BEGIN2", 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27},
128 -#line 54 "gperf-config.txt"
129 - {"RBEGIN", 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31},
122 +#line 40 "gperf-config.txt"
123 + {"DYNCFG_REGISTER_MODULE", 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22},
124 +#line 36 "gperf-config.txt"
125 + {"OVERWRITE", 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 18},
126 + {(char*)0},
127 +#line 15 "gperf-config.txt"
128 + {"FLUSH", 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1},
129 #line 27 "gperf-config.txt"
130 {"CLABEL", 34, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 10},
131 #line 21 "gperf-config.txt"
132 {"HOST_LABEL", 74, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 7},
133 #line 19 "gperf-config.txt"
134 {"HOST_DEFINE", 72, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 5},
136 -#line 53 "gperf-config.txt"
137 - {"CHART_DEFINITION_END", 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 30},
138 -#line 46 "gperf-config.txt"
139 - {"CLAIMED_ID", 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26},
140 -#line 15 "gperf-config.txt"
141 - {"FLUSH", 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1},
135 +#line 55 "gperf-config.txt"
136 + {"CHART_DEFINITION_END", 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31},
137 +#line 48 "gperf-config.txt"
138 + {"CLAIMED_ID", 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 27},
139 +#line 31 "gperf-config.txt"
140 + {"FUNCTION", 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14},
141 #line 20 "gperf-config.txt"
142 {"HOST_DEFINE_END", 73, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 6},
143 #line 28 "gperf-config.txt"
144 {"CLABEL_COMMIT", 35, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 11},
146 -#line 31 "gperf-config.txt"
147 - {"FUNCTION", 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14},
148 -#line 34 "gperf-config.txt"
149 - {"OVERWRITE", 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17},
145 +#line 25 "gperf-config.txt"
146 + {"BEGIN", 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8},
147 +#line 49 "gperf-config.txt"
148 + {"BEGIN2", 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28},
149 +#line 56 "gperf-config.txt"
150 + {"RBEGIN", 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32},
151 +#line 38 "gperf-config.txt"
152 + {"VARIABLE", 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 20},
153 + {(char*)0}, {(char*)0},
154 #line 33 "gperf-config.txt"
151 - {"LABEL", 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 16},
152 -#line 36 "gperf-config.txt"
153 - {"VARIABLE", 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 19},
154 - {(char*)0}, {(char*)0}, {(char*)0}, {(char*)0},
155 - {(char*)0}, {(char*)0}, {(char*)0}, {(char*)0},
156 - {(char*)0},
155 + {"FUNCTION_PROGRESS", 43, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16},
156 + {(char*)0}, {(char*)0}, {(char*)0},
157 #line 32 "gperf-config.txt"
158 - {"FUNCTION_RESULT_BEGIN", 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15}
158 + {"FUNCTION_RESULT_BEGIN", 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15},
159 + {(char*)0}, {(char*)0}, {(char*)0},
160 +#line 35 "gperf-config.txt"
161 + {"LABEL", 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17}
162 };
163
164 PARSER_KEYWORD *
collectors/plugins.d/plugins_d.h
-3
@@ -10,9 +10,6 @@
10 #define PLUGINSD_CMD_MAX (FILENAME_MAX*2)
11 #define PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH 0
12
13 -#define PLUGINSD_KEYWORD_FUNCTION_PAYLOAD "FUNCTION_PAYLOAD"
14 -#define PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END "FUNCTION_PAYLOAD_END"
15 -
13 #define PLUGINSD_KEYWORD_DYNCFG_ENABLE "DYNCFG_ENABLE"
14 #define PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE "DYNCFG_REGISTER_MODULE"
15 #define PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB "DYNCFG_REGISTER_JOB"
collectors/plugins.d/pluginsd_dyncfg.c new
+585
@@ -0,0 +1,585 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "pluginsd_dyncfg.h"
4 +
5 +struct mutex_cond {
6 + pthread_mutex_t lock;
7 + pthread_cond_t cond;
8 + int rc;
9 +};
10 +
11 +static void virt_fnc_got_data_cb(BUFFER *wb __maybe_unused, int code, void *callback_data)
12 +{
13 + struct mutex_cond *ctx = callback_data;
14 + pthread_mutex_lock(&ctx->lock);
15 + ctx->rc = code;
16 + pthread_cond_broadcast(&ctx->cond);
17 + pthread_mutex_unlock(&ctx->lock);
18 +}
19 +
20 +#define VIRT_FNC_TIMEOUT_S 10
21 +#define VIRT_FNC_BUF_SIZE (4096)
22 +void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data) {
23 + PARSER *parser = NULL;
24 +
25 + //TODO simplify (as we really need only first parameter to get plugin name maybe we can avoid parsing all)
26 + char *words[PLUGINSD_MAX_WORDS];
27 + char *function_with_params = strdupz(name);
28 + size_t num_words = quoted_strings_splitter(function_with_params, words, PLUGINSD_MAX_WORDS, isspace_map_pluginsd);
29 +
30 + if (num_words < 2) {
31 + netdata_log_error("PLUGINSD: virtual function name is empty.");
32 + freez(function_with_params);
33 + return;
34 + }
35 +
36 + const DICTIONARY_ITEM *cpi = dictionary_get_and_acquire_item(host->configurable_plugins, get_word(words, num_words, 1));
37 + if (unlikely(cpi == NULL)) {
38 + netdata_log_error("PLUGINSD: virtual function plugin '%s' not found.", name);
39 + freez(function_with_params);
40 + return;
41 + }
42 + struct configurable_plugin *cp = dictionary_acquired_item_value(cpi);
43 + parser = (PARSER *)cp->cb_usr_ctx;
44 +
45 + BUFFER *function_out = buffer_create(VIRT_FNC_BUF_SIZE, NULL);
46 + // if we are forwarding this to a plugin (as opposed to streaming/child) we have to remove the first parameter (plugin_name)
47 + buffer_strcat(function_out, get_word(words, num_words, 0));
48 + for (size_t i = 1; i < num_words; i++) {
49 + if (i == 1 && SERVING_PLUGINSD(parser))
50 + continue;
51 + buffer_sprintf(function_out, " %s", get_word(words, num_words, i));
52 + }
53 + freez(function_with_params);
54 +
55 + usec_t now_ut = now_monotonic_usec();
56 +
57 + struct inflight_function tmp = {
58 + .started_monotonic_ut = now_ut,
59 + .result_body_wb = wb,
60 + .timeout_s = VIRT_FNC_TIMEOUT_S,
61 + .function = string_strdupz(buffer_tostring(function_out)),
62 + .payload = payload != NULL ? strdupz(payload) : NULL,
63 + .virtual = true,
64 +
65 + .result = {
66 + .cb = callback,
67 + .data = callback_data,
68 + },
69 + .dyncfg = {
70 + .stop_monotonic_ut = now_ut + VIRT_FNC_TIMEOUT_S * USEC_PER_SEC,
71 + }
72 + };
73 + tmp.stop_monotonic_ut = &tmp.dyncfg.stop_monotonic_ut;
74 + buffer_free(function_out);
75 +
76 + uuid_generate_time(tmp.transaction);
77 + char key[UUID_COMPACT_STR_LEN];
78 + uuid_unparse_lower_compact(tmp.transaction, key);
79 +
80 + dictionary_write_lock(parser->inflight.functions);
81 +
82 + // if there is any error, our dictionary callbacks will call the caller callback to notify
83 + // the caller about the error - no need for error handling here.
84 + dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function));
85 +
86 + if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut)
87 + parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
88 +
89 + // garbage collect stale inflight functions
90 + if(parser->inflight.smaller_monotonic_timeout_ut < now_ut)
91 + pluginsd_inflight_functions_garbage_collect(parser, now_ut);
92 +
93 + dictionary_write_unlock(parser->inflight.functions);
94 +}
95 +
96 +
97 +dyncfg_config_t call_virtual_function_blocking(PARSER *parser, const char *name, int *rc, const char *payload) {
98 + usec_t now_ut = now_monotonic_usec();
99 + BUFFER *wb = buffer_create(VIRT_FNC_BUF_SIZE, NULL);
100 +
101 + struct mutex_cond cond = {
102 + .lock = PTHREAD_MUTEX_INITIALIZER,
103 + .cond = PTHREAD_COND_INITIALIZER
104 + };
105 +
106 + struct inflight_function tmp = {
107 + .started_monotonic_ut = now_ut,
108 + .result_body_wb = wb,
109 + .timeout_s = VIRT_FNC_TIMEOUT_S,
110 + .function = string_strdupz(name),
111 + .payload = payload != NULL ? strdupz(payload) : NULL,
112 + .virtual = true,
113 +
114 + .result = {
115 + .cb = virt_fnc_got_data_cb,
116 + .data = &cond,
117 + },
118 + .dyncfg = {
119 + .stop_monotonic_ut = now_ut + VIRT_FNC_TIMEOUT_S * USEC_PER_SEC,
120 + }
121 + };
122 + tmp.stop_monotonic_ut = &tmp.dyncfg.stop_monotonic_ut;
123 +
124 + uuid_generate_time(tmp.transaction);
125 +
126 + char key[UUID_COMPACT_STR_LEN];
127 + uuid_unparse_lower_compact(tmp.transaction, key);
128 +
129 + dictionary_write_lock(parser->inflight.functions);
130 +
131 + // if there is any error, our dictionary callbacks will call the caller callback to notify
132 + // the caller about the error - no need for error handling here.
133 + dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function));
134 +
135 + if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut)
136 + parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
137 +
138 + // garbage collect stale inflight functions
139 + if(parser->inflight.smaller_monotonic_timeout_ut < now_ut)
140 + pluginsd_inflight_functions_garbage_collect(parser, now_ut);
141 +
142 + dictionary_write_unlock(parser->inflight.functions);
143 +
144 + struct timespec tp;
145 + clock_gettime(CLOCK_REALTIME, &tp);
146 + tp.tv_sec += (time_t)VIRT_FNC_TIMEOUT_S;
147 +
148 + pthread_mutex_lock(&cond.lock);
149 +
150 + int ret = pthread_cond_timedwait(&cond.cond, &cond.lock, &tp);
151 + if (ret == ETIMEDOUT)
152 + netdata_log_error("PLUGINSD: DYNCFG virtual function %s timed out", name);
153 +
154 + pthread_mutex_unlock(&cond.lock);
155 +
156 + dyncfg_config_t cfg;
157 + cfg.data = strdupz(buffer_tostring(wb));
158 + cfg.data_size = buffer_strlen(wb);
159 +
160 + if (rc != NULL)
161 + *rc = cond.rc;
162 +
163 + buffer_free(wb);
164 + return cfg;
165 +}
166 +
167 +#define CVF_MAX_LEN (1024)
168 +static dyncfg_config_t get_plugin_config_cb(void *usr_ctx, const char *plugin_name)
169 +{
170 + PARSER *parser = usr_ctx;
171 +
172 + if (SERVING_STREAMING(parser)) {
173 + char buf[CVF_MAX_LEN + 1];
174 + snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG " %s", plugin_name);
175 + return call_virtual_function_blocking(parser, buf, NULL, NULL);
176 + }
177 +
178 + return call_virtual_function_blocking(parser, FUNCTION_NAME_GET_PLUGIN_CONFIG, NULL, NULL);
179 +}
180 +
181 +static dyncfg_config_t get_plugin_config_schema_cb(void *usr_ctx, const char *plugin_name)
182 +{
183 + PARSER *parser = usr_ctx;
184 +
185 + if (SERVING_STREAMING(parser)) {
186 + char buf[CVF_MAX_LEN + 1];
187 + snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA " %s", plugin_name);
188 + return call_virtual_function_blocking(parser, buf, NULL, NULL);
189 + }
190 +
191 + return call_virtual_function_blocking(parser, "get_plugin_config_schema", NULL, NULL);
192 +}
193 +
194 +static dyncfg_config_t get_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
195 +{
196 + PARSER *parser = usr_ctx;
197 + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
198 +
199 + buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG);
200 + if (SERVING_STREAMING(parser))
201 + buffer_sprintf(wb, " %s", plugin_name);
202 +
203 + buffer_sprintf(wb, " %s", module_name);
204 +
205 + dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
206 +
207 + buffer_free(wb);
208 +
209 + return ret;
210 +}
211 +
212 +static dyncfg_config_t get_module_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
213 +{
214 + PARSER *parser = usr_ctx;
215 + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
216 +
217 + buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA);
218 + if (SERVING_STREAMING(parser))
219 + buffer_sprintf(wb, " %s", plugin_name);
220 +
221 + buffer_sprintf(wb, " %s", module_name);
222 +
223 + dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
224 +
225 + buffer_free(wb);
226 +
227 + return ret;
228 +}
229 +
230 +static dyncfg_config_t get_job_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
231 +{
232 + PARSER *parser = usr_ctx;
233 + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
234 +
235 + buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA);
236 +
237 + if (SERVING_STREAMING(parser))
238 + buffer_sprintf(wb, " %s", plugin_name);
239 +
240 + buffer_sprintf(wb, " %s", module_name);
241 +
242 + dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
243 +
244 + buffer_free(wb);
245 +
246 + return ret;
247 +}
248 +
249 +static dyncfg_config_t get_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char* job_name)
250 +{
251 + PARSER *parser = usr_ctx;
252 + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
253 +
254 + buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG);
255 +
256 + if (SERVING_STREAMING(parser))
257 + buffer_sprintf(wb, " %s", plugin_name);
258 +
259 + buffer_sprintf(wb, " %s %s", module_name, job_name);
260 +
261 + dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
262 +
263 + buffer_free(wb);
264 +
265 + return ret;
266 +}
267 +
268 +enum set_config_result set_plugin_config_cb(void *usr_ctx, const char *plugin_name, dyncfg_config_t *cfg)
269 +{
270 + PARSER *parser = usr_ctx;
271 + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
272 +
273 + buffer_strcat(wb, FUNCTION_NAME_SET_PLUGIN_CONFIG);
274 +
275 + if (SERVING_STREAMING(parser))
276 + buffer_sprintf(wb, " %s", plugin_name);
277 +
278 + int rc;
279 + call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
280 +
281 + buffer_free(wb);
282 + if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
283 + return SET_CONFIG_REJECTED;
284 + return SET_CONFIG_ACCEPTED;
285 +}
286 +
287 +enum set_config_result set_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, dyncfg_config_t *cfg)
288 +{
289 + PARSER *parser = usr_ctx;
290 + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
291 +
292 + buffer_strcat(wb, FUNCTION_NAME_SET_MODULE_CONFIG);
293 +
294 + if (SERVING_STREAMING(parser))
295 + buffer_sprintf(wb, " %s", plugin_name);
296 +
297 + buffer_sprintf(wb, " %s", module_name);
298 +
299 + int rc;
300 + call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
301 +
302 + buffer_free(wb);
303 +
304 + if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
305 + return SET_CONFIG_REJECTED;
306 + return SET_CONFIG_ACCEPTED;
307 +}
308 +
309 +enum set_config_result set_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name, dyncfg_config_t *cfg)
310 +{
311 + PARSER *parser = usr_ctx;
312 + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
313 +
314 + buffer_strcat(wb, FUNCTION_NAME_SET_JOB_CONFIG);
315 +
316 + if (SERVING_STREAMING(parser))
317 + buffer_sprintf(wb, " %s", plugin_name);
318 +
319 + buffer_sprintf(wb, " %s %s", module_name, job_name);
320 +
321 + int rc;
322 + call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
323 +
324 + buffer_free(wb);
325 +
326 + if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
327 + return SET_CONFIG_REJECTED;
328 + return SET_CONFIG_ACCEPTED;
329 +}
330 +
331 +enum set_config_result delete_job_cb(void *usr_ctx, const char *plugin_name ,const char *module_name, const char *job_name)
332 +{
333 + PARSER *parser = usr_ctx;
334 + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
335 +
336 + buffer_strcat(wb, FUNCTION_NAME_DELETE_JOB);
337 +
338 + if (SERVING_STREAMING(parser))
339 + buffer_sprintf(wb, " %s", plugin_name);
340 +
341 + buffer_sprintf(wb, " %s %s", module_name, job_name);
342 +
343 + int rc;
344 + call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, NULL);
345 +
346 + buffer_free(wb);
347 +
348 + if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
349 + return SET_CONFIG_REJECTED;
350 + return SET_CONFIG_ACCEPTED;
351 +}
352 +
353 +
354 +PARSER_RC pluginsd_register_plugin(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
355 + netdata_log_info("PLUGINSD: DYNCFG_ENABLE");
356 +
357 + if (unlikely (num_words != 2))
358 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "missing name parameter");
359 +
360 + struct configurable_plugin *cfg = callocz(1, sizeof(struct configurable_plugin));
361 +
362 + cfg->name = strdupz(words[1]);
363 + cfg->set_config_cb = set_plugin_config_cb;
364 + cfg->get_config_cb = get_plugin_config_cb;
365 + cfg->get_config_schema_cb = get_plugin_config_schema_cb;
366 + cfg->cb_usr_ctx = parser;
367 +
368 + const DICTIONARY_ITEM *di = register_plugin(parser->user.host->configurable_plugins, cfg, SERVING_PLUGINSD(parser));
369 + if (unlikely(di == NULL)) {
370 + freez(cfg->name);
371 + freez(cfg);
372 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "error registering plugin");
373 + }
374 +
375 + if (SERVING_PLUGINSD(parser)) {
376 + // this is optimization for pluginsd to avoid extra dictionary lookup
377 + // as we know which plugin is comunicating with us
378 + parser->user.cd->cfg_dict_item = di;
379 + parser->user.cd->configuration = cfg;
380 + } else {
381 + // register_plugin keeps the item acquired, so we need to release it
382 + dictionary_acquired_item_release(parser->user.host->configurable_plugins, di);
383 + }
384 +
385 + rrdpush_send_dyncfg_enable(parser->user.host, cfg->name);
386 +
387 + return PARSER_RC_OK;
388 +}
389 +
390 +#define LOG_MSG_SIZE (1024)
391 +#define MODULE_NAME_IDX (SERVING_PLUGINSD(parser) ? 1 : 2)
392 +#define MODULE_TYPE_IDX (SERVING_PLUGINSD(parser) ? 2 : 3)
393 +
394 +PARSER_RC pluginsd_register_module(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
395 + netdata_log_info("PLUGINSD: DYNCFG_REG_MODULE");
396 +
397 + size_t expected_num_words = SERVING_PLUGINSD(parser) ? 3 : 4;
398 +
399 + if (unlikely(num_words != expected_num_words)) {
400 + char log[LOG_MSG_SIZE + 1];
401 + snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name module_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name ");
402 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, log);
403 + }
404 +
405 + struct configurable_plugin *plug_cfg;
406 + const DICTIONARY_ITEM *di = NULL;
407 + if (SERVING_PLUGINSD(parser)) {
408 + plug_cfg = parser->user.cd->configuration;
409 + if (unlikely(plug_cfg == NULL))
410 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "you have to enable dynamic configuration first using " PLUGINSD_KEYWORD_DYNCFG_ENABLE);
411 + } else {
412 + di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]);
413 + if (unlikely(di == NULL))
414 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "plugin not found");
415 +
416 + plug_cfg = (struct configurable_plugin *)dictionary_acquired_item_value(di);
417 + }
418 +
419 + struct module *mod = callocz(1, sizeof(struct module));
420 +
421 + mod->type = str2_module_type(words[MODULE_TYPE_IDX]);
422 + if (unlikely(mod->type == MOD_TYPE_UNKNOWN)) {
423 + freez(mod);
424 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "unknown module type (allowed: job_array, single)");
425 + }
426 +
427 + mod->name = strdupz(words[MODULE_NAME_IDX]);
428 +
429 + mod->set_config_cb = set_module_config_cb;
430 + mod->get_config_cb = get_module_config_cb;
431 + mod->get_config_schema_cb = get_module_config_schema_cb;
432 + mod->config_cb_usr_ctx = parser;
433 +
434 + mod->get_job_config_cb = get_job_config_cb;
435 + mod->get_job_config_schema_cb = get_job_config_schema_cb;
436 + mod->set_job_config_cb = set_job_config_cb;
437 + mod->delete_job_cb = delete_job_cb;
438 + mod->job_config_cb_usr_ctx = parser;
439 +
440 + register_module(parser->user.host->configurable_plugins, plug_cfg, mod, SERVING_PLUGINSD(parser));
441 +
442 + if (di != NULL)
443 + dictionary_acquired_item_release(parser->user.host->configurable_plugins, di);
444 +
445 + rrdpush_send_dyncfg_reg_module(parser->user.host, plug_cfg->name, mod->name, mod->type);
446 +
447 + return PARSER_RC_OK;
448 +}
449 +
450 +static inline PARSER_RC pluginsd_register_job_common(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused, const char *plugin_name) {
451 + const char *module_name = words[0];
452 + const char *job_name = words[1];
453 + const char *job_type_str = words[2];
454 + const char *flags_str = words[3];
455 +
456 + long f = str2l(flags_str);
457 +
458 + if (f < 0)
459 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "invalid flags received");
460 +
461 + dyncfg_job_flg_t flags = f;
462 +
463 + if (SERVING_PLUGINSD(parser))
464 + flags |= JOB_FLG_PLUGIN_PUSHED;
465 + else
466 + flags |= JOB_FLG_STREAMING_PUSHED;
467 +
468 + enum job_type job_type = dyncfg_str2job_type(job_type_str);
469 + if (job_type == JOB_TYPE_UNKNOWN)
470 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "unknown job type");
471 +
472 + if (SERVING_PLUGINSD(parser) && job_type == JOB_TYPE_USER)
473 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "plugins cannot push jobs of type \"user\" (this is allowed only in streaming)");
474 +
475 + if (register_job(parser->user.host->configurable_plugins, plugin_name, module_name, job_name, job_type, flags, 0)) // ignore existing is off as this is explicitly called register job
476 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "error registering job");
477 +
478 + rrdpush_send_dyncfg_reg_job(parser->user.host, plugin_name, module_name, job_name, job_type, flags);
479 +
480 + return PARSER_RC_OK;
481 +}
482 +
483 +PARSER_RC pluginsd_register_job(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
484 + size_t expected_num_words = SERVING_PLUGINSD(parser) ? 5 : 6;
485 +
486 + if (unlikely(num_words != expected_num_words)) {
487 + char log[LOG_MSG_SIZE + 1];
488 + snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name job_name job_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name ");
489 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, log);
490 + }
491 +
492 + if (SERVING_PLUGINSD(parser)) {
493 + return pluginsd_register_job_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name);
494 + }
495 + return pluginsd_register_job_common(&words[2], num_words - 2, parser, words[1]);
496 +}
497 +
498 +PARSER_RC pluginsd_dyncfg_reset(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
499 + if (unlikely(num_words != (SERVING_PLUGINSD(parser) ? 1 : 2)))
500 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, SERVING_PLUGINSD(parser) ? "expected 0 parameters" : "expected 1 parameter: plugin_name");
501 +
502 + if (SERVING_PLUGINSD(parser)) {
503 + unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item);
504 + rrdpush_send_dyncfg_reset(parser->user.host, parser->user.cd->configuration->name);
505 + parser->user.cd->configuration = NULL;
506 + } else {
507 + const DICTIONARY_ITEM *di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]);
508 + if (unlikely(di == NULL))
509 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, "plugin not found");
510 + unregister_plugin(parser->user.host->configurable_plugins, di);
511 + rrdpush_send_dyncfg_reset(parser->user.host, words[1]);
512 + }
513 +
514 + return PARSER_RC_OK;
515 +}
516 +
517 +static inline PARSER_RC pluginsd_job_status_common(char **words, size_t num_words, PARSER *parser, const char *plugin_name) {
518 + int state = str2i(words[3]);
519 +
520 + enum job_status status = str2job_state(words[2]);
521 + if (unlikely(SERVING_PLUGINSD(parser) && status == JOB_STATUS_UNKNOWN))
522 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "unknown job status");
523 +
524 + char *message = NULL;
525 + if (num_words == 5 && strlen(words[4]) > 0)
526 + message = words[4];
527 +
528 + const DICTIONARY_ITEM *plugin_item;
529 + DICTIONARY *job_dict;
530 + const DICTIONARY_ITEM *job_item = report_job_status_acq_lock(parser->user.host->configurable_plugins, &plugin_item, &job_dict, plugin_name, words[0], words[1], status, state, message);
531 +
532 + if (job_item != NULL) {
533 + struct job *job = dictionary_acquired_item_value(job_item);
534 + rrdpush_send_job_status_update(parser->user.host, plugin_name, words[0], job);
535 +
536 + pthread_mutex_unlock(&job->lock);
537 + dictionary_acquired_item_release(job_dict, job_item);
538 + dictionary_acquired_item_release(parser->user.host->configurable_plugins, plugin_item);
539 + }
540 +
541 + return PARSER_RC_OK;
542 +}
543 +
544 +// job_status [plugin_name if streaming] <module_name> <job_name> <status_code> <state> [message]
545 +PARSER_RC pluginsd_job_status(char **words, size_t num_words, PARSER *parser) {
546 + if (SERVING_PLUGINSD(parser)) {
547 + if (unlikely(num_words != 5 && num_words != 6))
548 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 4 or 5 parameters: module_name, job_name, status_code, state, [optional: message]");
549 + } else {
550 + if (unlikely(num_words != 6 && num_words != 7))
551 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 5 or 6 parameters: plugin_name, module_name, job_name, status_code, state, [optional: message]");
552 + }
553 +
554 + if (SERVING_PLUGINSD(parser)) {
555 + return pluginsd_job_status_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name);
556 + }
557 + return pluginsd_job_status_common(&words[2], num_words - 2, parser, words[1]);
558 +}
559 +
560 +PARSER_RC pluginsd_delete_job(char **words, size_t num_words, PARSER *parser) {
561 + // this can confuse a bit but there is a diference between KEYWORD_DELETE_JOB and actual delete_job function
562 + // they are of opossite direction
563 + if (num_words != 4)
564 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DELETE_JOB, "expected 2 parameters: plugin_name, module_name, job_name");
565 +
566 + const char *plugin_name = get_word(words, num_words, 1);
567 + const char *module_name = get_word(words, num_words, 2);
568 + const char *job_name = get_word(words, num_words, 3);
569 +
570 + if (SERVING_STREAMING(parser))
571 + delete_job_pname(parser->user.host->configurable_plugins, plugin_name, module_name, job_name);
572 +
573 + // forward to parent if any
574 + rrdpush_send_job_deleted(parser->user.host, plugin_name, module_name, job_name);
575 + return PARSER_RC_OK;
576 +}
577 +
578 +void pluginsd_dyncfg_cleanup(PARSER *parser) {
579 + if (parser->user.cd != NULL && parser->user.cd->configuration != NULL) {
580 + unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item);
581 + parser->user.cd->configuration = NULL;
582 + } else if (parser->user.host != NULL && SERVING_STREAMING(parser) && parser->user.host != localhost){
583 + dictionary_flush(parser->user.host->configurable_plugins);
584 + }
585 +}
collectors/plugins.d/pluginsd_dyncfg.h new
+17
@@ -0,0 +1,17 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_PLUGINSD_DYNCFG_H
4 +#define NETDATA_PLUGINSD_DYNCFG_H
5 +
6 +#include "pluginsd_internals.h"
7 +
8 +PARSER_RC pluginsd_register_plugin(char **words, size_t num_words, PARSER *parser);
9 +PARSER_RC pluginsd_register_module(char **words, size_t num_words, PARSER *parser);
10 +PARSER_RC pluginsd_register_job(char **words, size_t num_words, PARSER *parser);
11 +PARSER_RC pluginsd_dyncfg_reset(char **words, size_t num_words, PARSER *parser);
12 +PARSER_RC pluginsd_job_status(char **words, size_t num_words, PARSER *parser);
13 +PARSER_RC pluginsd_delete_job(char **words, size_t num_words, PARSER *parser);
14 +
15 +void pluginsd_dyncfg_cleanup(PARSER *parser);
16 +
17 +#endif //NETDATA_PLUGINSD_DYNCFG_H
collectors/plugins.d/pluginsd_functions.c new
+461
@@ -0,0 +1,461 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "pluginsd_functions.h"
4 +
5 +#define LOG_FUNCTIONS false
6 +
7 +// ----------------------------------------------------------------------------
8 +// execution of functions
9 +
10 +static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void *func, void *parser_ptr) {
11 + struct inflight_function *pf = func;
12 +
13 + PARSER *parser = parser_ptr;
14 +
15 + // leave this code as default, so that when the dictionary is destroyed this will be sent back to the caller
16 + pf->code = HTTP_RESP_GATEWAY_TIMEOUT;
17 +
18 + const char *transaction = dictionary_acquired_item_name(item);
19 +
20 + int rc = uuid_parse_flexi(transaction, pf->transaction);
21 + if(rc != 0)
22 + netdata_log_error("FUNCTION: '%s': cannot parse transaction UUID", string2str(pf->function));
23 +
24 + char buffer[2048 + 1];
25 + snprintfz(buffer, sizeof(buffer) - 1, "%s %s %d \"%s\"\n",
26 + pf->payload ? PLUGINSD_KEYWORD_FUNCTION_PAYLOAD : PLUGINSD_KEYWORD_FUNCTION,
27 + transaction,
28 + pf->timeout_s,
29 + string2str(pf->function));
30 +
31 + // send the command to the plugin
32 + ssize_t ret = send_to_plugin(buffer, parser);
33 +
34 + pf->sent_monotonic_ut = now_monotonic_usec();
35 +
36 + if(ret < 0) {
37 + netdata_log_error("FUNCTION '%s': failed to send it to the plugin, error %zd", string2str(pf->function), ret);
38 + rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE);
39 + }
40 + else {
41 + internal_error(LOG_FUNCTIONS,
42 + "FUNCTION '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)",
43 + string2str(pf->function), dictionary_acquired_item_name(item), ret,
44 + pf->sent_monotonic_ut - pf->started_monotonic_ut);
45 + }
46 +
47 + if (!pf->payload)
48 + return;
49 +
50 + // send the payload to the plugin
51 + ret = send_to_plugin(pf->payload, parser);
52 +
53 + if(ret < 0) {
54 + netdata_log_error("FUNCTION_PAYLOAD '%s': failed to send function to plugin, error %zd", string2str(pf->function), ret);
55 + rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE);
56 + }
57 + else {
58 + internal_error(LOG_FUNCTIONS,
59 + "FUNCTION_PAYLOAD '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)",
60 + string2str(pf->function), dictionary_acquired_item_name(item), ret,
61 + pf->sent_monotonic_ut - pf->started_monotonic_ut);
62 + }
63 +
64 + send_to_plugin("\nFUNCTION_PAYLOAD_END\n", parser);
65 +}
66 +
67 +static bool inflight_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused, void *new_func, void *parser_ptr __maybe_unused) {
68 + struct inflight_function *pf = new_func;
69 +
70 + netdata_log_error("PLUGINSD_PARSER: duplicate UUID on pending function '%s' detected. Ignoring the second one.", string2str(pf->function));
71 + pf->code = rrd_call_function_error(pf->result_body_wb, "This request is already in progress", HTTP_RESP_BAD_REQUEST);
72 + pf->result.cb(pf->result_body_wb, pf->code, pf->result.data);
73 + string_freez(pf->function);
74 +
75 + return false;
76 +}
77 +
78 +static void delete_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug, const char *fnc_sig, int code) {
79 + if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED)
80 + return;
81 +
82 + char *params_local = strdupz(fnc_sig);
83 + char *words[DYNCFG_MAX_WORDS];
84 + size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd);
85 +
86 + if (words_c != 3) {
87 + netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for delete_job");
88 + freez(params_local);
89 + return;
90 + }
91 +
92 + const char *module = words[1];
93 + const char *job = words[2];
94 +
95 + delete_job(plug, module, job);
96 +
97 + unlink_job(plug->name, module, job);
98 +
99 + rrdpush_send_job_deleted(localhost, plug->name, module, job);
100 +
101 + freez(params_local);
102 +}
103 +
104 +static void set_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug __maybe_unused, const char *fnc_sig, int code) {
105 + if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED)
106 + return;
107 +
108 + char *params_local = strdupz(fnc_sig);
109 + char *words[DYNCFG_MAX_WORDS];
110 + size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd);
111 +
112 + if (words_c != 3) {
113 + netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for set_job_config");
114 + freez(params_local);
115 + return;
116 + }
117 +
118 + const char *module_name = get_word(words, words_c, 1);
119 + const char *job_name = get_word(words, words_c, 2);
120 +
121 + if (register_job(parser->user.host->configurable_plugins, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED, 1)) {
122 + freez(params_local);
123 + return;
124 + }
125 +
126 + // only send this if it is not existing already (register_job cares for that)
127 + rrdpush_send_dyncfg_reg_job(localhost, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED);
128 +
129 + freez(params_local);
130 +}
131 +
132 +static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *parser_ptr) {
133 + struct inflight_function *pf = func;
134 + struct parser *parser = (struct parser *)parser_ptr;
135 +
136 + internal_error(LOG_FUNCTIONS,
137 + "FUNCTION '%s' result of transaction '%s' received from collector (%zu bytes, request %"PRIu64" usec, response %"PRIu64" usec)",
138 + string2str(pf->function), dictionary_acquired_item_name(item),
139 + buffer_strlen(pf->result_body_wb), pf->sent_monotonic_ut - pf->started_monotonic_ut, now_realtime_usec() - pf->sent_monotonic_ut);
140 +
141 + if (pf->virtual && SERVING_PLUGINSD(parser)) {
142 + if (pf->payload) {
143 + if (strncmp(string2str(pf->function), FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0)
144 + set_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code);
145 + dyn_conf_store_config(string2str(pf->function), pf->payload, parser->user.cd->configuration);
146 + } else if (strncmp(string2str(pf->function), FUNCTION_NAME_DELETE_JOB, strlen(FUNCTION_NAME_DELETE_JOB)) == 0) {
147 + delete_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code);
148 + }
149 + }
150 +
151 + pf->result.cb(pf->result_body_wb, pf->code, pf->result.data);
152 +
153 + string_freez(pf->function);
154 + freez((void *)pf->payload);
155 +}
156 +
157 +void pluginsd_inflight_functions_init(PARSER *parser) {
158 + parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions, 0);
159 + dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser);
160 + dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser);
161 + dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser);
162 +}
163 +
164 +void pluginsd_inflight_functions_cleanup(PARSER *parser) {
165 + dictionary_destroy(parser->inflight.functions);
166 +}
167 +
168 +// ----------------------------------------------------------------------------
169 +
170 +void pluginsd_inflight_functions_garbage_collect(PARSER *parser, usec_t now_ut) {
171 + parser->inflight.smaller_monotonic_timeout_ut = 0;
172 + struct inflight_function *pf;
173 + dfe_start_write(parser->inflight.functions, pf) {
174 + if (*pf->stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < now_ut) {
175 + internal_error(true,
176 + "FUNCTION '%s' removing expired transaction '%s', after %"PRIu64" usec.",
177 + string2str(pf->function), pf_dfe.name, now_ut - pf->started_monotonic_ut);
178 +
179 + if(!buffer_strlen(pf->result_body_wb) || pf->code == HTTP_RESP_OK)
180 + pf->code = rrd_call_function_error(pf->result_body_wb,
181 + "Timeout waiting for collector response.",
182 + HTTP_RESP_GATEWAY_TIMEOUT);
183 +
184 + dictionary_del(parser->inflight.functions, pf_dfe.name);
185 + }
186 +
187 + else if(!parser->inflight.smaller_monotonic_timeout_ut || *pf->stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut)
188 + parser->inflight.smaller_monotonic_timeout_ut = *pf->stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
189 + }
190 + dfe_done(pf);
191 +}
192 +
193 +// ----------------------------------------------------------------------------
194 +
195 +static void pluginsd_function_cancel(void *data) {
196 + struct inflight_function *look_for = data, *t;
197 +
198 + bool sent = false;
199 + dfe_start_read(look_for->parser->inflight.functions, t) {
200 + if(look_for == t) {
201 + const char *transaction = t_dfe.name;
202 +
203 + internal_error(true, "PLUGINSD: sending function cancellation to plugin for transaction '%s'", transaction);
204 +
205 + char buffer[2048 + 1];
206 + snprintfz(buffer, sizeof(buffer) - 1, "%s %s\n",
207 + PLUGINSD_KEYWORD_FUNCTION_CANCEL,
208 + transaction);
209 +
210 + // send the command to the plugin
211 + ssize_t ret = send_to_plugin(buffer, t->parser);
212 + if(ret < 0)
213 + sent = true;
214 +
215 + break;
216 + }
217 + }
218 + dfe_done(t);
219 +
220 + if(sent <= 0)
221 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
222 + "PLUGINSD: FUNCTION_CANCEL request didn't match any pending function requests in pluginsd.d.");
223 +}
224 +
225 +static void pluginsd_function_progress_to_plugin(void *data) {
226 + struct inflight_function *look_for = data, *t;
227 +
228 + bool sent = false;
229 + dfe_start_read(look_for->parser->inflight.functions, t) {
230 + if(look_for == t) {
231 + const char *transaction = t_dfe.name;
232 +
233 + internal_error(true, "PLUGINSD: sending function progress to plugin for transaction '%s'", transaction);
234 +
235 + char buffer[2048 + 1];
236 + snprintfz(buffer, sizeof(buffer) - 1, "%s %s\n",
237 + PLUGINSD_KEYWORD_FUNCTION_PROGRESS,
238 + transaction);
239 +
240 + // send the command to the plugin
241 + ssize_t ret = send_to_plugin(buffer, t->parser);
242 + if(ret < 0)
243 + sent = true;
244 +
245 + break;
246 + }
247 + }
248 + dfe_done(t);
249 +
250 + if(sent <= 0)
251 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
252 + "PLUGINSD: FUNCTION_PROGRESS request didn't match any pending function requests in pluginsd.d.");
253 +}
254 +
255 +// this is the function called from
256 +// rrd_call_function_and_wait() and rrd_call_function_async()
257 +static int pluginsd_function_execute_cb(uuid_t *transaction, BUFFER *result_body_wb,
258 + usec_t *stop_monotonic_ut, const char *function,
259 + void *execute_cb_data,
260 + rrd_function_result_callback_t result_cb, void *result_cb_data,
261 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
262 + rrd_function_is_cancelled_cb_t is_cancelled_cb __maybe_unused,
263 + void *is_cancelled_cb_data __maybe_unused,
264 + rrd_function_register_canceller_cb_t register_canceller_cb,
265 + void *register_canceller_cb_data,
266 + rrd_function_register_progresser_cb_t register_progresser_cb,
267 + void *register_progresser_cb_data) {
268 + PARSER *parser = execute_cb_data;
269 +
270 + usec_t now_ut = now_monotonic_usec();
271 +
272 + int timeout_s = (*stop_monotonic_ut - now_ut + USEC_PER_SEC / 2) / USEC_PER_SEC;
273 +
274 + struct inflight_function tmp = {
275 + .started_monotonic_ut = now_ut,
276 + .stop_monotonic_ut = stop_monotonic_ut,
277 + .result_body_wb = result_body_wb,
278 + .timeout_s = timeout_s,
279 + .function = string_strdupz(function),
280 + .payload = NULL,
281 + .parser = parser,
282 +
283 + .result = {
284 + .cb = result_cb,
285 + .data = result_cb_data,
286 + },
287 + .progress = {
288 + .cb = progress_cb,
289 + .data = progress_cb_data,
290 + },
291 + };
292 + uuid_copy(tmp.transaction, *transaction);
293 +
294 + char transaction_str[UUID_COMPACT_STR_LEN];
295 + uuid_unparse_lower_compact(tmp.transaction, transaction_str);
296 +
297 + dictionary_write_lock(parser->inflight.functions);
298 +
299 + // if there is any error, our dictionary callbacks will call the caller callback to notify
300 + // the caller about the error - no need for error handling here.
301 + void *t = dictionary_set(parser->inflight.functions, transaction_str, &tmp, sizeof(struct inflight_function));
302 + if(register_canceller_cb)
303 + register_canceller_cb(register_canceller_cb_data, pluginsd_function_cancel, t);
304 +
305 + if(register_progresser_cb && (parser->repertoire == PARSER_INIT_PLUGINSD ||
306 + (parser->repertoire == PARSER_INIT_STREAMING && stream_has_capability(&parser->user, STREAM_CAP_PROGRESS))))
307 + register_progresser_cb(register_progresser_cb_data, pluginsd_function_progress_to_plugin, t);
308 +
309 + if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut)
310 + parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
311 +
312 + // garbage collect stale inflight functions
313 + if(parser->inflight.smaller_monotonic_timeout_ut < now_ut)
314 + pluginsd_inflight_functions_garbage_collect(parser, now_ut);
315 +
316 + dictionary_write_unlock(parser->inflight.functions);
317 +
318 + return HTTP_RESP_OK;
319 +}
320 +
321 +PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser) {
322 + // a plugin or a child is registering a function
323 +
324 + bool global = false;
325 + size_t i = 1;
326 + if(num_words >= 2 && strcmp(get_word(words, num_words, 1), "GLOBAL") == 0) {
327 + i++;
328 + global = true;
329 + }
330 +
331 + char *name = get_word(words, num_words, i++);
332 + char *timeout_str = get_word(words, num_words, i++);
333 + char *help = get_word(words, num_words, i++);
334 + char *tags = get_word(words, num_words, i++);
335 + char *access_str = get_word(words, num_words, i++);
336 + char *priority_str = get_word(words, num_words, i++);
337 +
338 + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_FUNCTION);
339 + if(!host) return PARSER_RC_ERROR;
340 +
341 + RRDSET *st = (global)? NULL: pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_FUNCTION, PLUGINSD_KEYWORD_CHART);
342 + if(!st) global = true;
343 +
344 + if (unlikely(!timeout_str || !name || !help || (!global && !st))) {
345 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a FUNCTION, without providing the required data (global = '%s', name = '%s', timeout = '%s', help = '%s'). Ignoring it.",
346 + rrdhost_hostname(host),
347 + st?rrdset_id(st):"(unset)",
348 + global?"yes":"no",
349 + name?name:"(unset)",
350 + timeout_str ? timeout_str : "(unset)",
351 + help?help:"(unset)"
352 + );
353 + return PARSER_RC_ERROR;
354 + }
355 +
356 + int timeout_s = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
357 + if (timeout_str && *timeout_str) {
358 + timeout_s = str2i(timeout_str);
359 + if (unlikely(timeout_s <= 0))
360 + timeout_s = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
361 + }
362 +
363 + int priority = RRDFUNCTIONS_PRIORITY_DEFAULT;
364 + if(priority_str && *priority_str) {
365 + priority = str2i(priority_str);
366 + if(priority <= 0)
367 + priority = RRDFUNCTIONS_PRIORITY_DEFAULT;
368 + }
369 +
370 + rrd_function_add(host, st, name, timeout_s, priority, help, tags,
371 + http_access2id(access_str), false,
372 + pluginsd_function_execute_cb, parser);
373 +
374 + parser->user.data_collections_count++;
375 +
376 + return PARSER_RC_OK;
377 +}
378 +
379 +static void pluginsd_function_result_end(struct parser *parser, void *action_data) {
380 + STRING *key = action_data;
381 + if(key)
382 + dictionary_del(parser->inflight.functions, string2str(key));
383 + string_freez(key);
384 +
385 + parser->user.data_collections_count++;
386 +}
387 +
388 +static inline struct inflight_function *inflight_function_find(PARSER *parser, const char *transaction) {
389 + struct inflight_function *pf = NULL;
390 +
391 + if(transaction && *transaction)
392 + pf = (struct inflight_function *)dictionary_get(parser->inflight.functions, transaction);
393 +
394 + if(!pf)
395 + netdata_log_error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " for transaction '%s', but the transaction is not found.", transaction ? transaction : "(unset)");
396 +
397 + return pf;
398 +}
399 +
400 +PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, PARSER *parser) {
401 + char *transaction = get_word(words, num_words, 1);
402 + char *status = get_word(words, num_words, 2);
403 + char *format = get_word(words, num_words, 3);
404 + char *expires = get_word(words, num_words, 4);
405 +
406 + if (unlikely(!transaction || !*transaction || !status || !*status || !format || !*format || !expires || !*expires)) {
407 + netdata_log_error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " without providing the required data (key = '%s', status = '%s', format = '%s', expires = '%s')."
408 + , transaction ? transaction : "(unset)"
409 + , status ? status : "(unset)"
410 + , format ? format : "(unset)"
411 + , expires ? expires : "(unset)"
412 + );
413 + }
414 +
415 + int code = (status && *status) ? str2i(status) : 0;
416 + if (code <= 0)
417 + code = HTTP_RESP_BACKEND_RESPONSE_INVALID;
418 +
419 + time_t expiration = (expires && *expires) ? str2l(expires) : 0;
420 +
421 + struct inflight_function *pf = inflight_function_find(parser, transaction);
422 + if(pf) {
423 + if(format && *format)
424 + pf->result_body_wb->content_type = functions_format_to_content_type(format);
425 +
426 + pf->code = code;
427 +
428 + pf->result_body_wb->expires = expiration;
429 + if(expiration <= now_realtime_sec())
430 + buffer_no_cacheable(pf->result_body_wb);
431 + else
432 + buffer_cacheable(pf->result_body_wb);
433 + }
434 +
435 + parser->defer.response = (pf) ? pf->result_body_wb : NULL;
436 + parser->defer.end_keyword = PLUGINSD_KEYWORD_FUNCTION_RESULT_END;
437 + parser->defer.action = pluginsd_function_result_end;
438 + parser->defer.action_data = string_strdupz(transaction); // it is ok is key is NULL
439 + parser->flags |= PARSER_DEFER_UNTIL_KEYWORD;
440 +
441 + return PARSER_RC_OK;
442 +}
443 +
444 +PARSER_RC pluginsd_function_progress(char **words, size_t num_words, PARSER *parser) {
445 + size_t i = 1;
446 +
447 + char *transaction = get_word(words, num_words, i++);
448 + char *done_str = get_word(words, num_words, i++);
449 + char *all_str = get_word(words, num_words, i++);
450 +
451 + struct inflight_function *pf = inflight_function_find(parser, transaction);
452 + if(pf) {
453 + size_t done = done_str && *done_str ? str2u(done_str) : 0;
454 + size_t all = all_str && *all_str ? str2u(all_str) : 0;
455 +
456 + if(pf->progress.cb)
457 + pf->progress.cb(pf->progress.data, done, all);
458 + }
459 +
460 + return PARSER_RC_OK;
461 +}
collectors/plugins.d/pluginsd_functions.h new
+45
@@ -0,0 +1,45 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_PLUGINSD_FUNCTIONS_H
4 +#define NETDATA_PLUGINSD_FUNCTIONS_H
5 +
6 +#include "pluginsd_internals.h"
7 +
8 +struct inflight_function {
9 + uuid_t transaction;
10 +
11 + int code;
12 + int timeout_s;
13 + STRING *function;
14 + BUFFER *result_body_wb;
15 + usec_t *stop_monotonic_ut; // pointer to caller data
16 + usec_t started_monotonic_ut;
17 + usec_t sent_monotonic_ut;
18 + const char *payload;
19 + PARSER *parser;
20 + bool virtual;
21 +
22 + struct {
23 + rrd_function_result_callback_t cb;
24 + void *data;
25 + } result;
26 +
27 + struct {
28 + rrd_function_progress_cb_t cb;
29 + void *data;
30 + } progress;
31 +
32 + struct {
33 + usec_t stop_monotonic_ut;
34 + } dyncfg;
35 +};
36 +
37 +PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser);
38 +PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, PARSER *parser);
39 +PARSER_RC pluginsd_function_progress(char **words, size_t num_words, PARSER *parser);
40 +
41 +void pluginsd_inflight_functions_init(PARSER *parser);
42 +void pluginsd_inflight_functions_cleanup(PARSER *parser);
43 +void pluginsd_inflight_functions_garbage_collect(PARSER *parser, usec_t now_ut);
44 +
45 +#endif //NETDATA_PLUGINSD_FUNCTIONS_H
collectors/plugins.d/pluginsd_internals.c new
+121
@@ -0,0 +1,121 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "pluginsd_internals.h"
4 +
5 +ssize_t send_to_plugin(const char *txt, void *data) {
6 + PARSER *parser = data;
7 +
8 + if(!txt || !*txt)
9 + return 0;
10 +
11 +#ifdef ENABLE_H2O
12 + if(parser->h2o_ctx)
13 + return h2o_stream_write(parser->h2o_ctx, txt, strlen(txt));
14 +#endif
15 +
16 + errno = 0;
17 + spinlock_lock(&parser->writer.spinlock);
18 + ssize_t bytes = -1;
19 +
20 +#ifdef ENABLE_HTTPS
21 + NETDATA_SSL *ssl = parser->ssl_output;
22 + if(ssl) {
23 +
24 + if(SSL_connection(ssl))
25 + bytes = netdata_ssl_write(ssl, (void *) txt, strlen(txt));
26 +
27 + else
28 + netdata_log_error("PLUGINSD: cannot send command (SSL)");
29 +
30 + spinlock_unlock(&parser->writer.spinlock);
31 + return bytes;
32 + }
33 +#endif
34 +
35 + if(parser->fp_output) {
36 +
37 + bytes = fprintf(parser->fp_output, "%s", txt);
38 + if(bytes <= 0) {
39 + netdata_log_error("PLUGINSD: cannot send command (FILE)");
40 + bytes = -2;
41 + }
42 + else
43 + fflush(parser->fp_output);
44 +
45 + spinlock_unlock(&parser->writer.spinlock);
46 + return bytes;
47 + }
48 +
49 + if(parser->fd != -1) {
50 + bytes = 0;
51 + ssize_t total = (ssize_t)strlen(txt);
52 + ssize_t sent;
53 +
54 + do {
55 + sent = write(parser->fd, &txt[bytes], total - bytes);
56 + if(sent <= 0) {
57 + netdata_log_error("PLUGINSD: cannot send command (fd)");
58 + spinlock_unlock(&parser->writer.spinlock);
59 + return -3;
60 + }
61 + bytes += sent;
62 + }
63 + while(bytes < total);
64 +
65 + spinlock_unlock(&parser->writer.spinlock);
66 + return (int)bytes;
67 + }
68 +
69 + spinlock_unlock(&parser->writer.spinlock);
70 + netdata_log_error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)");
71 + return -4;
72 +}
73 +
74 +PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyword, const char *msg) {
75 + parser->user.enabled = 0;
76 +
77 + if(keyword && msg) {
78 + nd_log_limit_static_global_var(erl, 1, 0);
79 + nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_INFO,
80 + "PLUGINSD: keyword %s: %s", keyword, msg);
81 + }
82 +
83 + return PARSER_RC_ERROR;
84 +}
85 +
86 +void pluginsd_keywords_init(PARSER *parser, PARSER_REPERTOIRE repertoire) {
87 + parser_init_repertoire(parser, repertoire);
88 +
89 + if (repertoire & (PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING))
90 + pluginsd_inflight_functions_init(parser);
91 +}
92 +
93 +void parser_destroy(PARSER *parser) {
94 + if (unlikely(!parser))
95 + return;
96 +
97 + pluginsd_dyncfg_cleanup(parser);
98 + pluginsd_inflight_functions_cleanup(parser);
99 +
100 + freez(parser);
101 +}
102 +
103 +
104 +PARSER *parser_init(struct parser_user_object *user, FILE *fp_input, FILE *fp_output, int fd,
105 + PARSER_INPUT_TYPE flags, void *ssl __maybe_unused) {
106 + PARSER *parser;
107 +
108 + parser = callocz(1, sizeof(*parser));
109 + if(user)
110 + parser->user = *user;
111 + parser->fd = fd;
112 + parser->fp_input = fp_input;
113 + parser->fp_output = fp_output;
114 +#ifdef ENABLE_HTTPS
115 + parser->ssl_output = ssl;
116 +#endif
117 + parser->flags = flags;
118 +
119 + spinlock_init(&parser->writer.spinlock);
120 + return parser;
121 +}
collectors/plugins.d/pluginsd_internals.h new
+355
@@ -0,0 +1,355 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_PLUGINSD_INTERNALS_H
4 +#define NETDATA_PLUGINSD_INTERNALS_H
5 +
6 +#include "pluginsd_parser.h"
7 +#include "pluginsd_functions.h"
8 +#include "pluginsd_dyncfg.h"
9 +#include "pluginsd_replication.h"
10 +
11 +#define SERVING_STREAMING(parser) ((parser)->repertoire == PARSER_INIT_STREAMING)
12 +#define SERVING_PLUGINSD(parser) ((parser)->repertoire == PARSER_INIT_PLUGINSD)
13 +
14 +PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyword, const char *msg);
15 +
16 +ssize_t send_to_plugin(const char *txt, void *data);
17 +
18 +static inline RRDHOST *pluginsd_require_scope_host(PARSER *parser, const char *cmd) {
19 + RRDHOST *host = parser->user.host;
20 +
21 + if(unlikely(!host))
22 + netdata_log_error("PLUGINSD: command %s requires a host, but is not set.", cmd);
23 +
24 + return host;
25 +}
26 +
27 +static inline RRDSET *pluginsd_require_scope_chart(PARSER *parser, const char *cmd, const char *parent_cmd) {
28 + RRDSET *st = parser->user.st;
29 +
30 + if(unlikely(!st))
31 + netdata_log_error("PLUGINSD: command %s requires a chart defined via command %s, but is not set.", cmd, parent_cmd);
32 +
33 + return st;
34 +}
35 +
36 +static inline RRDSET *pluginsd_get_scope_chart(PARSER *parser) {
37 + return parser->user.st;
38 +}
39 +
40 +static inline void pluginsd_lock_rrdset_data_collection(PARSER *parser) {
41 + if(parser->user.st && !parser->user.v2.locked_data_collection) {
42 + spinlock_lock(&parser->user.st->data_collection_lock);
43 + parser->user.v2.locked_data_collection = true;
44 + }
45 +}
46 +
47 +static inline bool pluginsd_unlock_rrdset_data_collection(PARSER *parser) {
48 + if(parser->user.st && parser->user.v2.locked_data_collection) {
49 + spinlock_unlock(&parser->user.st->data_collection_lock);
50 + parser->user.v2.locked_data_collection = false;
51 + return true;
52 + }
53 +
54 + return false;
55 +}
56 +
57 +static inline void pluginsd_unlock_previous_scope_chart(PARSER *parser, const char *keyword, bool stale) {
58 + if(unlikely(pluginsd_unlock_rrdset_data_collection(parser))) {
59 + if(stale)
60 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s/' stale data collection lock found during %s; it has been unlocked",
61 + rrdhost_hostname(parser->user.st->rrdhost),
62 + rrdset_id(parser->user.st),
63 + keyword);
64 + }
65 +
66 + if(unlikely(parser->user.v2.ml_locked)) {
67 + ml_chart_update_end(parser->user.st);
68 + parser->user.v2.ml_locked = false;
69 +
70 + if(stale)
71 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s/' stale ML lock found during %s, it has been unlocked",
72 + rrdhost_hostname(parser->user.st->rrdhost),
73 + rrdset_id(parser->user.st),
74 + keyword);
75 + }
76 +}
77 +
78 +static inline void pluginsd_clear_scope_chart(PARSER *parser, const char *keyword) {
79 + pluginsd_unlock_previous_scope_chart(parser, keyword, true);
80 +
81 + if(parser->user.cleanup_slots && parser->user.st)
82 + rrdset_pluginsd_receive_unslot(parser->user.st);
83 +
84 + parser->user.st = NULL;
85 + parser->user.cleanup_slots = false;
86 +}
87 +
88 +static inline bool pluginsd_set_scope_chart(PARSER *parser, RRDSET *st, const char *keyword) {
89 + RRDSET *old_st = parser->user.st;
90 + pid_t old_collector_tid = (old_st) ? old_st->pluginsd.collector_tid : 0;
91 + pid_t my_collector_tid = gettid();
92 +
93 + if(unlikely(old_collector_tid)) {
94 + if(old_collector_tid != my_collector_tid) {
95 + nd_log_limit_static_global_var(erl, 1, 0);
96 + nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_WARNING,
97 + "PLUGINSD: keyword %s: 'host:%s/chart:%s' is collected twice (my tid %d, other collector tid %d)",
98 + keyword ? keyword : "UNKNOWN",
99 + rrdhost_hostname(st->rrdhost), rrdset_id(st),
100 + my_collector_tid, old_collector_tid);
101 +
102 + return false;
103 + }
104 +
105 + old_st->pluginsd.collector_tid = 0;
106 + }
107 +
108 + st->pluginsd.collector_tid = my_collector_tid;
109 +
110 + pluginsd_clear_scope_chart(parser, keyword);
111 +
112 + st->pluginsd.pos = 0;
113 + parser->user.st = st;
114 + parser->user.cleanup_slots = false;
115 +
116 + return true;
117 +}
118 +
119 +static inline void pluginsd_rrddim_put_to_slot(PARSER *parser, RRDSET *st, RRDDIM *rd, ssize_t slot, bool obsolete) {
120 + size_t wanted_size = st->pluginsd.size;
121 +
122 + if(slot >= 1) {
123 + st->pluginsd.dims_with_slots = true;
124 + wanted_size = slot;
125 + }
126 + else {
127 + st->pluginsd.dims_with_slots = false;
128 + wanted_size = dictionary_entries(st->rrddim_root_index);
129 + }
130 +
131 + if(wanted_size > st->pluginsd.size) {
132 + st->pluginsd.prd_array = reallocz(st->pluginsd.prd_array, wanted_size * sizeof(struct pluginsd_rrddim));
133 +
134 + // initialize the empty slots
135 + for(ssize_t i = (ssize_t) wanted_size - 1; i >= (ssize_t) st->pluginsd.size; i--) {
136 + st->pluginsd.prd_array[i].rda = NULL;
137 + st->pluginsd.prd_array[i].rd = NULL;
138 + st->pluginsd.prd_array[i].id = NULL;
139 + }
140 +
141 + st->pluginsd.size = wanted_size;
142 + }
143 +
144 + if(st->pluginsd.dims_with_slots) {
145 + struct pluginsd_rrddim *prd = &st->pluginsd.prd_array[slot - 1];
146 +
147 + if(prd->rd != rd) {
148 + prd->rda = rrddim_find_and_acquire(st, string2str(rd->id));
149 + prd->rd = rrddim_acquired_to_rrddim(prd->rda);
150 + prd->id = string2str(prd->rd->id);
151 + }
152 +
153 + if(obsolete)
154 + parser->user.cleanup_slots = true;
155 + }
156 +}
157 +
158 +static inline RRDDIM *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *st, const char *dimension, ssize_t slot, const char *cmd) {
159 + if (unlikely(!dimension || !*dimension)) {
160 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, without a dimension.",
161 + rrdhost_hostname(host), rrdset_id(st), cmd);
162 + return NULL;
163 + }
164 +
165 + if (unlikely(!st->pluginsd.size)) {
166 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, but the chart has no dimensions.",
167 + rrdhost_hostname(host), rrdset_id(st), cmd);
168 + return NULL;
169 + }
170 +
171 + struct pluginsd_rrddim *prd;
172 + RRDDIM *rd;
173 +
174 + if(likely(st->pluginsd.dims_with_slots)) {
175 + // caching with slots
176 +
177 + if(unlikely(slot < 1 || slot > st->pluginsd.size)) {
178 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s with slot %zd, but slots in the range [1 - %u] are expected.",
179 + rrdhost_hostname(host), rrdset_id(st), cmd, slot, st->pluginsd.size);
180 + return NULL;
181 + }
182 +
183 + prd = &st->pluginsd.prd_array[slot - 1];
184 +
185 + rd = prd->rd;
186 + if(likely(rd)) {
187 +#ifdef NETDATA_INTERNAL_CHECKS
188 + if(strcmp(prd->id, dimension) != 0) {
189 + ssize_t t;
190 + for(t = 0; t < st->pluginsd.size ;t++) {
191 + if (strcmp(st->pluginsd.prd_array[t].id, dimension) == 0)
192 + break;
193 + }
194 + if(t >= st->pluginsd.size)
195 + t = -1;
196 +
197 + internal_fatal(true,
198 + "PLUGINSD: expected to find dimension '%s' on slot %zd, but found '%s', "
199 + "the right slot is %zd",
200 + dimension, slot, prd->id, t);
201 + }
202 +#endif
203 + return rd;
204 + }
205 + }
206 + else {
207 + // caching without slots
208 +
209 + if(unlikely(st->pluginsd.pos >= st->pluginsd.size))
210 + st->pluginsd.pos = 0;
211 +
212 + prd = &st->pluginsd.prd_array[st->pluginsd.pos++];
213 +
214 + rd = prd->rd;
215 + if(likely(rd)) {
216 + const char *id = prd->id;
217 +
218 + if(strcmp(id, dimension) == 0) {
219 + // we found it cached
220 + return rd;
221 + }
222 + else {
223 + // the cached one is not good for us
224 + rrddim_acquired_release(prd->rda);
225 + prd->rda = NULL;
226 + prd->rd = NULL;
227 + prd->id = NULL;
228 + }
229 + }
230 + }
231 +
232 + // we need to find the dimension and set it to prd
233 +
234 + RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(st, dimension);
235 + if (unlikely(!rda)) {
236 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s but dimension does not exist.",
237 + rrdhost_hostname(host), rrdset_id(st), dimension, cmd);
238 +
239 + return NULL;
240 + }
241 +
242 + prd->rda = rda;
243 + prd->rd = rd = rrddim_acquired_to_rrddim(rda);
244 + prd->id = string2str(rd->id);
245 +
246 + return rd;
247 +}
248 +
249 +static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, const char *cmd) {
250 + if (unlikely(!chart || !*chart)) {
251 + netdata_log_error("PLUGINSD: 'host:%s' got a %s without a chart id.",
252 + rrdhost_hostname(host), cmd);
253 + return NULL;
254 + }
255 +
256 + RRDSET *st = rrdset_find(host, chart);
257 + if (unlikely(!st))
258 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s but chart does not exist.",
259 + rrdhost_hostname(host), chart, cmd);
260 +
261 + return st;
262 +}
263 +
264 +static inline ssize_t pluginsd_parse_rrd_slot(char **words, size_t num_words) {
265 + ssize_t slot = -1;
266 + char *id = get_word(words, num_words, 1);
267 + if(id && id[0] == PLUGINSD_KEYWORD_SLOT[0] && id[1] == PLUGINSD_KEYWORD_SLOT[1] &&
268 + id[2] == PLUGINSD_KEYWORD_SLOT[2] && id[3] == PLUGINSD_KEYWORD_SLOT[3] && id[4] == ':') {
269 + slot = (ssize_t) str2ull_encoded(&id[5]);
270 + if(slot < 0) slot = 0; // to make the caller increment its idx of the words
271 + }
272 +
273 + return slot;
274 +}
275 +
276 +static inline void pluginsd_rrdset_cache_put_to_slot(PARSER *parser, RRDSET *st, ssize_t slot, bool obsolete) {
277 + // clean possible old cached data
278 + rrdset_pluginsd_receive_unslot(st);
279 +
280 + if(unlikely(slot < 1 || slot >= INT32_MAX))
281 + return;
282 +
283 + RRDHOST *host = st->rrdhost;
284 +
285 + if(unlikely((size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size)) {
286 + spinlock_lock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock);
287 + size_t old_slots = host->rrdpush.receive.pluginsd_chart_slots.size;
288 + size_t new_slots = (old_slots < PLUGINSD_MIN_RRDSET_POINTERS_CACHE) ? PLUGINSD_MIN_RRDSET_POINTERS_CACHE : old_slots * 2;
289 +
290 + if(new_slots < (size_t)slot)
291 + new_slots = slot;
292 +
293 + host->rrdpush.receive.pluginsd_chart_slots.array =
294 + reallocz(host->rrdpush.receive.pluginsd_chart_slots.array, new_slots * sizeof(RRDSET *));
295 +
296 + for(size_t i = old_slots; i < new_slots ;i++)
297 + host->rrdpush.receive.pluginsd_chart_slots.array[i] = NULL;
298 +
299 + host->rrdpush.receive.pluginsd_chart_slots.size = new_slots;
300 + spinlock_unlock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock);
301 + }
302 +
303 + host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1] = st;
304 + st->pluginsd.last_slot = (int32_t)slot - 1;
305 + parser->user.cleanup_slots = obsolete;
306 +}
307 +
308 +static inline RRDSET *pluginsd_rrdset_cache_get_from_slot(PARSER *parser, RRDHOST *host, const char *id, ssize_t slot, const char *keyword) {
309 + if(unlikely(slot < 1 || (size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size))
310 + return pluginsd_find_chart(host, id, keyword);
311 +
312 + RRDSET *st = host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1];
313 +
314 + if(!st) {
315 + st = pluginsd_find_chart(host, id, keyword);
316 + if(st)
317 + pluginsd_rrdset_cache_put_to_slot(parser, st, slot, rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE));
318 + }
319 + else {
320 + internal_fatal(string_strcmp(st->id, id) != 0,
321 + "PLUGINSD: wrong chart in slot %zd, expected '%s', found '%s'",
322 + slot - 1, id, string2str(st->id));
323 + }
324 +
325 + return st;
326 +}
327 +
328 +static inline SN_FLAGS pluginsd_parse_storage_number_flags(const char *flags_str) {
329 + SN_FLAGS flags = SN_FLAG_NONE;
330 +
331 + char c;
332 + while ((c = *flags_str++)) {
333 + switch (c) {
334 + case 'A':
335 + flags |= SN_FLAG_NOT_ANOMALOUS;
336 + break;
337 +
338 + case 'R':
339 + flags |= SN_FLAG_RESET;
340 + break;
341 +
342 + case 'E':
343 + flags = SN_EMPTY_SLOT;
344 + return flags;
345 +
346 + default:
347 + internal_error(true, "Unknown SN_FLAGS flag '%c'", c);
348 + break;
349 + }
350 + }
351 +
352 + return flags;
353 +}
354 +
355 +#endif //NETDATA_PLUGINSD_INTERNALS_H
collectors/plugins.d/pluginsd_parser.c
+26 -1800
@@ -1,402 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "pluginsd_parser.h"
4 -
5 -#define LOG_FUNCTIONS false
6 -
7 -#define SERVING_STREAMING(parser) ((parser)->repertoire == PARSER_INIT_STREAMING)
8 -#define SERVING_PLUGINSD(parser) ((parser)->repertoire == PARSER_INIT_PLUGINSD)
9 -
10 -static ssize_t send_to_plugin(const char *txt, void *data) {
11 - PARSER *parser = data;
12 -
13 - if(!txt || !*txt)
14 - return 0;
15 -
16 -#ifdef ENABLE_H2O
17 - if(parser->h2o_ctx)
18 - return h2o_stream_write(parser->h2o_ctx, txt, strlen(txt));
19 -#endif
20 -
21 - errno = 0;
22 - spinlock_lock(&parser->writer.spinlock);
23 - ssize_t bytes = -1;
24 -
25 -#ifdef ENABLE_HTTPS
26 - NETDATA_SSL *ssl = parser->ssl_output;
27 - if(ssl) {
28 -
29 - if(SSL_connection(ssl))
30 - bytes = netdata_ssl_write(ssl, (void *) txt, strlen(txt));
31 -
32 - else
33 - netdata_log_error("PLUGINSD: cannot send command (SSL)");
34 -
35 - spinlock_unlock(&parser->writer.spinlock);
36 - return bytes;
37 - }
38 -#endif
39 -
40 - if(parser->fp_output) {
41 -
42 - bytes = fprintf(parser->fp_output, "%s", txt);
43 - if(bytes <= 0) {
44 - netdata_log_error("PLUGINSD: cannot send command (FILE)");
45 - bytes = -2;
46 - }
47 - else
48 - fflush(parser->fp_output);
49 -
50 - spinlock_unlock(&parser->writer.spinlock);
51 - return bytes;
52 - }
53 -
54 - if(parser->fd != -1) {
55 - bytes = 0;
56 - ssize_t total = (ssize_t)strlen(txt);
57 - ssize_t sent;
58 -
59 - do {
60 - sent = write(parser->fd, &txt[bytes], total - bytes);
61 - if(sent <= 0) {
62 - netdata_log_error("PLUGINSD: cannot send command (fd)");
63 - spinlock_unlock(&parser->writer.spinlock);
64 - return -3;
65 - }
66 - bytes += sent;
67 - }
68 - while(bytes < total);
69 -
70 - spinlock_unlock(&parser->writer.spinlock);
71 - return (int)bytes;
72 - }
73 -
74 - spinlock_unlock(&parser->writer.spinlock);
75 - netdata_log_error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)");
76 - return -4;
77 -}
78 -
79 -static inline RRDHOST *pluginsd_require_scope_host(PARSER *parser, const char *cmd) {
80 - RRDHOST *host = parser->user.host;
81 -
82 - if(unlikely(!host))
83 - netdata_log_error("PLUGINSD: command %s requires a host, but is not set.", cmd);
84 -
85 - return host;
86 -}
87 -
88 -static inline RRDSET *pluginsd_require_scope_chart(PARSER *parser, const char *cmd, const char *parent_cmd) {
89 - RRDSET *st = parser->user.st;
90 -
91 - if(unlikely(!st))
92 - netdata_log_error("PLUGINSD: command %s requires a chart defined via command %s, but is not set.", cmd, parent_cmd);
93 -
94 - return st;
95 -}
96 -
97 -static inline RRDSET *pluginsd_get_scope_chart(PARSER *parser) {
98 - return parser->user.st;
99 -}
100 -
101 -static inline void pluginsd_lock_rrdset_data_collection(PARSER *parser) {
102 - if(parser->user.st && !parser->user.v2.locked_data_collection) {
103 - spinlock_lock(&parser->user.st->data_collection_lock);
104 - parser->user.v2.locked_data_collection = true;
105 - }
106 -}
107 -
108 -static inline bool pluginsd_unlock_rrdset_data_collection(PARSER *parser) {
109 - if(parser->user.st && parser->user.v2.locked_data_collection) {
110 - spinlock_unlock(&parser->user.st->data_collection_lock);
111 - parser->user.v2.locked_data_collection = false;
112 - return true;
113 - }
114 -
115 - return false;
116 -}
117 -
118 -static inline void pluginsd_unlock_previous_scope_chart(PARSER *parser, const char *keyword, bool stale) {
119 - if(unlikely(pluginsd_unlock_rrdset_data_collection(parser))) {
120 - if(stale)
121 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s/' stale data collection lock found during %s; it has been unlocked",
122 - rrdhost_hostname(parser->user.st->rrdhost),
123 - rrdset_id(parser->user.st),
124 - keyword);
125 - }
126 -
127 - if(unlikely(parser->user.v2.ml_locked)) {
128 - ml_chart_update_end(parser->user.st);
129 - parser->user.v2.ml_locked = false;
130 -
131 - if(stale)
132 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s/' stale ML lock found during %s, it has been unlocked",
133 - rrdhost_hostname(parser->user.st->rrdhost),
134 - rrdset_id(parser->user.st),
135 - keyword);
136 - }
137 -}
138 -
139 -static inline void pluginsd_clear_scope_chart(PARSER *parser, const char *keyword) {
140 - pluginsd_unlock_previous_scope_chart(parser, keyword, true);
141 -
142 - if(parser->user.cleanup_slots && parser->user.st)
143 - rrdset_pluginsd_receive_unslot(parser->user.st);
144 -
145 - parser->user.st = NULL;
146 - parser->user.cleanup_slots = false;
147 -}
148 -
149 -static inline bool pluginsd_set_scope_chart(PARSER *parser, RRDSET *st, const char *keyword) {
150 - RRDSET *old_st = parser->user.st;
151 - pid_t old_collector_tid = (old_st) ? old_st->pluginsd.collector_tid : 0;
152 - pid_t my_collector_tid = gettid();
153 -
154 - if(unlikely(old_collector_tid)) {
155 - if(old_collector_tid != my_collector_tid) {
156 - nd_log_limit_static_global_var(erl, 1, 0);
157 - nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_WARNING,
158 - "PLUGINSD: keyword %s: 'host:%s/chart:%s' is collected twice (my tid %d, other collector tid %d)",
159 - keyword ? keyword : "UNKNOWN",
160 - rrdhost_hostname(st->rrdhost), rrdset_id(st),
161 - my_collector_tid, old_collector_tid);
162 -
163 - return false;
164 - }
165 -
166 - old_st->pluginsd.collector_tid = 0;
167 - }
168 -
169 - st->pluginsd.collector_tid = my_collector_tid;
170 -
171 - pluginsd_clear_scope_chart(parser, keyword);
172 -
173 - st->pluginsd.pos = 0;
174 - parser->user.st = st;
175 - parser->user.cleanup_slots = false;
176 -
177 - return true;
178 -}
179 -
180 -static inline void pluginsd_rrddim_put_to_slot(PARSER *parser, RRDSET *st, RRDDIM *rd, ssize_t slot, bool obsolete) {
181 - size_t wanted_size = st->pluginsd.size;
182 -
183 - if(slot >= 1) {
184 - st->pluginsd.dims_with_slots = true;
185 - wanted_size = slot;
186 - }
187 - else {
188 - st->pluginsd.dims_with_slots = false;
189 - wanted_size = dictionary_entries(st->rrddim_root_index);
190 - }
191 -
192 - if(wanted_size > st->pluginsd.size) {
193 - st->pluginsd.prd_array = reallocz(st->pluginsd.prd_array, wanted_size * sizeof(struct pluginsd_rrddim));
194 -
195 - // initialize the empty slots
196 - for(ssize_t i = (ssize_t) wanted_size - 1; i >= (ssize_t) st->pluginsd.size; i--) {
197 - st->pluginsd.prd_array[i].rda = NULL;
198 - st->pluginsd.prd_array[i].rd = NULL;
199 - st->pluginsd.prd_array[i].id = NULL;
200 - }
201 -
202 - st->pluginsd.size = wanted_size;
203 - }
204 -
205 - if(st->pluginsd.dims_with_slots) {
206 - struct pluginsd_rrddim *prd = &st->pluginsd.prd_array[slot - 1];
207 -
208 - if(prd->rd != rd) {
209 - prd->rda = rrddim_find_and_acquire(st, string2str(rd->id));
210 - prd->rd = rrddim_acquired_to_rrddim(prd->rda);
211 - prd->id = string2str(prd->rd->id);
212 - }
213 -
214 - if(obsolete)
215 - parser->user.cleanup_slots = true;
216 - }
217 -}
218 -
219 -static inline RRDDIM *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *st, const char *dimension, ssize_t slot, const char *cmd) {
220 - if (unlikely(!dimension || !*dimension)) {
221 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, without a dimension.",
222 - rrdhost_hostname(host), rrdset_id(st), cmd);
223 - return NULL;
224 - }
225 -
226 - if (unlikely(!st->pluginsd.size)) {
227 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, but the chart has no dimensions.",
228 - rrdhost_hostname(host), rrdset_id(st), cmd);
229 - return NULL;
230 - }
231 -
232 - struct pluginsd_rrddim *prd;
233 - RRDDIM *rd;
234 -
235 - if(likely(st->pluginsd.dims_with_slots)) {
236 - // caching with slots
237 -
238 - if(unlikely(slot < 1 || slot > st->pluginsd.size)) {
239 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s with slot %zd, but slots in the range [1 - %u] are expected.",
240 - rrdhost_hostname(host), rrdset_id(st), cmd, slot, st->pluginsd.size);
241 - return NULL;
242 - }
243 -
244 - prd = &st->pluginsd.prd_array[slot - 1];
245 -
246 - rd = prd->rd;
247 - if(likely(rd)) {
248 -#ifdef NETDATA_INTERNAL_CHECKS
249 - if(strcmp(prd->id, dimension) != 0) {
250 - ssize_t t;
251 - for(t = 0; t < st->pluginsd.size ;t++) {
252 - if (strcmp(st->pluginsd.prd_array[t].id, dimension) == 0)
253 - break;
254 - }
255 - if(t >= st->pluginsd.size)
256 - t = -1;
257 -
258 - internal_fatal(true,
259 - "PLUGINSD: expected to find dimension '%s' on slot %zd, but found '%s', "
260 - "the right slot is %zd",
261 - dimension, slot, prd->id, t);
262 - }
263 -#endif
264 - return rd;
265 - }
266 - }
267 - else {
268 - // caching without slots
269 -
270 - if(unlikely(st->pluginsd.pos >= st->pluginsd.size))
271 - st->pluginsd.pos = 0;
272 -
273 - prd = &st->pluginsd.prd_array[st->pluginsd.pos++];
274 -
275 - rd = prd->rd;
276 - if(likely(rd)) {
277 - const char *id = prd->id;
278 -
279 - if(strcmp(id, dimension) == 0) {
280 - // we found it cached
281 - return rd;
282 - }
283 - else {
284 - // the cached one is not good for us
285 - rrddim_acquired_release(prd->rda);
286 - prd->rda = NULL;
287 - prd->rd = NULL;
288 - prd->id = NULL;
289 - }
290 - }
291 - }
292 -
293 - // we need to find the dimension and set it to prd
294 -
295 - RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(st, dimension);
296 - if (unlikely(!rda)) {
297 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s but dimension does not exist.",
298 - rrdhost_hostname(host), rrdset_id(st), dimension, cmd);
299 -
300 - return NULL;
301 - }
302 -
303 - prd->rda = rda;
304 - prd->rd = rd = rrddim_acquired_to_rrddim(rda);
305 - prd->id = string2str(rd->id);
306 -
307 - return rd;
308 -}
309 -
310 -static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, const char *cmd) {
311 - if (unlikely(!chart || !*chart)) {
312 - netdata_log_error("PLUGINSD: 'host:%s' got a %s without a chart id.",
313 - rrdhost_hostname(host), cmd);
314 - return NULL;
315 - }
316 -
317 - RRDSET *st = rrdset_find(host, chart);
318 - if (unlikely(!st))
319 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s but chart does not exist.",
320 - rrdhost_hostname(host), chart, cmd);
321 -
322 - return st;
323 -}
324 -
325 -static inline ssize_t pluginsd_parse_rrd_slot(char **words, size_t num_words) {
326 - ssize_t slot = -1;
327 - char *id = get_word(words, num_words, 1);
328 - if(id && id[0] == PLUGINSD_KEYWORD_SLOT[0] && id[1] == PLUGINSD_KEYWORD_SLOT[1] &&
329 - id[2] == PLUGINSD_KEYWORD_SLOT[2] && id[3] == PLUGINSD_KEYWORD_SLOT[3] && id[4] == ':') {
330 - slot = (ssize_t) str2ull_encoded(&id[5]);
331 - if(slot < 0) slot = 0; // to make the caller increment its idx of the words
332 - }
333 -
334 - return slot;
335 -}
336 -
337 -static inline void pluginsd_rrdset_cache_put_to_slot(PARSER *parser, RRDSET *st, ssize_t slot, bool obsolete) {
338 - // clean possible old cached data
339 - rrdset_pluginsd_receive_unslot(st);
340 -
341 - if(unlikely(slot < 1 || slot >= INT32_MAX))
342 - return;
343 -
344 - RRDHOST *host = st->rrdhost;
345 -
346 - if(unlikely((size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size)) {
347 - spinlock_lock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock);
348 - size_t old_slots = host->rrdpush.receive.pluginsd_chart_slots.size;
349 - size_t new_slots = (old_slots < PLUGINSD_MIN_RRDSET_POINTERS_CACHE) ? PLUGINSD_MIN_RRDSET_POINTERS_CACHE : old_slots * 2;
350 -
351 - if(new_slots < (size_t)slot)
352 - new_slots = slot;
353 -
354 - host->rrdpush.receive.pluginsd_chart_slots.array =
355 - reallocz(host->rrdpush.receive.pluginsd_chart_slots.array, new_slots * sizeof(RRDSET *));
356 -
357 - for(size_t i = old_slots; i < new_slots ;i++)
358 - host->rrdpush.receive.pluginsd_chart_slots.array[i] = NULL;
359 -
360 - host->rrdpush.receive.pluginsd_chart_slots.size = new_slots;
361 - spinlock_unlock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock);
362 - }
363 -
364 - host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1] = st;
365 - st->pluginsd.last_slot = (int32_t)slot - 1;
366 - parser->user.cleanup_slots = obsolete;
367 -}
368 -
369 -static inline RRDSET *pluginsd_rrdset_cache_get_from_slot(PARSER *parser, RRDHOST *host, const char *id, ssize_t slot, const char *keyword) {
370 - if(unlikely(slot < 1 || (size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size))
371 - return pluginsd_find_chart(host, id, keyword);
372 -
373 - RRDSET *st = host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1];
374 -
375 - if(!st) {
376 - st = pluginsd_find_chart(host, id, keyword);
377 - if(st)
378 - pluginsd_rrdset_cache_put_to_slot(parser, st, slot, rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE));
379 - }
380 - else {
381 - internal_fatal(string_strcmp(st->id, id) != 0,
382 - "PLUGINSD: wrong chart in slot %zd, expected '%s', found '%s'",
383 - slot - 1, id, string2str(st->id));
384 - }
385 -
386 - return st;
387 -}
388 -
389 -static inline PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyword, const char *msg) {
390 - parser->user.enabled = 0;
391 -
392 - if(keyword && msg) {
393 - nd_log_limit_static_global_var(erl, 1, 0);
394 - nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_INFO,
395 - "PLUGINSD: keyword %s: %s", keyword, msg);
396 - }
397 -
398 - return PARSER_RC_ERROR;
399 -}
3 +#include "pluginsd_internals.h"
4
5 static inline PARSER_RC pluginsd_set(char **words, size_t num_words, PARSER *parser) {
6 int idx = 1;
@@ -900,384 +504,6 @@ static inline PARSER_RC pluginsd_dimension(char **words, size_t num_words, PARSE
504 return PARSER_RC_OK;
505 }
506
903 -// ----------------------------------------------------------------------------
904 -// execution of functions
905 -
906 -struct inflight_function {
907 - int code;
908 - int timeout;
909 - STRING *function;
910 - BUFFER *result_body_wb;
911 - rrd_function_result_callback_t result_cb;
912 - void *result_cb_data;
913 - usec_t timeout_ut;
914 - usec_t started_ut;
915 - usec_t sent_ut;
916 - const char *payload;
917 - PARSER *parser;
918 - bool virtual;
919 -};
920 -
921 -static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void *func, void *parser_ptr) {
922 - struct inflight_function *pf = func;
923 -
924 - PARSER *parser = parser_ptr;
925 -
926 - // leave this code as default, so that when the dictionary is destroyed this will be sent back to the caller
927 - pf->code = HTTP_RESP_GATEWAY_TIMEOUT;
928 -
929 - const char *transaction = dictionary_acquired_item_name(item);
930 -
931 - char buffer[2048 + 1];
932 - snprintfz(buffer, sizeof(buffer) - 1, "%s %s %d \"%s\"\n",
933 - pf->payload ? "FUNCTION_PAYLOAD" : "FUNCTION",
934 - transaction,
935 - pf->timeout,
936 - string2str(pf->function));
937 -
938 - // send the command to the plugin
939 - ssize_t ret = send_to_plugin(buffer, parser);
940 -
941 - pf->sent_ut = now_realtime_usec();
942 -
943 - if(ret < 0) {
944 - netdata_log_error("FUNCTION '%s': failed to send it to the plugin, error %zd", string2str(pf->function), ret);
945 - rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE);
946 - }
947 - else {
948 - internal_error(LOG_FUNCTIONS,
949 - "FUNCTION '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)",
950 - string2str(pf->function), dictionary_acquired_item_name(item), ret,
951 - pf->sent_ut - pf->started_ut);
952 - }
953 -
954 - if (!pf->payload)
955 - return;
956 -
957 - // send the payload to the plugin
958 - ret = send_to_plugin(pf->payload, parser);
959 -
960 - if(ret < 0) {
961 - netdata_log_error("FUNCTION_PAYLOAD '%s': failed to send function to plugin, error %zd", string2str(pf->function), ret);
962 - rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE);
963 - }
964 - else {
965 - internal_error(LOG_FUNCTIONS,
966 - "FUNCTION_PAYLOAD '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)",
967 - string2str(pf->function), dictionary_acquired_item_name(item), ret,
968 - pf->sent_ut - pf->started_ut);
969 - }
970 -
971 - send_to_plugin("\nFUNCTION_PAYLOAD_END\n", parser);
972 -}
973 -
974 -static bool inflight_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused, void *new_func, void *parser_ptr __maybe_unused) {
975 - struct inflight_function *pf = new_func;
976 -
977 - netdata_log_error("PLUGINSD_PARSER: duplicate UUID on pending function '%s' detected. Ignoring the second one.", string2str(pf->function));
978 - pf->code = rrd_call_function_error(pf->result_body_wb, "This request is already in progress", HTTP_RESP_BAD_REQUEST);
979 - pf->result_cb(pf->result_body_wb, pf->code, pf->result_cb_data);
980 - string_freez(pf->function);
981 -
982 - return false;
983 -}
984 -
985 -void delete_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug, const char *fnc_sig, int code) {
986 - if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED)
987 - return;
988 -
989 - char *params_local = strdupz(fnc_sig);
990 - char *words[DYNCFG_MAX_WORDS];
991 - size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd);
992 -
993 - if (words_c != 3) {
994 - netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for delete_job");
995 - freez(params_local);
996 - return;
997 - }
998 -
999 - const char *module = words[1];
1000 - const char *job = words[2];
1001 -
1002 - delete_job(plug, module, job);
1003 -
1004 - unlink_job(plug->name, module, job);
1005 -
1006 - rrdpush_send_job_deleted(localhost, plug->name, module, job);
1007 -
1008 - freez(params_local);
1009 -}
1010 -
1011 -void set_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug __maybe_unused, const char *fnc_sig, int code) {
1012 - if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED)
1013 - return;
1014 -
1015 - char *params_local = strdupz(fnc_sig);
1016 - char *words[DYNCFG_MAX_WORDS];
1017 - size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd);
1018 -
1019 - if (words_c != 3) {
1020 - netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for set_job_config");
1021 - freez(params_local);
1022 - return;
1023 - }
1024 -
1025 - const char *module_name = get_word(words, words_c, 1);
1026 - const char *job_name = get_word(words, words_c, 2);
1027 -
1028 - if (register_job(parser->user.host->configurable_plugins, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED, 1)) {
1029 - freez(params_local);
1030 - return;
1031 - }
1032 -
1033 - // only send this if it is not existing already (register_job cares for that)
1034 - rrdpush_send_dyncfg_reg_job(localhost, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED);
1035 -
1036 - freez(params_local);
1037 -}
1038 -
1039 -static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *parser_ptr) {
1040 - struct inflight_function *pf = func;
1041 - struct parser *parser = (struct parser *)parser_ptr;
1042 -
1043 - internal_error(LOG_FUNCTIONS,
1044 - "FUNCTION '%s' result of transaction '%s' received from collector (%zu bytes, request %"PRIu64" usec, response %"PRIu64" usec)",
1045 - string2str(pf->function), dictionary_acquired_item_name(item),
1046 - buffer_strlen(pf->result_body_wb), pf->sent_ut - pf->started_ut, now_realtime_usec() - pf->sent_ut);
1047 -
1048 - if (pf->virtual && SERVING_PLUGINSD(parser)) {
1049 - if (pf->payload) {
1050 - if (strncmp(string2str(pf->function), FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0)
1051 - set_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code);
1052 - dyn_conf_store_config(string2str(pf->function), pf->payload, parser->user.cd->configuration);
1053 - } else if (strncmp(string2str(pf->function), FUNCTION_NAME_DELETE_JOB, strlen(FUNCTION_NAME_DELETE_JOB)) == 0) {
1054 - delete_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code);
1055 - }
1056 - }
1057 -
1058 - pf->result_cb(pf->result_body_wb, pf->code, pf->result_cb_data);
1059 -
1060 - string_freez(pf->function);
1061 - freez((void *)pf->payload);
1062 -}
1063 -
1064 -void inflight_functions_init(PARSER *parser) {
1065 - parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions, 0);
1066 - dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser);
1067 - dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser);
1068 - dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser);
1069 -}
1070 -
1071 -static void inflight_functions_garbage_collect(PARSER *parser, usec_t now) {
1072 - parser->inflight.smaller_timeout = 0;
1073 - struct inflight_function *pf;
1074 - dfe_start_write(parser->inflight.functions, pf) {
1075 - if (pf->timeout_ut < now) {
1076 - internal_error(true,
1077 - "FUNCTION '%s' removing expired transaction '%s', after %"PRIu64" usec.",
1078 - string2str(pf->function), pf_dfe.name, now - pf->started_ut);
1079 -
1080 - if(!buffer_strlen(pf->result_body_wb) || pf->code == HTTP_RESP_OK)
1081 - pf->code = rrd_call_function_error(pf->result_body_wb,
1082 - "Timeout waiting for collector response.",
1083 - HTTP_RESP_GATEWAY_TIMEOUT);
1084 -
1085 - dictionary_del(parser->inflight.functions, pf_dfe.name);
1086 - }
1087 -
1088 - else if(!parser->inflight.smaller_timeout || pf->timeout_ut < parser->inflight.smaller_timeout)
1089 - parser->inflight.smaller_timeout = pf->timeout_ut;
1090 - }
1091 - dfe_done(pf);
1092 -}
1093 -
1094 -void pluginsd_function_cancel(void *data) {
1095 - struct inflight_function *look_for = data, *t;
1096 -
1097 - bool sent = false;
1098 - dfe_start_read(look_for->parser->inflight.functions, t) {
1099 - if(look_for == t) {
1100 - const char *transaction = t_dfe.name;
1101 -
1102 - internal_error(true, "PLUGINSD: sending function cancellation to plugin for transaction '%s'", transaction);
1103 -
1104 - char buffer[2048 + 1];
1105 - snprintfz(buffer, sizeof(buffer) - 1, "%s %s\n",
1106 - PLUGINSD_KEYWORD_FUNCTION_CANCEL,
1107 - transaction);
1108 -
1109 - // send the command to the plugin
1110 - ssize_t ret = send_to_plugin(buffer, t->parser);
1111 - if(ret < 0)
1112 - sent = true;
1113 -
1114 - break;
1115 - }
1116 - }
1117 - dfe_done(t);
1118 -
1119 - if(sent <= 0)
1120 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
1121 - "PLUGINSD: FUNCTION_CANCEL request didn't match any pending function requests in pluginsd.d.");
1122 -}
1123 -
1124 -// this is the function that is called from
1125 -// rrd_call_function_and_wait() and rrd_call_function_async()
1126 -static int pluginsd_function_execute_cb(BUFFER *result_body_wb, int timeout, const char *function,
1127 - void *execute_cb_data,
1128 - rrd_function_result_callback_t result_cb, void *result_cb_data,
1129 - rrd_function_is_cancelled_cb_t is_cancelled_cb __maybe_unused,
1130 - void *is_cancelled_cb_data __maybe_unused,
1131 - rrd_function_register_canceller_cb_t register_canceller_cb,
1132 - void *register_canceller_db_data) {
1133 - PARSER *parser = execute_cb_data;
1134 -
1135 - usec_t now = now_realtime_usec();
1136 -
1137 - struct inflight_function tmp = {
1138 - .started_ut = now,
1139 - .timeout_ut = now + timeout * USEC_PER_SEC + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT,
1140 - .result_body_wb = result_body_wb,
1141 - .timeout = timeout,
1142 - .function = string_strdupz(function),
1143 - .result_cb = result_cb,
1144 - .result_cb_data = result_cb_data,
1145 - .payload = NULL,
1146 - .parser = parser,
1147 - };
1148 -
1149 - uuid_t uuid;
1150 - uuid_generate_random(uuid);
1151 -
1152 - char transaction[UUID_STR_LEN];
1153 - uuid_unparse_lower(uuid, transaction);
1154 -
1155 - dictionary_write_lock(parser->inflight.functions);
1156 -
1157 - // if there is any error, our dictionary callbacks will call the caller callback to notify
1158 - // the caller about the error - no need for error handling here.
1159 - void *t = dictionary_set(parser->inflight.functions, transaction, &tmp, sizeof(struct inflight_function));
1160 - if(register_canceller_cb)
1161 - register_canceller_cb(register_canceller_db_data, pluginsd_function_cancel, t);
1162 -
1163 - if(!parser->inflight.smaller_timeout || tmp.timeout_ut < parser->inflight.smaller_timeout)
1164 - parser->inflight.smaller_timeout = tmp.timeout_ut;
1165 -
1166 - // garbage collect stale inflight functions
1167 - if(parser->inflight.smaller_timeout < now)
1168 - inflight_functions_garbage_collect(parser, now);
1169 -
1170 - dictionary_write_unlock(parser->inflight.functions);
1171 -
1172 - return HTTP_RESP_OK;
1173 -}
1174 -
1175 -static inline PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser) {
1176 - // a plugin or a child is registering a function
1177 -
1178 - bool global = false;
1179 - size_t i = 1;
1180 - if(num_words >= 2 && strcmp(get_word(words, num_words, 1), "GLOBAL") == 0) {
1181 - i++;
1182 - global = true;
1183 - }
1184 -
1185 - char *name = get_word(words, num_words, i++);
1186 - char *timeout_s = get_word(words, num_words, i++);
1187 - char *help = get_word(words, num_words, i++);
1188 -
1189 - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_FUNCTION);
1190 - if(!host) return PARSER_RC_ERROR;
1191 -
1192 - RRDSET *st = (global)? NULL: pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_FUNCTION, PLUGINSD_KEYWORD_CHART);
1193 - if(!st) global = true;
1194 -
1195 - if (unlikely(!timeout_s || !name || !help || (!global && !st))) {
1196 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a FUNCTION, without providing the required data (global = '%s', name = '%s', timeout = '%s', help = '%s'). Ignoring it.",
1197 - rrdhost_hostname(host),
1198 - st?rrdset_id(st):"(unset)",
1199 - global?"yes":"no",
1200 - name?name:"(unset)",
1201 - timeout_s?timeout_s:"(unset)",
1202 - help?help:"(unset)"
1203 - );
1204 - return PARSER_RC_ERROR;
1205 - }
1206 -
1207 - int timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
1208 - if (timeout_s && *timeout_s) {
1209 - timeout = str2i(timeout_s);
1210 - if (unlikely(timeout <= 0))
1211 - timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT;
1212 - }
1213 -
1214 - rrd_function_add(host, st, name, timeout, help, false, pluginsd_function_execute_cb, parser);
1215 -
1216 - parser->user.data_collections_count++;
1217 -
1218 - return PARSER_RC_OK;
1219 -}
1220 -
1221 -static void pluginsd_function_result_end(struct parser *parser, void *action_data) {
1222 - STRING *key = action_data;
1223 - if(key)
1224 - dictionary_del(parser->inflight.functions, string2str(key));
1225 - string_freez(key);
1226 -
1227 - parser->user.data_collections_count++;
1228 -}
1229 -
1230 -static inline PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, PARSER *parser) {
1231 - char *key = get_word(words, num_words, 1);
1232 - char *status = get_word(words, num_words, 2);
1233 - char *format = get_word(words, num_words, 3);
1234 - char *expires = get_word(words, num_words, 4);
1235 -
1236 - if (unlikely(!key || !*key || !status || !*status || !format || !*format || !expires || !*expires)) {
1237 - netdata_log_error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " without providing the required data (key = '%s', status = '%s', format = '%s', expires = '%s')."
1238 - , key ? key : "(unset)"
1239 - , status ? status : "(unset)"
1240 - , format ? format : "(unset)"
1241 - , expires ? expires : "(unset)"
1242 - );
1243 - }
1244 -
1245 - int code = (status && *status) ? str2i(status) : 0;
1246 - if (code <= 0)
1247 - code = HTTP_RESP_BACKEND_RESPONSE_INVALID;
1248 -
1249 - time_t expiration = (expires && *expires) ? str2l(expires) : 0;
1250 -
1251 - struct inflight_function *pf = NULL;
1252 -
1253 - if(key && *key)
1254 - pf = (struct inflight_function *)dictionary_get(parser->inflight.functions, key);
1255 -
1256 - if(!pf) {
1257 - netdata_log_error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " for transaction '%s', but the transaction is not found.", key?key:"(unset)");
1258 - }
1259 - else {
1260 - if(format && *format)
1261 - pf->result_body_wb->content_type = functions_format_to_content_type(format);
1262 -
1263 - pf->code = code;
1264 -
1265 - pf->result_body_wb->expires = expiration;
1266 - if(expiration <= now_realtime_sec())
1267 - buffer_no_cacheable(pf->result_body_wb);
1268 - else
1269 - buffer_cacheable(pf->result_body_wb);
1270 - }
1271 -
1272 - parser->defer.response = (pf) ? pf->result_body_wb : NULL;
1273 - parser->defer.end_keyword = PLUGINSD_KEYWORD_FUNCTION_RESULT_END;
1274 - parser->defer.action = pluginsd_function_result_end;
1275 - parser->defer.action_data = string_strdupz(key); // it is ok is key is NULL
1276 - parser->flags |= PARSER_DEFER_UNTIL_KEYWORD;
1277 -
1278 - return PARSER_RC_OK;
1279 -}
1280 -
507 // ----------------------------------------------------------------------------
508
509 static inline PARSER_RC pluginsd_variable(char **words, size_t num_words, PARSER *parser) {
@@ -1472,426 +698,31 @@ static inline PARSER_RC pluginsd_clabel(char **words, size_t num_words, PARSER *
698
699 rrdlabels_add(parser->user.chart_rrdlabels_linked_temporarily, name, value, str2l(label_source));
700
1475 - return PARSER_RC_OK;
1476 -}
1477 -
1478 -static inline PARSER_RC pluginsd_clabel_commit(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
1479 - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT);
1480 - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1481 -
1482 - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT, PLUGINSD_KEYWORD_BEGIN);
1483 - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1484 -
1485 - netdata_log_debug(D_PLUGINSD, "requested to commit chart labels");
1486 -
1487 - if(!parser->user.chart_rrdlabels_linked_temporarily) {
1488 - netdata_log_error("PLUGINSD: 'host:%s' got CLABEL_COMMIT, without a CHART or BEGIN. Ignoring it.", rrdhost_hostname(host));
1489 - return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1490 - }
1491 -
1492 - rrdlabels_remove_all_unmarked(parser->user.chart_rrdlabels_linked_temporarily);
1493 -
1494 - rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
1495 - rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
1496 - rrdset_metadata_updated(st);
1497 -
1498 - parser->user.chart_rrdlabels_linked_temporarily = NULL;
1499 - return PARSER_RC_OK;
1500 -}
1501 -
1502 -static inline PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PARSER *parser) {
1503 - int idx = 1;
1504 - ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
1505 - if(slot >= 0) idx++;
1506 -
1507 - char *id = get_word(words, num_words, idx++);
1508 - char *start_time_str = get_word(words, num_words, idx++);
1509 - char *end_time_str = get_word(words, num_words, idx++);
1510 - char *child_now_str = get_word(words, num_words, idx++);
1511 -
1512 - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1513 - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1514 -
1515 - RRDSET *st;
1516 - if (likely(!id || !*id))
1517 - st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1518 - else
1519 - st = pluginsd_rrdset_cache_get_from_slot(parser, host, id, slot, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1520 -
1521 - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1522 -
1523 - if(!pluginsd_set_scope_chart(parser, st, PLUGINSD_KEYWORD_REPLAY_BEGIN))
1524 - return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1525 -
1526 - if(start_time_str && end_time_str) {
1527 - time_t start_time = (time_t) str2ull_encoded(start_time_str);
1528 - time_t end_time = (time_t) str2ull_encoded(end_time_str);
1529 -
1530 - time_t wall_clock_time = 0, tolerance;
1531 - bool wall_clock_comes_from_child; (void)wall_clock_comes_from_child;
1532 - if(child_now_str) {
1533 - wall_clock_time = (time_t) str2ull_encoded(child_now_str);
1534 - tolerance = st->update_every + 1;
1535 - wall_clock_comes_from_child = true;
1536 - }
1537 -
1538 - if(wall_clock_time <= 0) {
1539 - wall_clock_time = now_realtime_sec();
1540 - tolerance = st->update_every + 5;
1541 - wall_clock_comes_from_child = false;
1542 - }
1543 -
1544 -#ifdef NETDATA_LOG_REPLICATION_REQUESTS
1545 - internal_error(
1546 - (!st->replay.start_streaming && (end_time < st->replay.after || start_time > st->replay.before)),
1547 - "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, which does not match our request (%ld to %ld).",
1548 - rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time, st->replay.after, st->replay.before);
1549 -
1550 - internal_error(
1551 - true,
1552 - "REPLAY: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, child wall clock is %ld (%s), had requested %ld to %ld",
1553 - rrdhost_hostname(st->rrdhost), rrdset_id(st),
1554 - start_time, end_time, wall_clock_time, wall_clock_comes_from_child ? "from child" : "parent time",
1555 - st->replay.after, st->replay.before);
1556 -#endif
1557 -
1558 - if(start_time && end_time && start_time < wall_clock_time + tolerance && end_time < wall_clock_time + tolerance && start_time < end_time) {
1559 - if (unlikely(end_time - start_time != st->update_every))
1560 - rrdset_set_update_every_s(st, end_time - start_time);
1561 -
1562 - st->last_collected_time.tv_sec = end_time;
1563 - st->last_collected_time.tv_usec = 0;
1564 -
1565 - st->last_updated.tv_sec = end_time;
1566 - st->last_updated.tv_usec = 0;
1567 -
1568 - st->counter++;
1569 - st->counter_done++;
1570 -
1571 - // these are only needed for db mode RAM, SAVE, MAP, ALLOC
1572 - st->db.current_entry++;
1573 - if(st->db.current_entry >= st->db.entries)
1574 - st->db.current_entry -= st->db.entries;
1575 -
1576 - parser->user.replay.start_time = start_time;
1577 - parser->user.replay.end_time = end_time;
1578 - parser->user.replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC;
1579 - parser->user.replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC;
1580 - parser->user.replay.wall_clock_time = wall_clock_time;
1581 - parser->user.replay.rset_enabled = true;
1582 -
1583 - return PARSER_RC_OK;
1584 - }
1585 -
1586 - netdata_log_error("PLUGINSD REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN
1587 - " from %ld to %ld, but timestamps are invalid "
1588 - "(now is %ld [%s], tolerance %ld). Ignoring " PLUGINSD_KEYWORD_REPLAY_SET,
1589 - rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time,
1590 - wall_clock_time, wall_clock_comes_from_child ? "child wall clock" : "parent wall clock",
1591 - tolerance);
1592 - }
1593 -
1594 - // the child sends an RBEGIN without any parameters initially
1595 - // setting rset_enabled to false, means the RSET should not store any metrics
1596 - // to store metrics, the RBEGIN needs to have timestamps
1597 - parser->user.replay.start_time = 0;
1598 - parser->user.replay.end_time = 0;
1599 - parser->user.replay.start_time_ut = 0;
1600 - parser->user.replay.end_time_ut = 0;
1601 - parser->user.replay.wall_clock_time = 0;
1602 - parser->user.replay.rset_enabled = false;
1603 - return PARSER_RC_OK;
1604 -}
1605 -
1606 -static inline SN_FLAGS pluginsd_parse_storage_number_flags(const char *flags_str) {
1607 - SN_FLAGS flags = SN_FLAG_NONE;
1608 -
1609 - char c;
1610 - while ((c = *flags_str++)) {
1611 - switch (c) {
1612 - case 'A':
1613 - flags |= SN_FLAG_NOT_ANOMALOUS;
1614 - break;
1615 -
1616 - case 'R':
1617 - flags |= SN_FLAG_RESET;
1618 - break;
1619 -
1620 - case 'E':
1621 - flags = SN_EMPTY_SLOT;
1622 - return flags;
1623 -
1624 - default:
1625 - internal_error(true, "Unknown SN_FLAGS flag '%c'", c);
1626 - break;
1627 - }
1628 - }
1629 -
1630 - return flags;
1631 -}
1632 -
1633 -static inline PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARSER *parser) {
1634 - int idx = 1;
1635 - ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
1636 - if(slot >= 0) idx++;
1637 -
1638 - char *dimension = get_word(words, num_words, idx++);
1639 - char *value_str = get_word(words, num_words, idx++);
1640 - char *flags_str = get_word(words, num_words, idx++);
1641 -
1642 - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_SET);
1643 - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1644 -
1645 - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1646 - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1647 -
1648 - if(!parser->user.replay.rset_enabled) {
1649 - nd_log_limit_static_thread_var(erl, 1, 0);
1650 - nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_ERR,
1651 - "PLUGINSD: 'host:%s/chart:%s' got a %s but it is disabled by %s errors",
1652 - rrdhost_hostname(host), rrdset_id(st), PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1653 -
1654 - // we have to return OK here
1655 - return PARSER_RC_OK;
1656 - }
1657 -
1658 - RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_SET);
1659 - if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1660 -
1661 - st->pluginsd.set = true;
1662 -
1663 - if (unlikely(!parser->user.replay.start_time || !parser->user.replay.end_time)) {
1664 - netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s with invalid timestamps %ld to %ld from a %s. Disabling it.",
1665 - rrdhost_hostname(host),
1666 - rrdset_id(st),
1667 - dimension,
1668 - PLUGINSD_KEYWORD_REPLAY_SET,
1669 - parser->user.replay.start_time,
1670 - parser->user.replay.end_time,
1671 - PLUGINSD_KEYWORD_REPLAY_BEGIN);
1672 - return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1673 - }
1674 -
1675 - if (unlikely(!value_str || !*value_str))
1676 - value_str = "NAN";
1677 -
1678 - if(unlikely(!flags_str))
1679 - flags_str = "";
1680 -
1681 - if (likely(value_str)) {
1682 - RRDDIM_FLAGS rd_flags = rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED);
1683 -
1684 - if(!(rd_flags & RRDDIM_FLAG_ARCHIVED)) {
1685 - NETDATA_DOUBLE value = str2ndd_encoded(value_str, NULL);
1686 - SN_FLAGS flags = pluginsd_parse_storage_number_flags(flags_str);
1687 -
1688 - if (!netdata_double_isnumber(value) || (flags == SN_EMPTY_SLOT)) {
1689 - value = NAN;
1690 - flags = SN_EMPTY_SLOT;
1691 - }
1692 -
1693 - rrddim_store_metric(rd, parser->user.replay.end_time_ut, value, flags);
1694 - rd->collector.last_collected_time.tv_sec = parser->user.replay.end_time;
1695 - rd->collector.last_collected_time.tv_usec = 0;
1696 - rd->collector.counter++;
1697 - }
1698 - else {
1699 - nd_log_limit_static_global_var(erl, 1, 0);
1700 - nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_WARNING,
1701 - "PLUGINSD: 'host:%s/chart:%s/dim:%s' has the ARCHIVED flag set, but it is replicated. "
1702 - "Ignoring data.",
1703 - rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_name(rd));
1704 - }
1705 - }
1706 -
1707 - return PARSER_RC_OK;
1708 -}
1709 -
1710 -static inline PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, PARSER *parser) {
1711 - if(parser->user.replay.rset_enabled == false)
1712 - return PARSER_RC_OK;
1713 -
1714 - int idx = 1;
1715 - ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
1716 - if(slot >= 0) idx++;
1717 -
1718 - char *dimension = get_word(words, num_words, idx++);
1719 - char *last_collected_ut_str = get_word(words, num_words, idx++);
1720 - char *last_collected_value_str = get_word(words, num_words, idx++);
1721 - char *last_calculated_value_str = get_word(words, num_words, idx++);
1722 - char *last_stored_value_str = get_word(words, num_words, idx++);
1723 -
1724 - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
1725 - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1726 -
1727 - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1728 - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1729 -
1730 - if(st->pluginsd.set) {
1731 - // reset pos to reuse the same RDAs
1732 - st->pluginsd.pos = 0;
1733 - st->pluginsd.set = false;
1734 - }
1735 -
1736 - RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
1737 - if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1738 -
1739 - usec_t dim_last_collected_ut = (usec_t)rd->collector.last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->collector.last_collected_time.tv_usec;
1740 - usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0;
1741 - if(last_collected_ut > dim_last_collected_ut) {
1742 - rd->collector.last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC);
1743 - rd->collector.last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC);
1744 - }
1745 -
1746 - rd->collector.last_collected_value = last_collected_value_str ? str2ll_encoded(last_collected_value_str) : 0;
1747 - rd->collector.last_calculated_value = last_calculated_value_str ? str2ndd_encoded(last_calculated_value_str, NULL) : 0;
1748 - rd->collector.last_stored_value = last_stored_value_str ? str2ndd_encoded(last_stored_value_str, NULL) : 0.0;
1749 -
1750 - return PARSER_RC_OK;
1751 -}
1752 -
1753 -static inline PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, PARSER *parser) {
1754 - if(parser->user.replay.rset_enabled == false)
1755 - return PARSER_RC_OK;
1756 -
1757 - char *last_collected_ut_str = get_word(words, num_words, 1);
1758 - char *last_updated_ut_str = get_word(words, num_words, 2);
1759 -
1760 - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE);
1761 - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1762 -
1763 - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE,
1764 - PLUGINSD_KEYWORD_REPLAY_BEGIN);
1765 - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1766 -
1767 - usec_t chart_last_collected_ut = (usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec;
1768 - usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0;
1769 - if(last_collected_ut > chart_last_collected_ut) {
1770 - st->last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC);
1771 - st->last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC);
1772 - }
1773 -
1774 - usec_t chart_last_updated_ut = (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec;
1775 - usec_t last_updated_ut = last_updated_ut_str ? str2ull_encoded(last_updated_ut_str) : 0;
1776 - if(last_updated_ut > chart_last_updated_ut) {
1777 - st->last_updated.tv_sec = (time_t)(last_updated_ut / USEC_PER_SEC);
1778 - st->last_updated.tv_usec = (last_updated_ut % USEC_PER_SEC);
1779 - }
1780 -
1781 - st->counter++;
1782 - st->counter_done++;
1783 -
1784 - return PARSER_RC_OK;
1785 -}
1786 -
1787 -static inline PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARSER *parser) {
1788 - if (num_words < 7) { // accepts 7, but the 7th is optional
1789 - netdata_log_error("REPLAY: malformed " PLUGINSD_KEYWORD_REPLAY_END " command");
1790 - return PARSER_RC_ERROR;
1791 - }
1792 -
1793 - const char *update_every_child_txt = get_word(words, num_words, 1);
1794 - const char *first_entry_child_txt = get_word(words, num_words, 2);
1795 - const char *last_entry_child_txt = get_word(words, num_words, 3);
1796 - const char *start_streaming_txt = get_word(words, num_words, 4);
1797 - const char *first_entry_requested_txt = get_word(words, num_words, 5);
1798 - const char *last_entry_requested_txt = get_word(words, num_words, 6);
1799 - const char *child_world_time_txt = get_word(words, num_words, 7); // optional
1800 -
1801 - time_t update_every_child = (time_t) str2ull_encoded(update_every_child_txt);
1802 - time_t first_entry_child = (time_t) str2ull_encoded(first_entry_child_txt);
1803 - time_t last_entry_child = (time_t) str2ull_encoded(last_entry_child_txt);
1804 -
1805 - bool start_streaming = (strcmp(start_streaming_txt, "true") == 0);
1806 - time_t first_entry_requested = (time_t) str2ull_encoded(first_entry_requested_txt);
1807 - time_t last_entry_requested = (time_t) str2ull_encoded(last_entry_requested_txt);
1808 -
1809 - // the optional child world time
1810 - time_t child_world_time = (child_world_time_txt && *child_world_time_txt) ? (time_t) str2ull_encoded(
1811 - child_world_time_txt) : now_realtime_sec();
1812 -
1813 - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_END);
1814 - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1815 -
1816 - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END, PLUGINSD_KEYWORD_REPLAY_BEGIN);
1817 - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
1818 -
1819 -#ifdef NETDATA_LOG_REPLICATION_REQUESTS
1820 - internal_error(true,
1821 - "PLUGINSD REPLAY: 'host:%s/chart:%s': got a " PLUGINSD_KEYWORD_REPLAY_END " child db from %llu to %llu, start_streaming %s, had requested from %llu to %llu, wall clock %llu",
1822 - rrdhost_hostname(host), rrdset_id(st),
1823 - (unsigned long long)first_entry_child, (unsigned long long)last_entry_child,
1824 - start_streaming?"true":"false",
1825 - (unsigned long long)first_entry_requested, (unsigned long long)last_entry_requested,
1826 - (unsigned long long)child_world_time
1827 - );
1828 -#endif
1829 -
1830 - parser->user.data_collections_count++;
1831 -
1832 - if(parser->user.replay.rset_enabled && st->rrdhost->receiver) {
1833 - time_t now = now_realtime_sec();
1834 - time_t started = st->rrdhost->receiver->replication_first_time_t;
1835 - time_t current = parser->user.replay.end_time;
1836 -
1837 - if(started && current > started) {
1838 - host->rrdpush_receiver_replication_percent = (NETDATA_DOUBLE) (current - started) * 100.0 / (NETDATA_DOUBLE) (now - started);
1839 - worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION,
1840 - host->rrdpush_receiver_replication_percent);
1841 - }
1842 - }
1843 -
1844 - parser->user.replay.start_time = 0;
1845 - parser->user.replay.end_time = 0;
1846 - parser->user.replay.start_time_ut = 0;
1847 - parser->user.replay.end_time_ut = 0;
1848 - parser->user.replay.wall_clock_time = 0;
1849 - parser->user.replay.rset_enabled = false;
1850 -
1851 - st->counter++;
1852 - st->counter_done++;
1853 - store_metric_collection_completed();
1854 -
1855 -#ifdef NETDATA_LOG_REPLICATION_REQUESTS
1856 - st->replay.start_streaming = false;
1857 - st->replay.after = 0;
1858 - st->replay.before = 0;
1859 - if(start_streaming)
1860 - st->replay.log_next_data_collection = true;
1861 -#endif
1862 -
1863 - if (start_streaming) {
1864 - if (st->update_every != update_every_child)
1865 - rrdset_set_update_every_s(st, update_every_child);
701 + return PARSER_RC_OK;
702 +}
703
1867 - if(rrdset_flag_check(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS)) {
1868 - rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
1869 - rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS);
1870 - rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK);
1871 - rrdhost_receiver_replicating_charts_minus_one(st->rrdhost);
1872 - }
1873 -#ifdef NETDATA_LOG_REPLICATION_REQUESTS
1874 - else
1875 - internal_error(true, "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_END " with enable_streaming = true, but there is no replication in progress for this chart.",
1876 - rrdhost_hostname(host), rrdset_id(st));
1877 -#endif
704 +static inline PARSER_RC pluginsd_clabel_commit(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
705 + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT);
706 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
707
1879 - pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END);
708 + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT, PLUGINSD_KEYWORD_BEGIN);
709 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
710
1881 - host->rrdpush_receiver_replication_percent = 100.0;
1882 - worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, host->rrdpush_receiver_replication_percent);
711 + netdata_log_debug(D_PLUGINSD, "requested to commit chart labels");
712
1884 - return PARSER_RC_OK;
713 + if(!parser->user.chart_rrdlabels_linked_temporarily) {
714 + netdata_log_error("PLUGINSD: 'host:%s' got CLABEL_COMMIT, without a CHART or BEGIN. Ignoring it.", rrdhost_hostname(host));
715 + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
716 }
717
1887 - pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END);
718 + rrdlabels_remove_all_unmarked(parser->user.chart_rrdlabels_linked_temporarily);
719
1889 - rrdcontext_updated_retention_rrdset(st);
720 + rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
721 + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE);
722 + rrdset_metadata_updated(st);
723
1891 - bool ok = replicate_chart_request(send_to_plugin, parser, host, st,
1892 - first_entry_child, last_entry_child, child_world_time,
1893 - first_entry_requested, last_entry_requested);
1894 - return ok ? PARSER_RC_OK : PARSER_RC_ERROR;
724 + parser->user.chart_rrdlabels_linked_temporarily = NULL;
725 + return PARSER_RC_OK;
726 }
727
728 static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER *parser) {
@@ -2160,11 +991,6 @@ static inline PARSER_RC pluginsd_set_v2(char **words, size_t num_words, PARSER *
991 return PARSER_RC_OK;
992 }
993
2163 -void pluginsd_cleanup_v2(PARSER *parser) {
2164 - // this is called when the thread is stopped while processing
2165 - pluginsd_clear_scope_chart(parser, "THREAD CLEANUP");
2166 -}
2167 -
994 static inline PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) {
995 timing_init();
996
@@ -2243,569 +1069,6 @@ static inline PARSER_RC pluginsd_exit(char **words __maybe_unused, size_t num_wo
1069 return PARSER_RC_STOP;
1070 }
1071
2246 -struct mutex_cond {
2247 - pthread_mutex_t lock;
2248 - pthread_cond_t cond;
2249 - int rc;
2250 -};
2251 -
2252 -static void virt_fnc_got_data_cb(BUFFER *wb __maybe_unused, int code, void *callback_data)
2253 -{
2254 - struct mutex_cond *ctx = callback_data;
2255 - pthread_mutex_lock(&ctx->lock);
2256 - ctx->rc = code;
2257 - pthread_cond_broadcast(&ctx->cond);
2258 - pthread_mutex_unlock(&ctx->lock);
2259 -}
2260 -
2261 -#define VIRT_FNC_TIMEOUT 1
2262 -#define VIRT_FNC_BUF_SIZE (4096)
2263 -void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data) {
2264 - PARSER *parser = NULL;
2265 -
2266 - //TODO simplify (as we really need only first parameter to get plugin name maybe we can avoid parsing all)
2267 - char *words[PLUGINSD_MAX_WORDS];
2268 - char *function_with_params = strdupz(name);
2269 - size_t num_words = quoted_strings_splitter(function_with_params, words, PLUGINSD_MAX_WORDS, isspace_map_pluginsd);
2270 -
2271 - if (num_words < 2) {
2272 - netdata_log_error("PLUGINSD: virtual function name is empty.");
2273 - freez(function_with_params);
2274 - return;
2275 - }
2276 -
2277 - const DICTIONARY_ITEM *cpi = dictionary_get_and_acquire_item(host->configurable_plugins, get_word(words, num_words, 1));
2278 - if (unlikely(cpi == NULL)) {
2279 - netdata_log_error("PLUGINSD: virtual function plugin '%s' not found.", name);
2280 - freez(function_with_params);
2281 - return;
2282 - }
2283 - struct configurable_plugin *cp = dictionary_acquired_item_value(cpi);
2284 - parser = (PARSER *)cp->cb_usr_ctx;
2285 -
2286 - BUFFER *function_out = buffer_create(VIRT_FNC_BUF_SIZE, NULL);
2287 - // if we are forwarding this to a plugin (as opposed to streaming/child) we have to remove the first parameter (plugin_name)
2288 - buffer_strcat(function_out, get_word(words, num_words, 0));
2289 - for (size_t i = 1; i < num_words; i++) {
2290 - if (i == 1 && SERVING_PLUGINSD(parser))
2291 - continue;
2292 - buffer_sprintf(function_out, " %s", get_word(words, num_words, i));
2293 - }
2294 - freez(function_with_params);
2295 -
2296 - usec_t now = now_realtime_usec();
2297 -
2298 - struct inflight_function tmp = {
2299 - .started_ut = now,
2300 - .timeout_ut = now + VIRT_FNC_TIMEOUT + USEC_PER_SEC,
2301 - .result_body_wb = wb,
2302 - .timeout = VIRT_FNC_TIMEOUT * 10,
2303 - .function = string_strdupz(buffer_tostring(function_out)),
2304 - .result_cb = callback,
2305 - .result_cb_data = callback_data,
2306 - .payload = payload != NULL ? strdupz(payload) : NULL,
2307 - .virtual = true,
2308 - };
2309 - buffer_free(function_out);
2310 -
2311 - uuid_t uuid;
2312 - uuid_generate_time(uuid);
2313 -
2314 - char key[UUID_STR_LEN];
2315 - uuid_unparse_lower(uuid, key);
2316 -
2317 - dictionary_write_lock(parser->inflight.functions);
2318 -
2319 - // if there is any error, our dictionary callbacks will call the caller callback to notify
2320 - // the caller about the error - no need for error handling here.
2321 - dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function));
2322 -
2323 - if(!parser->inflight.smaller_timeout || tmp.timeout_ut < parser->inflight.smaller_timeout)
2324 - parser->inflight.smaller_timeout = tmp.timeout_ut;
2325 -
2326 - // garbage collect stale inflight functions
2327 - if(parser->inflight.smaller_timeout < now)
2328 - inflight_functions_garbage_collect(parser, now);
2329 -
2330 - dictionary_write_unlock(parser->inflight.functions);
2331 -}
2332 -
2333 -
2334 -dyncfg_config_t call_virtual_function_blocking(PARSER *parser, const char *name, int *rc, const char *payload) {
2335 - usec_t now = now_realtime_usec();
2336 - BUFFER *wb = buffer_create(VIRT_FNC_BUF_SIZE, NULL);
2337 -
2338 - struct mutex_cond cond = {
2339 - .lock = PTHREAD_MUTEX_INITIALIZER,
2340 - .cond = PTHREAD_COND_INITIALIZER
2341 - };
2342 -
2343 - struct inflight_function tmp = {
2344 - .started_ut = now,
2345 - .timeout_ut = now + VIRT_FNC_TIMEOUT + USEC_PER_SEC,
2346 - .result_body_wb = wb,
2347 - .timeout = VIRT_FNC_TIMEOUT,
2348 - .function = string_strdupz(name),
2349 - .result_cb = virt_fnc_got_data_cb,
2350 - .result_cb_data = &cond,
2351 - .payload = payload != NULL ? strdupz(payload) : NULL,
2352 - .virtual = true,
2353 - };
2354 -
2355 - uuid_t uuid;
2356 - uuid_generate_time(uuid);
2357 -
2358 - char key[UUID_STR_LEN];
2359 - uuid_unparse_lower(uuid, key);
2360 -
2361 - dictionary_write_lock(parser->inflight.functions);
2362 -
2363 - // if there is any error, our dictionary callbacks will call the caller callback to notify
2364 - // the caller about the error - no need for error handling here.
2365 - dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function));
2366 -
2367 - if(!parser->inflight.smaller_timeout || tmp.timeout_ut < parser->inflight.smaller_timeout)
2368 - parser->inflight.smaller_timeout = tmp.timeout_ut;
2369 -
2370 - // garbage collect stale inflight functions
2371 - if(parser->inflight.smaller_timeout < now)
2372 - inflight_functions_garbage_collect(parser, now);
2373 -
2374 - dictionary_write_unlock(parser->inflight.functions);
2375 -
2376 - struct timespec tp;
2377 - clock_gettime(CLOCK_REALTIME, &tp);
2378 - tp.tv_sec += (time_t)VIRT_FNC_TIMEOUT;
2379 -
2380 - pthread_mutex_lock(&cond.lock);
2381 -
2382 - int ret = pthread_cond_timedwait(&cond.cond, &cond.lock, &tp);
2383 - if (ret == ETIMEDOUT)
2384 - netdata_log_error("PLUGINSD: DYNCFG virtual function %s timed out", name);
2385 -
2386 - pthread_mutex_unlock(&cond.lock);
2387 -
2388 - dyncfg_config_t cfg;
2389 - cfg.data = strdupz(buffer_tostring(wb));
2390 - cfg.data_size = buffer_strlen(wb);
2391 -
2392 - if (rc != NULL)
2393 - *rc = cond.rc;
2394 -
2395 - buffer_free(wb);
2396 - return cfg;
2397 -}
2398 -
2399 -#define CVF_MAX_LEN (1024)
2400 -static dyncfg_config_t get_plugin_config_cb(void *usr_ctx, const char *plugin_name)
2401 -{
2402 - PARSER *parser = usr_ctx;
2403 -
2404 - if (SERVING_STREAMING(parser)) {
2405 - char buf[CVF_MAX_LEN + 1];
2406 - snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG " %s", plugin_name);
2407 - return call_virtual_function_blocking(parser, buf, NULL, NULL);
2408 - }
2409 -
2410 - return call_virtual_function_blocking(parser, FUNCTION_NAME_GET_PLUGIN_CONFIG, NULL, NULL);
2411 -}
2412 -
2413 -static dyncfg_config_t get_plugin_config_schema_cb(void *usr_ctx, const char *plugin_name)
2414 -{
2415 - PARSER *parser = usr_ctx;
2416 -
2417 - if (SERVING_STREAMING(parser)) {
2418 - char buf[CVF_MAX_LEN + 1];
2419 - snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA " %s", plugin_name);
2420 - return call_virtual_function_blocking(parser, buf, NULL, NULL);
2421 - }
2422 -
2423 - return call_virtual_function_blocking(parser, "get_plugin_config_schema", NULL, NULL);
2424 -}
2425 -
2426 -static dyncfg_config_t get_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
2427 -{
2428 - PARSER *parser = usr_ctx;
2429 - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
2430 -
2431 - buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG);
2432 - if (SERVING_STREAMING(parser))
2433 - buffer_sprintf(wb, " %s", plugin_name);
2434 -
2435 - buffer_sprintf(wb, " %s", module_name);
2436 -
2437 - dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
2438 -
2439 - buffer_free(wb);
2440 -
2441 - return ret;
2442 -}
2443 -
2444 -static dyncfg_config_t get_module_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
2445 -{
2446 - PARSER *parser = usr_ctx;
2447 - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
2448 -
2449 - buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA);
2450 - if (SERVING_STREAMING(parser))
2451 - buffer_sprintf(wb, " %s", plugin_name);
2452 -
2453 - buffer_sprintf(wb, " %s", module_name);
2454 -
2455 - dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
2456 -
2457 - buffer_free(wb);
2458 -
2459 - return ret;
2460 -}
2461 -
2462 -static dyncfg_config_t get_job_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name)
2463 -{
2464 - PARSER *parser = usr_ctx;
2465 - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
2466 -
2467 - buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA);
2468 -
2469 - if (SERVING_STREAMING(parser))
2470 - buffer_sprintf(wb, " %s", plugin_name);
2471 -
2472 - buffer_sprintf(wb, " %s", module_name);
2473 -
2474 - dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
2475 -
2476 - buffer_free(wb);
2477 -
2478 - return ret;
2479 -}
2480 -
2481 -static dyncfg_config_t get_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char* job_name)
2482 -{
2483 - PARSER *parser = usr_ctx;
2484 - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
2485 -
2486 - buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG);
2487 -
2488 - if (SERVING_STREAMING(parser))
2489 - buffer_sprintf(wb, " %s", plugin_name);
2490 -
2491 - buffer_sprintf(wb, " %s %s", module_name, job_name);
2492 -
2493 - dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL);
2494 -
2495 - buffer_free(wb);
2496 -
2497 - return ret;
2498 -}
2499 -
2500 -enum set_config_result set_plugin_config_cb(void *usr_ctx, const char *plugin_name, dyncfg_config_t *cfg)
2501 -{
2502 - PARSER *parser = usr_ctx;
2503 - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
2504 -
2505 - buffer_strcat(wb, FUNCTION_NAME_SET_PLUGIN_CONFIG);
2506 -
2507 - if (SERVING_STREAMING(parser))
2508 - buffer_sprintf(wb, " %s", plugin_name);
2509 -
2510 - int rc;
2511 - call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
2512 -
2513 - buffer_free(wb);
2514 - if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
2515 - return SET_CONFIG_REJECTED;
2516 - return SET_CONFIG_ACCEPTED;
2517 -}
2518 -
2519 -enum set_config_result set_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, dyncfg_config_t *cfg)
2520 -{
2521 - PARSER *parser = usr_ctx;
2522 - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
2523 -
2524 - buffer_strcat(wb, FUNCTION_NAME_SET_MODULE_CONFIG);
2525 -
2526 - if (SERVING_STREAMING(parser))
2527 - buffer_sprintf(wb, " %s", plugin_name);
2528 -
2529 - buffer_sprintf(wb, " %s", module_name);
2530 -
2531 - int rc;
2532 - call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
2533 -
2534 - buffer_free(wb);
2535 -
2536 - if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
2537 - return SET_CONFIG_REJECTED;
2538 - return SET_CONFIG_ACCEPTED;
2539 -}
2540 -
2541 -enum set_config_result set_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name, dyncfg_config_t *cfg)
2542 -{
2543 - PARSER *parser = usr_ctx;
2544 - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
2545 -
2546 - buffer_strcat(wb, FUNCTION_NAME_SET_JOB_CONFIG);
2547 -
2548 - if (SERVING_STREAMING(parser))
2549 - buffer_sprintf(wb, " %s", plugin_name);
2550 -
2551 - buffer_sprintf(wb, " %s %s", module_name, job_name);
2552 -
2553 - int rc;
2554 - call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data);
2555 -
2556 - buffer_free(wb);
2557 -
2558 - if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
2559 - return SET_CONFIG_REJECTED;
2560 - return SET_CONFIG_ACCEPTED;
2561 -}
2562 -
2563 -enum set_config_result delete_job_cb(void *usr_ctx, const char *plugin_name ,const char *module_name, const char *job_name)
2564 -{
2565 - PARSER *parser = usr_ctx;
2566 - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL);
2567 -
2568 - buffer_strcat(wb, FUNCTION_NAME_DELETE_JOB);
2569 -
2570 - if (SERVING_STREAMING(parser))
2571 - buffer_sprintf(wb, " %s", plugin_name);
2572 -
2573 - buffer_sprintf(wb, " %s %s", module_name, job_name);
2574 -
2575 - int rc;
2576 - call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, NULL);
2577 -
2578 - buffer_free(wb);
2579 -
2580 - if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED)
2581 - return SET_CONFIG_REJECTED;
2582 - return SET_CONFIG_ACCEPTED;
2583 -}
2584 -
2585 -
2586 -static inline PARSER_RC pluginsd_register_plugin(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
2587 - netdata_log_info("PLUGINSD: DYNCFG_ENABLE");
2588 -
2589 - if (unlikely (num_words != 2))
2590 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "missing name parameter");
2591 -
2592 - struct configurable_plugin *cfg = callocz(1, sizeof(struct configurable_plugin));
2593 -
2594 - cfg->name = strdupz(words[1]);
2595 - cfg->set_config_cb = set_plugin_config_cb;
2596 - cfg->get_config_cb = get_plugin_config_cb;
2597 - cfg->get_config_schema_cb = get_plugin_config_schema_cb;
2598 - cfg->cb_usr_ctx = parser;
2599 -
2600 - const DICTIONARY_ITEM *di = register_plugin(parser->user.host->configurable_plugins, cfg, SERVING_PLUGINSD(parser));
2601 - if (unlikely(di == NULL)) {
2602 - freez(cfg->name);
2603 - freez(cfg);
2604 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "error registering plugin");
2605 - }
2606 -
2607 - if (SERVING_PLUGINSD(parser)) {
2608 - // this is optimization for pluginsd to avoid extra dictionary lookup
2609 - // as we know which plugin is comunicating with us
2610 - parser->user.cd->cfg_dict_item = di;
2611 - parser->user.cd->configuration = cfg;
2612 - } else {
2613 - // register_plugin keeps the item acquired, so we need to release it
2614 - dictionary_acquired_item_release(parser->user.host->configurable_plugins, di);
2615 - }
2616 -
2617 - rrdpush_send_dyncfg_enable(parser->user.host, cfg->name);
2618 -
2619 - return PARSER_RC_OK;
2620 -}
2621 -
2622 -#define LOG_MSG_SIZE (1024)
2623 -#define MODULE_NAME_IDX (SERVING_PLUGINSD(parser) ? 1 : 2)
2624 -#define MODULE_TYPE_IDX (SERVING_PLUGINSD(parser) ? 2 : 3)
2625 -static inline PARSER_RC pluginsd_register_module(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
2626 - netdata_log_info("PLUGINSD: DYNCFG_REG_MODULE");
2627 -
2628 - size_t expected_num_words = SERVING_PLUGINSD(parser) ? 3 : 4;
2629 -
2630 - if (unlikely(num_words != expected_num_words)) {
2631 - char log[LOG_MSG_SIZE + 1];
2632 - snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name module_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name ");
2633 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, log);
2634 - }
2635 -
2636 - struct configurable_plugin *plug_cfg;
2637 - const DICTIONARY_ITEM *di = NULL;
2638 - if (SERVING_PLUGINSD(parser)) {
2639 - plug_cfg = parser->user.cd->configuration;
2640 - if (unlikely(plug_cfg == NULL))
2641 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "you have to enable dynamic configuration first using " PLUGINSD_KEYWORD_DYNCFG_ENABLE);
2642 - } else {
2643 - di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]);
2644 - if (unlikely(di == NULL))
2645 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "plugin not found");
2646 -
2647 - plug_cfg = (struct configurable_plugin *)dictionary_acquired_item_value(di);
2648 - }
2649 -
2650 - struct module *mod = callocz(1, sizeof(struct module));
2651 -
2652 - mod->type = str2_module_type(words[MODULE_TYPE_IDX]);
2653 - if (unlikely(mod->type == MOD_TYPE_UNKNOWN)) {
2654 - freez(mod);
2655 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "unknown module type (allowed: job_array, single)");
2656 - }
2657 -
2658 - mod->name = strdupz(words[MODULE_NAME_IDX]);
2659 -
2660 - mod->set_config_cb = set_module_config_cb;
2661 - mod->get_config_cb = get_module_config_cb;
2662 - mod->get_config_schema_cb = get_module_config_schema_cb;
2663 - mod->config_cb_usr_ctx = parser;
2664 -
2665 - mod->get_job_config_cb = get_job_config_cb;
2666 - mod->get_job_config_schema_cb = get_job_config_schema_cb;
2667 - mod->set_job_config_cb = set_job_config_cb;
2668 - mod->delete_job_cb = delete_job_cb;
2669 - mod->job_config_cb_usr_ctx = parser;
2670 -
2671 - register_module(parser->user.host->configurable_plugins, plug_cfg, mod, SERVING_PLUGINSD(parser));
2672 -
2673 - if (di != NULL)
2674 - dictionary_acquired_item_release(parser->user.host->configurable_plugins, di);
2675 -
2676 - rrdpush_send_dyncfg_reg_module(parser->user.host, plug_cfg->name, mod->name, mod->type);
2677 -
2678 - return PARSER_RC_OK;
2679 -}
2680 -
2681 -static inline PARSER_RC pluginsd_register_job_common(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused, const char *plugin_name) {
2682 - const char *module_name = words[0];
2683 - const char *job_name = words[1];
2684 - const char *job_type_str = words[2];
2685 - const char *flags_str = words[3];
2686 -
2687 - long f = str2l(flags_str);
2688 -
2689 - if (f < 0)
2690 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "invalid flags received");
2691 -
2692 - dyncfg_job_flg_t flags = f;
2693 -
2694 - if (SERVING_PLUGINSD(parser))
2695 - flags |= JOB_FLG_PLUGIN_PUSHED;
2696 - else
2697 - flags |= JOB_FLG_STREAMING_PUSHED;
2698 -
2699 - enum job_type job_type = dyncfg_str2job_type(job_type_str);
2700 - if (job_type == JOB_TYPE_UNKNOWN)
2701 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "unknown job type");
2702 -
2703 - if (SERVING_PLUGINSD(parser) && job_type == JOB_TYPE_USER)
2704 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "plugins cannot push jobs of type \"user\" (this is allowed only in streaming)");
2705 -
2706 - if (register_job(parser->user.host->configurable_plugins, plugin_name, module_name, job_name, job_type, flags, 0)) // ignore existing is off as this is explicitly called register job
2707 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "error registering job");
2708 -
2709 - rrdpush_send_dyncfg_reg_job(parser->user.host, plugin_name, module_name, job_name, job_type, flags);
2710 -
2711 - return PARSER_RC_OK;
2712 -}
2713 -
2714 -static inline PARSER_RC pluginsd_register_job(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
2715 - size_t expected_num_words = SERVING_PLUGINSD(parser) ? 5 : 6;
2716 -
2717 - if (unlikely(num_words != expected_num_words)) {
2718 - char log[LOG_MSG_SIZE + 1];
2719 - snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name job_name job_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name ");
2720 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, log);
2721 - }
2722 -
2723 - if (SERVING_PLUGINSD(parser)) {
2724 - return pluginsd_register_job_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name);
2725 - }
2726 - return pluginsd_register_job_common(&words[2], num_words - 2, parser, words[1]);
2727 -}
2728 -
2729 -static inline PARSER_RC pluginsd_dyncfg_reset(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
2730 - if (unlikely(num_words != (SERVING_PLUGINSD(parser) ? 1 : 2)))
2731 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, SERVING_PLUGINSD(parser) ? "expected 0 parameters" : "expected 1 parameter: plugin_name");
2732 -
2733 - if (SERVING_PLUGINSD(parser)) {
2734 - unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item);
2735 - rrdpush_send_dyncfg_reset(parser->user.host, parser->user.cd->configuration->name);
2736 - parser->user.cd->configuration = NULL;
2737 - } else {
2738 - const DICTIONARY_ITEM *di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]);
2739 - if (unlikely(di == NULL))
2740 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, "plugin not found");
2741 - unregister_plugin(parser->user.host->configurable_plugins, di);
2742 - rrdpush_send_dyncfg_reset(parser->user.host, words[1]);
2743 - }
2744 -
2745 - return PARSER_RC_OK;
2746 -}
2747 -
2748 -static inline PARSER_RC pluginsd_job_status_common(char **words, size_t num_words, PARSER *parser, const char *plugin_name) {
2749 - int state = str2i(words[3]);
2750 -
2751 - enum job_status status = str2job_state(words[2]);
2752 - if (unlikely(SERVING_PLUGINSD(parser) && status == JOB_STATUS_UNKNOWN))
2753 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "unknown job status");
2754 -
2755 - char *message = NULL;
2756 - if (num_words == 5 && strlen(words[4]) > 0)
2757 - message = words[4];
2758 -
2759 - const DICTIONARY_ITEM *plugin_item;
2760 - DICTIONARY *job_dict;
2761 - const DICTIONARY_ITEM *job_item = report_job_status_acq_lock(parser->user.host->configurable_plugins, &plugin_item, &job_dict, plugin_name, words[0], words[1], status, state, message);
2762 -
2763 - if (job_item != NULL) {
2764 - struct job *job = dictionary_acquired_item_value(job_item);
2765 - rrdpush_send_job_status_update(parser->user.host, plugin_name, words[0], job);
2766 -
2767 - pthread_mutex_unlock(&job->lock);
2768 - dictionary_acquired_item_release(job_dict, job_item);
2769 - dictionary_acquired_item_release(parser->user.host->configurable_plugins, plugin_item);
2770 - }
2771 -
2772 - return PARSER_RC_OK;
2773 -}
2774 -
2775 -// job_status [plugin_name if streaming] <module_name> <job_name> <status_code> <state> [message]
2776 -static PARSER_RC pluginsd_job_status(char **words, size_t num_words, PARSER *parser) {
2777 - if (SERVING_PLUGINSD(parser)) {
2778 - if (unlikely(num_words != 5 && num_words != 6))
2779 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 4 or 5 parameters: module_name, job_name, status_code, state, [optional: message]");
2780 - } else {
2781 - if (unlikely(num_words != 6 && num_words != 7))
2782 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 5 or 6 parameters: plugin_name, module_name, job_name, status_code, state, [optional: message]");
2783 - }
2784 -
2785 - if (SERVING_PLUGINSD(parser)) {
2786 - return pluginsd_job_status_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name);
2787 - }
2788 - return pluginsd_job_status_common(&words[2], num_words - 2, parser, words[1]);
2789 -}
2790 -
2791 -static PARSER_RC pluginsd_delete_job(char **words, size_t num_words, PARSER *parser) {
2792 - // this can confuse a bit but there is a diference between KEYWORD_DELETE_JOB and actual delete_job function
2793 - // they are of opossite direction
2794 - if (num_words != 4)
2795 - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DELETE_JOB, "expected 2 parameters: plugin_name, module_name, job_name");
2796 -
2797 - const char *plugin_name = get_word(words, num_words, 1);
2798 - const char *module_name = get_word(words, num_words, 2);
2799 - const char *job_name = get_word(words, num_words, 3);
2800 -
2801 - if (SERVING_STREAMING(parser))
2802 - delete_job_pname(parser->user.host->configurable_plugins, plugin_name, module_name, job_name);
2803 -
2804 - // forward to parent if any
2805 - rrdpush_send_job_deleted(parser->user.host, plugin_name, module_name, job_name);
2806 - return PARSER_RC_OK;
2807 -}
2808 -
1072 static inline PARSER_RC streaming_claimed_id(char **words, size_t num_words, PARSER *parser)
1073 {
1074 const char *host_uuid_str = get_word(words, num_words, 1);
@@ -2855,6 +1118,11 @@ static inline PARSER_RC streaming_claimed_id(char **words, size_t num_words, PAR
1118
1119 // ----------------------------------------------------------------------------
1120
1121 +void pluginsd_cleanup_v2(PARSER *parser) {
1122 + // this is called when the thread is stopped while processing
1123 + pluginsd_clear_scope_chart(parser, "THREAD CLEANUP");
1124 +}
1125 +
1126 void pluginsd_process_thread_cleanup(void *ptr) {
1127 PARSER *parser = (PARSER *)ptr;
1128
@@ -2996,32 +1264,6 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugi
1264 return count;
1265 }
1266
2999 -void pluginsd_keywords_init(PARSER *parser, PARSER_REPERTOIRE repertoire) {
3000 - parser_init_repertoire(parser, repertoire);
3001 -
3002 - if (repertoire & (PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING))
3003 - inflight_functions_init(parser);
3004 -}
3005 -
3006 -PARSER *parser_init(struct parser_user_object *user, FILE *fp_input, FILE *fp_output, int fd,
3007 - PARSER_INPUT_TYPE flags, void *ssl __maybe_unused) {
3008 - PARSER *parser;
3009 -
3010 - parser = callocz(1, sizeof(*parser));
3011 - if(user)
3012 - parser->user = *user;
3013 - parser->fd = fd;
3014 - parser->fp_input = fp_input;
3015 - parser->fp_output = fp_output;
3016 -#ifdef ENABLE_HTTPS
3017 - parser->ssl_output = ssl;
3018 -#endif
3019 - parser->flags = flags;
3020 -
3021 - spinlock_init(&parser->writer.spinlock);
3022 - return parser;
3023 -}
3024 -
1267 PARSER_RC parser_execute(PARSER *parser, PARSER_KEYWORD *keyword, char **words, size_t num_words) {
1268 switch(keyword->id) {
1269 case 1:
@@ -3078,6 +1320,9 @@ PARSER_RC parser_execute(PARSER *parser, PARSER_KEYWORD *keyword, char **words,
1320 case 42:
1321 return pluginsd_function_result_begin(words, num_words, parser);
1322
1323 + case 43:
1324 + return pluginsd_function_progress(words, num_words, parser);
1325 +
1326 case 51:
1327 return pluginsd_label(words, num_words, parser);
1328
@@ -3145,25 +1390,6 @@ void parser_init_repertoire(PARSER *parser, PARSER_REPERTOIRE repertoire) {
1390 }
1391 }
1392
3148 -static void parser_destroy_dyncfg(PARSER *parser) {
3149 - if (parser->user.cd != NULL && parser->user.cd->configuration != NULL) {
3150 - unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item);
3151 - parser->user.cd->configuration = NULL;
3152 - } else if (parser->user.host != NULL && SERVING_STREAMING(parser) && parser->user.host != localhost){
3153 - dictionary_flush(parser->user.host->configurable_plugins);
3154 - }
3155 -}
3156 -
3157 -void parser_destroy(PARSER *parser) {
3158 - if (unlikely(!parser))
3159 - return;
3160 -
3161 - parser_destroy_dyncfg(parser);
3162 -
3163 - dictionary_destroy(parser->inflight.functions);
3164 - freez(parser);
3165 -}
3166 -
1393 int pluginsd_parser_unittest(void) {
1394 PARSER *p = parser_init(NULL, NULL, NULL, -1, PARSER_INPUT_SPLIT, NULL);
1395 pluginsd_keywords_init(p, PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING);
collectors/plugins.d/pluginsd_parser.h
+2 -3
@@ -123,7 +123,7 @@ typedef struct parser {
123
124 struct {
125 DICTIONARY *functions;
126 - usec_t smaller_timeout;
126 + usec_t smaller_monotonic_timeout_ut;
127 } inflight;
128
129 struct {
@@ -136,7 +136,6 @@ PARSER *parser_init(struct parser_user_object *user, FILE *fp_input, FILE *fp_ou
136 void parser_init_repertoire(PARSER *parser, PARSER_REPERTOIRE repertoire);
137 void parser_destroy(PARSER *working_parser);
138 void pluginsd_cleanup_v2(PARSER *parser);
139 -void inflight_functions_init(PARSER *parser);
139 void pluginsd_keywords_init(PARSER *parser, PARSER_REPERTOIRE repertoire);
140 PARSER_RC parser_execute(PARSER *parser, PARSER_KEYWORD *keyword, char **words, size_t num_words);
141
@@ -186,7 +185,7 @@ static inline int parser_action(PARSER *parser, char *input) {
185 if(buffer_strlen(parser->defer.response) > PLUGINSD_MAX_DEFERRED_SIZE) {
186 // more than PLUGINSD_MAX_DEFERRED_SIZE of data,
187 // or a bad plugin that did not send the end_keyword
189 - internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response));
188 + nd_log(NDLS_DAEMON, NDLP_ERR, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response));
189 return 1;
190 }
191 }
collectors/plugins.d/pluginsd_replication.c new
+371
@@ -0,0 +1,371 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "pluginsd_replication.h"
4 +
5 +PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PARSER *parser) {
6 + int idx = 1;
7 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
8 + if(slot >= 0) idx++;
9 +
10 + char *id = get_word(words, num_words, idx++);
11 + char *start_time_str = get_word(words, num_words, idx++);
12 + char *end_time_str = get_word(words, num_words, idx++);
13 + char *child_now_str = get_word(words, num_words, idx++);
14 +
15 + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN);
16 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
17 +
18 + RRDSET *st;
19 + if (likely(!id || !*id))
20 + st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN, PLUGINSD_KEYWORD_REPLAY_BEGIN);
21 + else
22 + st = pluginsd_rrdset_cache_get_from_slot(parser, host, id, slot, PLUGINSD_KEYWORD_REPLAY_BEGIN);
23 +
24 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
25 +
26 + if(!pluginsd_set_scope_chart(parser, st, PLUGINSD_KEYWORD_REPLAY_BEGIN))
27 + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
28 +
29 + if(start_time_str && end_time_str) {
30 + time_t start_time = (time_t) str2ull_encoded(start_time_str);
31 + time_t end_time = (time_t) str2ull_encoded(end_time_str);
32 +
33 + time_t wall_clock_time = 0, tolerance;
34 + bool wall_clock_comes_from_child; (void)wall_clock_comes_from_child;
35 + if(child_now_str) {
36 + wall_clock_time = (time_t) str2ull_encoded(child_now_str);
37 + tolerance = st->update_every + 1;
38 + wall_clock_comes_from_child = true;
39 + }
40 +
41 + if(wall_clock_time <= 0) {
42 + wall_clock_time = now_realtime_sec();
43 + tolerance = st->update_every + 5;
44 + wall_clock_comes_from_child = false;
45 + }
46 +
47 +#ifdef NETDATA_LOG_REPLICATION_REQUESTS
48 + internal_error(
49 + (!st->replay.start_streaming && (end_time < st->replay.after || start_time > st->replay.before)),
50 + "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, which does not match our request (%ld to %ld).",
51 + rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time, st->replay.after, st->replay.before);
52 +
53 + internal_error(
54 + true,
55 + "REPLAY: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, child wall clock is %ld (%s), had requested %ld to %ld",
56 + rrdhost_hostname(st->rrdhost), rrdset_id(st),
57 + start_time, end_time, wall_clock_time, wall_clock_comes_from_child ? "from child" : "parent time",
58 + st->replay.after, st->replay.before);
59 +#endif
60 +
61 + if(start_time && end_time && start_time < wall_clock_time + tolerance && end_time < wall_clock_time + tolerance && start_time < end_time) {
62 + if (unlikely(end_time - start_time != st->update_every))
63 + rrdset_set_update_every_s(st, end_time - start_time);
64 +
65 + st->last_collected_time.tv_sec = end_time;
66 + st->last_collected_time.tv_usec = 0;
67 +
68 + st->last_updated.tv_sec = end_time;
69 + st->last_updated.tv_usec = 0;
70 +
71 + st->counter++;
72 + st->counter_done++;
73 +
74 + // these are only needed for db mode RAM, SAVE, MAP, ALLOC
75 + st->db.current_entry++;
76 + if(st->db.current_entry >= st->db.entries)
77 + st->db.current_entry -= st->db.entries;
78 +
79 + parser->user.replay.start_time = start_time;
80 + parser->user.replay.end_time = end_time;
81 + parser->user.replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC;
82 + parser->user.replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC;
83 + parser->user.replay.wall_clock_time = wall_clock_time;
84 + parser->user.replay.rset_enabled = true;
85 +
86 + return PARSER_RC_OK;
87 + }
88 +
89 + netdata_log_error("PLUGINSD REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN
90 + " from %ld to %ld, but timestamps are invalid "
91 + "(now is %ld [%s], tolerance %ld). Ignoring " PLUGINSD_KEYWORD_REPLAY_SET,
92 + rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time,
93 + wall_clock_time, wall_clock_comes_from_child ? "child wall clock" : "parent wall clock",
94 + tolerance);
95 + }
96 +
97 + // the child sends an RBEGIN without any parameters initially
98 + // setting rset_enabled to false, means the RSET should not store any metrics
99 + // to store metrics, the RBEGIN needs to have timestamps
100 + parser->user.replay.start_time = 0;
101 + parser->user.replay.end_time = 0;
102 + parser->user.replay.start_time_ut = 0;
103 + parser->user.replay.end_time_ut = 0;
104 + parser->user.replay.wall_clock_time = 0;
105 + parser->user.replay.rset_enabled = false;
106 + return PARSER_RC_OK;
107 +}
108 +
109 +PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARSER *parser) {
110 + int idx = 1;
111 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
112 + if(slot >= 0) idx++;
113 +
114 + char *dimension = get_word(words, num_words, idx++);
115 + char *value_str = get_word(words, num_words, idx++);
116 + char *flags_str = get_word(words, num_words, idx++);
117 +
118 + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_SET);
119 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
120 +
121 + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN);
122 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
123 +
124 + if(!parser->user.replay.rset_enabled) {
125 + nd_log_limit_static_thread_var(erl, 1, 0);
126 + nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_ERR,
127 + "PLUGINSD: 'host:%s/chart:%s' got a %s but it is disabled by %s errors",
128 + rrdhost_hostname(host), rrdset_id(st), PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN);
129 +
130 + // we have to return OK here
131 + return PARSER_RC_OK;
132 + }
133 +
134 + RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_SET);
135 + if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
136 +
137 + st->pluginsd.set = true;
138 +
139 + if (unlikely(!parser->user.replay.start_time || !parser->user.replay.end_time)) {
140 + netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s with invalid timestamps %ld to %ld from a %s. Disabling it.",
141 + rrdhost_hostname(host),
142 + rrdset_id(st),
143 + dimension,
144 + PLUGINSD_KEYWORD_REPLAY_SET,
145 + parser->user.replay.start_time,
146 + parser->user.replay.end_time,
147 + PLUGINSD_KEYWORD_REPLAY_BEGIN);
148 + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
149 + }
150 +
151 + if (unlikely(!value_str || !*value_str))
152 + value_str = "NAN";
153 +
154 + if(unlikely(!flags_str))
155 + flags_str = "";
156 +
157 + if (likely(value_str)) {
158 + RRDDIM_FLAGS rd_flags = rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED);
159 +
160 + if(!(rd_flags & RRDDIM_FLAG_ARCHIVED)) {
161 + NETDATA_DOUBLE value = str2ndd_encoded(value_str, NULL);
162 + SN_FLAGS flags = pluginsd_parse_storage_number_flags(flags_str);
163 +
164 + if (!netdata_double_isnumber(value) || (flags == SN_EMPTY_SLOT)) {
165 + value = NAN;
166 + flags = SN_EMPTY_SLOT;
167 + }
168 +
169 + rrddim_store_metric(rd, parser->user.replay.end_time_ut, value, flags);
170 + rd->collector.last_collected_time.tv_sec = parser->user.replay.end_time;
171 + rd->collector.last_collected_time.tv_usec = 0;
172 + rd->collector.counter++;
173 + }
174 + else {
175 + nd_log_limit_static_global_var(erl, 1, 0);
176 + nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_WARNING,
177 + "PLUGINSD: 'host:%s/chart:%s/dim:%s' has the ARCHIVED flag set, but it is replicated. "
178 + "Ignoring data.",
179 + rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_name(rd));
180 + }
181 + }
182 +
183 + return PARSER_RC_OK;
184 +}
185 +
186 +PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, PARSER *parser) {
187 + if(parser->user.replay.rset_enabled == false)
188 + return PARSER_RC_OK;
189 +
190 + int idx = 1;
191 + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words);
192 + if(slot >= 0) idx++;
193 +
194 + char *dimension = get_word(words, num_words, idx++);
195 + char *last_collected_ut_str = get_word(words, num_words, idx++);
196 + char *last_collected_value_str = get_word(words, num_words, idx++);
197 + char *last_calculated_value_str = get_word(words, num_words, idx++);
198 + char *last_stored_value_str = get_word(words, num_words, idx++);
199 +
200 + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
201 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
202 +
203 + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN);
204 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
205 +
206 + if(st->pluginsd.set) {
207 + // reset pos to reuse the same RDAs
208 + st->pluginsd.pos = 0;
209 + st->pluginsd.set = false;
210 + }
211 +
212 + RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE);
213 + if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
214 +
215 + usec_t dim_last_collected_ut = (usec_t)rd->collector.last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->collector.last_collected_time.tv_usec;
216 + usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0;
217 + if(last_collected_ut > dim_last_collected_ut) {
218 + rd->collector.last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC);
219 + rd->collector.last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC);
220 + }
221 +
222 + rd->collector.last_collected_value = last_collected_value_str ? str2ll_encoded(last_collected_value_str) : 0;
223 + rd->collector.last_calculated_value = last_calculated_value_str ? str2ndd_encoded(last_calculated_value_str, NULL) : 0;
224 + rd->collector.last_stored_value = last_stored_value_str ? str2ndd_encoded(last_stored_value_str, NULL) : 0.0;
225 +
226 + return PARSER_RC_OK;
227 +}
228 +
229 +PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, PARSER *parser) {
230 + if(parser->user.replay.rset_enabled == false)
231 + return PARSER_RC_OK;
232 +
233 + char *last_collected_ut_str = get_word(words, num_words, 1);
234 + char *last_updated_ut_str = get_word(words, num_words, 2);
235 +
236 + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE);
237 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
238 +
239 + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE,
240 + PLUGINSD_KEYWORD_REPLAY_BEGIN);
241 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
242 +
243 + usec_t chart_last_collected_ut = (usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec;
244 + usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0;
245 + if(last_collected_ut > chart_last_collected_ut) {
246 + st->last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC);
247 + st->last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC);
248 + }
249 +
250 + usec_t chart_last_updated_ut = (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec;
251 + usec_t last_updated_ut = last_updated_ut_str ? str2ull_encoded(last_updated_ut_str) : 0;
252 + if(last_updated_ut > chart_last_updated_ut) {
253 + st->last_updated.tv_sec = (time_t)(last_updated_ut / USEC_PER_SEC);
254 + st->last_updated.tv_usec = (last_updated_ut % USEC_PER_SEC);
255 + }
256 +
257 + st->counter++;
258 + st->counter_done++;
259 +
260 + return PARSER_RC_OK;
261 +}
262 +
263 +PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARSER *parser) {
264 + if (num_words < 7) { // accepts 7, but the 7th is optional
265 + netdata_log_error("REPLAY: malformed " PLUGINSD_KEYWORD_REPLAY_END " command");
266 + return PARSER_RC_ERROR;
267 + }
268 +
269 + const char *update_every_child_txt = get_word(words, num_words, 1);
270 + const char *first_entry_child_txt = get_word(words, num_words, 2);
271 + const char *last_entry_child_txt = get_word(words, num_words, 3);
272 + const char *start_streaming_txt = get_word(words, num_words, 4);
273 + const char *first_entry_requested_txt = get_word(words, num_words, 5);
274 + const char *last_entry_requested_txt = get_word(words, num_words, 6);
275 + const char *child_world_time_txt = get_word(words, num_words, 7); // optional
276 +
277 + time_t update_every_child = (time_t) str2ull_encoded(update_every_child_txt);
278 + time_t first_entry_child = (time_t) str2ull_encoded(first_entry_child_txt);
279 + time_t last_entry_child = (time_t) str2ull_encoded(last_entry_child_txt);
280 +
281 + bool start_streaming = (strcmp(start_streaming_txt, "true") == 0);
282 + time_t first_entry_requested = (time_t) str2ull_encoded(first_entry_requested_txt);
283 + time_t last_entry_requested = (time_t) str2ull_encoded(last_entry_requested_txt);
284 +
285 + // the optional child world time
286 + time_t child_world_time = (child_world_time_txt && *child_world_time_txt) ? (time_t) str2ull_encoded(
287 + child_world_time_txt) : now_realtime_sec();
288 +
289 + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_END);
290 + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
291 +
292 + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END, PLUGINSD_KEYWORD_REPLAY_BEGIN);
293 + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL);
294 +
295 +#ifdef NETDATA_LOG_REPLICATION_REQUESTS
296 + internal_error(true,
297 + "PLUGINSD REPLAY: 'host:%s/chart:%s': got a " PLUGINSD_KEYWORD_REPLAY_END " child db from %llu to %llu, start_streaming %s, had requested from %llu to %llu, wall clock %llu",
298 + rrdhost_hostname(host), rrdset_id(st),
299 + (unsigned long long)first_entry_child, (unsigned long long)last_entry_child,
300 + start_streaming?"true":"false",
301 + (unsigned long long)first_entry_requested, (unsigned long long)last_entry_requested,
302 + (unsigned long long)child_world_time
303 + );
304 +#endif
305 +
306 + parser->user.data_collections_count++;
307 +
308 + if(parser->user.replay.rset_enabled && st->rrdhost->receiver) {
309 + time_t now = now_realtime_sec();
310 + time_t started = st->rrdhost->receiver->replication_first_time_t;
311 + time_t current = parser->user.replay.end_time;
312 +
313 + if(started && current > started) {
314 + host->rrdpush_receiver_replication_percent = (NETDATA_DOUBLE) (current - started) * 100.0 / (NETDATA_DOUBLE) (now - started);
315 + worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION,
316 + host->rrdpush_receiver_replication_percent);
317 + }
318 + }
319 +
320 + parser->user.replay.start_time = 0;
321 + parser->user.replay.end_time = 0;
322 + parser->user.replay.start_time_ut = 0;
323 + parser->user.replay.end_time_ut = 0;
324 + parser->user.replay.wall_clock_time = 0;
325 + parser->user.replay.rset_enabled = false;
326 +
327 + st->counter++;
328 + st->counter_done++;
329 + store_metric_collection_completed();
330 +
331 +#ifdef NETDATA_LOG_REPLICATION_REQUESTS
332 + st->replay.start_streaming = false;
333 + st->replay.after = 0;
334 + st->replay.before = 0;
335 + if(start_streaming)
336 + st->replay.log_next_data_collection = true;
337 +#endif
338 +
339 + if (start_streaming) {
340 + if (st->update_every != update_every_child)
341 + rrdset_set_update_every_s(st, update_every_child);
342 +
343 + if(rrdset_flag_check(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS)) {
344 + rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
345 + rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS);
346 + rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK);
347 + rrdhost_receiver_replicating_charts_minus_one(st->rrdhost);
348 + }
349 +#ifdef NETDATA_LOG_REPLICATION_REQUESTS
350 + else
351 + internal_error(true, "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_END " with enable_streaming = true, but there is no replication in progress for this chart.",
352 + rrdhost_hostname(host), rrdset_id(st));
353 +#endif
354 +
355 + pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END);
356 +
357 + host->rrdpush_receiver_replication_percent = 100.0;
358 + worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, host->rrdpush_receiver_replication_percent);
359 +
360 + return PARSER_RC_OK;
361 + }
362 +
363 + pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END);
364 +
365 + rrdcontext_updated_retention_rrdset(st);
366 +
367 + bool ok = replicate_chart_request(send_to_plugin, parser, host, st,
368 + first_entry_child, last_entry_child, child_world_time,
369 + first_entry_requested, last_entry_requested);
370 + return ok ? PARSER_RC_OK : PARSER_RC_ERROR;
371 +}
collectors/plugins.d/pluginsd_replication.h new
+14
@@ -0,0 +1,14 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_PLUGINSD_REPLICATION_H
4 +#define NETDATA_PLUGINSD_REPLICATION_H
5 +
6 +#include "pluginsd_internals.h"
7 +
8 +PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PARSER *parser);
9 +PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARSER *parser);
10 +PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, PARSER *parser);
11 +PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, PARSER *parser);
12 +PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARSER *parser);
13 +
14 +#endif //NETDATA_PLUGINSD_REPLICATION_H
collectors/proc.plugin/proc_diskstats.c
+13 -7
@@ -1033,12 +1033,16 @@ static void add_labels_to_disk(struct disk *d, RRDSET *st) {
1033 rrdlabels_add(st->rrdlabels, "device_type", get_disk_type_string(d->type), RRDLABEL_SRC_AUTO);
1034 }
1035
1036 -static int diskstats_function_block_devices(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused,
1037 - void *collector_data __maybe_unused,
1038 - rrd_function_result_callback_t result_cb, void *result_cb_data,
1039 - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
1040 - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
1041 - void *register_canceller_cb_data __maybe_unused) {
1036 +static int diskstats_function_block_devices(uuid_t *transaction __maybe_unused, BUFFER *wb,
1037 + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
1038 + void *collector_data __maybe_unused,
1039 + rrd_function_result_callback_t result_cb, void *result_cb_data,
1040 + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
1041 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
1042 + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
1043 + void *register_canceller_cb_data __maybe_unused,
1044 + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
1045 + void *register_progresser_cb_data __maybe_unused) {
1046
1047 buffer_flush(wb);
1048 wb->content_type = CT_APPLICATION_JSON;
@@ -1490,7 +1494,9 @@ int do_proc_diskstats(int update_every, usec_t dt) {
1494
1495 static bool add_func = true;
1496 if (add_func) {
1493 - rrd_function_add(localhost, NULL, "block-devices", 10, RRDFUNCTIONS_DISKSTATS_HELP, true, diskstats_function_block_devices, NULL);
1497 + rrd_function_add(localhost, NULL, "block-devices", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSTATS_HELP,
1498 + "top", HTTP_ACCESS_ANY, true,
1499 + diskstats_function_block_devices, NULL);
1500 add_func = false;
1501 }
1502
collectors/proc.plugin/proc_net_dev.c
+13 -8
@@ -610,12 +610,16 @@ static inline void netdev_rename_all_lock(void) {
610
611 // ----------------------------------------------------------------------------
612
613 -int netdev_function_net_interfaces(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused,
614 - void *collector_data __maybe_unused,
615 - rrd_function_result_callback_t result_cb, void *result_cb_data,
616 - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
617 - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
618 - void *register_canceller_cb_data __maybe_unused) {
613 +int netdev_function_net_interfaces(uuid_t *transaction __maybe_unused, BUFFER *wb,
614 + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
615 + void *collector_data __maybe_unused,
616 + rrd_function_result_callback_t result_cb, void *result_cb_data,
617 + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
618 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
619 + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
620 + void *register_canceller_cb_data __maybe_unused,
621 + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
622 + void *register_progresser_cb_data __maybe_unused) {
623
624 buffer_flush(wb);
625 wb->content_type = CT_APPLICATION_JSON;
@@ -1926,8 +1930,9 @@ void *netdev_main(void *ptr)
1930
1931 netdata_thread_cleanup_push(netdev_main_cleanup, ptr) {
1932 rrd_collector_started();
1929 - rrd_function_add(localhost, NULL, "network-interfaces", 10, RRDFUNCTIONS_NETDEV_HELP, true
1930 - , netdev_function_net_interfaces, NULL);
1933 + rrd_function_add(localhost, NULL, "network-interfaces", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_NETDEV_HELP,
1934 + "top", HTTP_ACCESS_ANY,
1935 + true, netdev_function_net_interfaces, NULL);
1936
1937 usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
1938 heartbeat_t hb;
collectors/systemd-journal.plugin/systemd-internals.h
+2 -4
@@ -88,7 +88,6 @@ struct journal_file {
88
89 extern DICTIONARY *journal_files_registry;
90 extern DICTIONARY *used_hashes_registry;
91 -extern DICTIONARY *function_query_status_dict;
91 extern DICTIONARY *boot_ids_to_first_ut;
92
93 int journal_file_dict_items_backward_compar(const void *a, const void *b);
@@ -120,8 +119,7 @@ struct journal_directory {
119 extern struct journal_directory journal_directories[MAX_JOURNAL_DIRECTORIES];
120
121 void journal_init_files_and_directories(void);
123 -void journal_init_query_status(void);
124 -void function_systemd_journal(const char *transaction, char *function, int timeout, bool *cancelled);
122 +void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled);
123 void journal_file_update_header(const char *filename, struct journal_file *jf);
124
125 void netdata_systemd_journal_message_ids_init(void);
@@ -130,7 +128,7 @@ void netdata_systemd_journal_transform_message_id(FACETS *facets __maybe_unused,
128 void *journal_watcher_main(void *arg);
129
130 #ifdef ENABLE_SYSTEMD_DBUS
133 -void function_systemd_units(const char *transaction, char *function, int timeout, bool *cancelled);
131 +void function_systemd_units(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled);
132 #endif
133
134 static inline void send_newline_and_flush(void) {
collectors/systemd-journal.plugin/systemd-journal.c
+23 -124
@@ -26,6 +26,8 @@
26 #define SYSTEMD_JOURNAL_SAMPLING_SLOTS 1000
27 #define SYSTEMD_JOURNAL_SAMPLING_RECALIBRATE 10000
28
29 +#define SYSTEMD_JOURNAL_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
30 +
31 #define JOURNAL_PARAMETER_HELP "help"
32 #define JOURNAL_PARAMETER_AFTER "after"
33 #define JOURNAL_PARAMETER_BEFORE "before"
@@ -39,8 +41,6 @@
41 #define JOURNAL_PARAMETER_DATA_ONLY "data_only"
42 #define JOURNAL_PARAMETER_SOURCE "source"
43 #define JOURNAL_PARAMETER_INFO "info"
42 -#define JOURNAL_PARAMETER_ID "id"
43 -#define JOURNAL_PARAMETER_PROGRESS "progress"
44 #define JOURNAL_PARAMETER_SLICE "slice"
45 #define JOURNAL_PARAMETER_DELTA "delta"
46 #define JOURNAL_PARAMETER_TAIL "tail"
@@ -183,11 +183,11 @@
183
184 typedef struct function_query_status {
185 bool *cancelled; // a pointer to the cancelling boolean
186 - usec_t stop_monotonic_ut;
187 -
188 - usec_t started_monotonic_ut;
186 + usec_t *stop_monotonic_ut;
187
188 // request
189 + const char *transaction;
190 +
191 SD_JOURNAL_FILE_SOURCE_TYPE source_type;
192 SIMPLE_PATTERN *sources;
193 usec_t after_ut;
@@ -807,7 +807,7 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward(
807 FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes);
808 last_bytes = bytes;
809
810 - status = check_stop(fqs->cancelled, &fqs->stop_monotonic_ut);
810 + status = check_stop(fqs->cancelled, fqs->stop_monotonic_ut);
811 }
812 }
813 else if(sample == SAMPLING_SKIP_FIELDS)
@@ -914,7 +914,7 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward(
914 FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes);
915 last_bytes = bytes;
916
917 - status = check_stop(fqs->cancelled, &fqs->stop_monotonic_ut);
917 + status = check_stop(fqs->cancelled, fqs->stop_monotonic_ut);
918 }
919 }
920 else if(sample == SAMPLING_SKIP_FIELDS)
@@ -1150,6 +1150,7 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1150 usec_t started_ut = query_started_ut;
1151 usec_t ended_ut = started_ut;
1152 usec_t duration_ut = 0, max_duration_ut = 0;
1153 + usec_t progress_duration_ut = 0;
1154
1155 sampling_query_init(fqs, facets);
1156
@@ -1164,9 +1165,7 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1165 started_ut = ended_ut;
1166
1167 // do not even try to do the query if we expect it to pass the timeout
1167 - if(ended_ut > (query_started_ut + (fqs->stop_monotonic_ut - query_started_ut) * 3 / 4) &&
1168 - ended_ut + max_duration_ut * 2 >= fqs->stop_monotonic_ut) {
1169 -
1168 + if(ended_ut + max_duration_ut * 3 >= *fqs->stop_monotonic_ut) {
1169 partial = true;
1170 status = ND_SD_JOURNAL_TIMED_OUT;
1171 break;
@@ -1211,6 +1210,14 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1210 if(duration_ut > max_duration_ut)
1211 max_duration_ut = duration_ut;
1212
1213 + progress_duration_ut += duration_ut;
1214 + if(progress_duration_ut >= SYSTEMD_JOURNAL_PROGRESS_EVERY_UT) {
1215 + progress_duration_ut = 0;
1216 + netdata_mutex_lock(&stdout_mutex);
1217 + pluginsd_function_progress_to_stdout(fqs->transaction, f + 1, files_used);
1218 + netdata_mutex_unlock(&stdout_mutex);
1219 + }
1220 +
1221 buffer_json_add_array_item_object(wb); // journal file
1222 {
1223 // information about the file
@@ -1422,15 +1429,6 @@ static void netdata_systemd_journal_function_help(const char *transaction) {
1429 " all the available systemd journal sources.\n"
1430 " When `"JOURNAL_PARAMETER_INFO"` is requested, all other parameters are ignored.\n"
1431 "\n"
1425 - " "JOURNAL_PARAMETER_ID":STRING\n"
1426 - " Caller supplied unique ID of the request.\n"
1427 - " This can be used later to request a progress report of the query.\n"
1428 - " Optional, but if omitted no `"JOURNAL_PARAMETER_PROGRESS"` can be requested.\n"
1429 - "\n"
1430 - " "JOURNAL_PARAMETER_PROGRESS"\n"
1431 - " Request a progress report (the `id` of a running query is required).\n"
1432 - " When `"JOURNAL_PARAMETER_PROGRESS"` is requested, only parameter `"JOURNAL_PARAMETER_ID"` is used.\n"
1433 - "\n"
1432 " "JOURNAL_PARAMETER_DATA_ONLY":true or "JOURNAL_PARAMETER_DATA_ONLY":false\n"
1433 " Quickly respond with data requested, without generating a\n"
1434 " `histogram`, `facets` counters and `items`.\n"
@@ -1522,67 +1520,7 @@ static void netdata_systemd_journal_function_help(const char *transaction) {
1520 buffer_free(wb);
1521 }
1522
1525 -DICTIONARY *function_query_status_dict = NULL;
1526 -
1527 -static void function_systemd_journal_progress(BUFFER *wb, const char *transaction, const char *progress_id) {
1528 - if(!progress_id || !(*progress_id)) {
1529 - netdata_mutex_lock(&stdout_mutex);
1530 - pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST, "missing progress id");
1531 - netdata_mutex_unlock(&stdout_mutex);
1532 - return;
1533 - }
1534 -
1535 - const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(function_query_status_dict, progress_id);
1536 -
1537 - if(!item) {
1538 - netdata_mutex_lock(&stdout_mutex);
1539 - pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, "progress id is not found here");
1540 - netdata_mutex_unlock(&stdout_mutex);
1541 - return;
1542 - }
1543 -
1544 - FUNCTION_QUERY_STATUS *fqs = dictionary_acquired_item_value(item);
1545 -
1546 - usec_t now_monotonic_ut = now_monotonic_usec();
1547 - if(now_monotonic_ut + 10 * USEC_PER_SEC > fqs->stop_monotonic_ut)
1548 - fqs->stop_monotonic_ut = now_monotonic_ut + 10 * USEC_PER_SEC;
1549 -
1550 - usec_t duration_ut = now_monotonic_ut - fqs->started_monotonic_ut;
1551 -
1552 - size_t files_matched = fqs->files_matched;
1553 - size_t file_working = fqs->file_working;
1554 - if(file_working > files_matched)
1555 - files_matched = file_working;
1556 -
1557 - size_t rows_read = __atomic_load_n(&fqs->rows_read, __ATOMIC_RELAXED);
1558 - size_t bytes_read = __atomic_load_n(&fqs->bytes_read, __ATOMIC_RELAXED);
1559 -
1560 - buffer_flush(wb);
1561 - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
1562 - buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
1563 - buffer_json_member_add_string(wb, "type", "table");
1564 - buffer_json_member_add_uint64(wb, "running_duration_usec", duration_ut);
1565 - buffer_json_member_add_double(wb, "progress", (double)file_working * 100.0 / (double)files_matched);
1566 - char msg[1024 + 1];
1567 - snprintfz(msg, sizeof(msg) - 1,
1568 - "Read %zu rows (%0.0f rows/s), "
1569 - "data %0.1f MB (%0.1f MB/s), "
1570 - "file %zu of %zu",
1571 - rows_read, (double)rows_read / (double)duration_ut * (double)USEC_PER_SEC,
1572 - (double)bytes_read / 1024.0 / 1024.0, ((double)bytes_read / (double)duration_ut * (double)USEC_PER_SEC) / 1024.0 / 1024.0,
1573 - file_working, files_matched
1574 - );
1575 - buffer_json_member_add_string(wb, "message", msg);
1576 - buffer_json_finalize(wb);
1577 -
1578 - netdata_mutex_lock(&stdout_mutex);
1579 - pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", now_realtime_sec() + 1, wb);
1580 - netdata_mutex_unlock(&stdout_mutex);
1581 -
1582 - dictionary_acquired_item_release(function_query_status_dict, item);
1583 -}
1584 -
1585 -void function_systemd_journal(const char *transaction, char *function, int timeout, bool *cancelled) {
1523 +void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled) {
1524 fstat_thread_calls = 0;
1525 fstat_thread_cached_responses = 0;
1526
@@ -1590,11 +1528,9 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1528 buffer_flush(wb);
1529 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
1530
1593 - usec_t now_monotonic_ut = now_monotonic_usec();
1531 FUNCTION_QUERY_STATUS tmp_fqs = {
1532 .cancelled = cancelled,
1596 - .started_monotonic_ut = now_monotonic_ut,
1597 - .stop_monotonic_ut = now_monotonic_ut + (timeout * USEC_PER_SEC),
1533 + .stop_monotonic_ut = stop_monotonic_ut,
1534 };
1535 FUNCTION_QUERY_STATUS *fqs = NULL;
1536 const DICTIONARY_ITEM *fqs_item = NULL;
@@ -1616,8 +1552,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1552 facets_accepted_param(facets, JOURNAL_PARAMETER_HISTOGRAM);
1553 facets_accepted_param(facets, JOURNAL_PARAMETER_IF_MODIFIED_SINCE);
1554 facets_accepted_param(facets, JOURNAL_PARAMETER_DATA_ONLY);
1619 - facets_accepted_param(facets, JOURNAL_PARAMETER_ID);
1620 - facets_accepted_param(facets, JOURNAL_PARAMETER_PROGRESS);
1555 facets_accepted_param(facets, JOURNAL_PARAMETER_DELTA);
1556 facets_accepted_param(facets, JOURNAL_PARAMETER_TAIL);
1557 facets_accepted_param(facets, JOURNAL_PARAMETER_SAMPLING);
@@ -1724,7 +1658,7 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1658 // ------------------------------------------------------------------------
1659 // parse the parameters
1660
1727 - bool info = false, data_only = false, progress = false, slice = JOURNAL_DEFAULT_SLICE_MODE, delta = false, tail = false;
1661 + bool info = false, data_only = false, slice = JOURNAL_DEFAULT_SLICE_MODE, delta = false, tail = false;
1662 time_t after_s = 0, before_s = 0;
1663 usec_t anchor = 0;
1664 usec_t if_modified_since = 0;
@@ -1733,7 +1667,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1667 const char *query = NULL;
1668 const char *chart = NULL;
1669 SIMPLE_PATTERN *sources = NULL;
1736 - const char *progress_id = NULL;
1670 SD_JOURNAL_FILE_SOURCE_TYPE source_type = SDJF_ALL;
1671 size_t filters = 0;
1672 size_t sampling = SYSTEMD_JOURNAL_DEFAULT_ITEMS_SAMPLING;
@@ -1753,9 +1686,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1686 else if(strcmp(keyword, JOURNAL_PARAMETER_INFO) == 0) {
1687 info = true;
1688 }
1756 - else if(strcmp(keyword, JOURNAL_PARAMETER_PROGRESS) == 0) {
1757 - progress = true;
1758 - }
1689 else if(strncmp(keyword, JOURNAL_PARAMETER_DELTA ":", sizeof(JOURNAL_PARAMETER_DELTA ":") - 1) == 0) {
1690 char *v = &keyword[sizeof(JOURNAL_PARAMETER_DELTA ":") - 1];
1691
@@ -1791,12 +1721,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1721 else
1722 slice = true;
1723 }
1794 - else if(strncmp(keyword, JOURNAL_PARAMETER_ID ":", sizeof(JOURNAL_PARAMETER_ID ":") - 1) == 0) {
1795 - char *id = &keyword[sizeof(JOURNAL_PARAMETER_ID ":") - 1];
1796 -
1797 - if(*id)
1798 - progress_id = id;
1799 - }
1724 else if(strncmp(keyword, JOURNAL_PARAMETER_SOURCE ":", sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1) == 0) {
1725 const char *value = &keyword[sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1];
1726
@@ -1930,15 +1854,8 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1854 // ------------------------------------------------------------------------
1855 // put this request into the progress db
1856
1933 - if(progress_id && *progress_id) {
1934 - fqs_item = dictionary_set_and_acquire_item(function_query_status_dict, progress_id, &tmp_fqs, sizeof(tmp_fqs));
1935 - fqs = dictionary_acquired_item_value(fqs_item);
1936 - }
1937 - else {
1938 - // no progress id given, proceed without registering our progress in the dictionary
1939 - fqs = &tmp_fqs;
1940 - fqs_item = NULL;
1941 - }
1857 + fqs = &tmp_fqs;
1858 + fqs_item = NULL;
1859
1860 // ------------------------------------------------------------------------
1861 // validate parameters
@@ -1969,6 +1886,7 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1886 // ------------------------------------------------------------------------
1887 // set query time-frame, anchors and direction
1888
1889 + fqs->transaction = transaction;
1890 fqs->after_ut = after_s * USEC_PER_SEC;
1891 fqs->before_ut = (before_s * USEC_PER_SEC) + USEC_PER_SEC - 1;
1892 fqs->if_modified_since = if_modified_since;
@@ -2045,11 +1963,9 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
1963 buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_INFO, false);
1964 buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_SLICE, fqs->slice);
1965 buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DATA_ONLY, fqs->data_only);
2048 - buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_PROGRESS, false);
1966 buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DELTA, fqs->delta);
1967 buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_TAIL, fqs->tail);
1968 buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_SAMPLING, fqs->sampling);
2052 - buffer_json_member_add_string(wb, JOURNAL_PARAMETER_ID, progress_id);
1969 buffer_json_member_add_uint64(wb, "source_type", fqs->source_type);
1970 buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_AFTER, fqs->after_ut / USEC_PER_SEC);
1971 buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_BEFORE, fqs->before_ut / USEC_PER_SEC);
@@ -2098,11 +2014,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo
2014 goto output;
2015 }
2016
2101 - if(progress) {
2102 - function_systemd_journal_progress(wb, transaction, progress_id);
2103 - goto cleanup;
2104 - }
2105 -
2017 response = netdata_systemd_journal_query(wb, facets, fqs);
2018
2019 // ------------------------------------------------------------------------
@@ -2124,16 +2035,4 @@ cleanup:
2035 simple_pattern_free(sources);
2036 facets_destroy(facets);
2037 buffer_free(wb);
2127 -
2128 - if(fqs_item) {
2129 - dictionary_del(function_query_status_dict, dictionary_acquired_item_name(fqs_item));
2130 - dictionary_acquired_item_release(function_query_status_dict, fqs_item);
2131 - dictionary_garbage_collect(function_query_status_dict);
2132 - }
2133 -}
2134 -
2135 -void journal_init_query_status(void) {
2136 - function_query_status_dict = dictionary_create_advanced(
2137 - DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
2138 - NULL, sizeof(FUNCTION_QUERY_STATUS));
2038 }
collectors/systemd-journal.plugin/systemd-main.c
+10 -7
@@ -29,7 +29,6 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
29 // initialization
30
31 netdata_systemd_journal_message_ids_init();
32 - journal_init_query_status();
32 journal_init_files_and_directories();
33
34 if (!journal_data_direcories_exist()) {
@@ -46,17 +45,19 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
45 journal_files_registry_update();
46
47 bool cancelled = false;
48 + usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
49 char buf[] = "systemd-journal after:-8640000 before:0 direction:backward last:200 data_only:false slice:true source:all";
50 // char buf[] = "systemd-journal after:1695332964 before:1695937764 direction:backward last:100 slice:true source:all DHKucpqUoe1:PtVoyIuX.MU";
51 // char buf[] = "systemd-journal after:1694511062 before:1694514662 anchor:1694514122024403";
52 - function_systemd_journal("123", buf, 600, &cancelled);
52 + function_systemd_journal("123", buf, &stop_monotonic_ut, &cancelled);
53 // function_systemd_units("123", "systemd-units", 600, &cancelled);
54 exit(1);
55 }
56 #ifdef ENABLE_SYSTEMD_DBUS
57 if(argc == 2 && strcmp(argv[1], "debug-units") == 0) {
58 bool cancelled = false;
59 - function_systemd_units("123", "systemd-units", 600, &cancelled);
59 + usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
60 + function_systemd_units("123", "systemd-units", &stop_monotonic_ut, &cancelled);
61 exit(1);
62 }
63 #endif
@@ -87,12 +88,14 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
88
89 netdata_mutex_lock(&stdout_mutex);
90
90 - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n",
91 - SYSTEMD_JOURNAL_FUNCTION_NAME, SYSTEMD_JOURNAL_DEFAULT_TIMEOUT, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);
91 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" \"members\" %d\n",
92 + SYSTEMD_JOURNAL_FUNCTION_NAME, SYSTEMD_JOURNAL_DEFAULT_TIMEOUT, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION,
93 + RRDFUNCTIONS_PRIORITY_DEFAULT);
94
95 #ifdef ENABLE_SYSTEMD_DBUS
94 - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n",
95 - SYSTEMD_UNITS_FUNCTION_NAME, SYSTEMD_UNITS_DEFAULT_TIMEOUT, SYSTEMD_UNITS_FUNCTION_DESCRIPTION);
96 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"top\" \"members\" %d\n",
97 + SYSTEMD_UNITS_FUNCTION_NAME, SYSTEMD_UNITS_DEFAULT_TIMEOUT, SYSTEMD_UNITS_FUNCTION_DESCRIPTION,
98 + RRDFUNCTIONS_PRIORITY_DEFAULT);
99 #endif
100
101 fflush(stdout);
collectors/systemd-journal.plugin/systemd-units.c
+1 -1
@@ -1596,7 +1596,7 @@ void systemd_units_assign_priority(UnitInfo *base) {
1596 }
1597 }
1598
1599 -void function_systemd_units(const char *transaction, char *function, int timeout, bool *cancelled) {
1599 +void function_systemd_units(const char *transaction, char *function, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) {
1600 char *words[SYSTEMD_UNITS_MAX_PARAMS] = { NULL };
1601 size_t num_words = quoted_strings_splitter_pluginsd(function, words, SYSTEMD_UNITS_MAX_PARAMS);
1602 for(int i = 1; i < SYSTEMD_UNITS_MAX_PARAMS ;i++) {
daemon/main.c
+5
@@ -1395,6 +1395,7 @@ void replication_initialize(void);
1395 void bearer_tokens_init(void);
1396 int unittest_rrdpush_compressions(void);
1397 int uuid_unittest(void);
1398 +int progress_unittest(void);
1399
1400 int main(int argc, char **argv) {
1401 // initialize the system clocks
@@ -1620,6 +1621,10 @@ int main(int argc, char **argv) {
1621 unittest_running = true;
1622 return unittest_rrdpush_compressions();
1623 }
1624 + else if(strcmp(optarg, "progresstest") == 0) {
1625 + unittest_running = true;
1626 + return progress_unittest();
1627 + }
1628 else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
1629 optarg += strlen(createdataset_string);
1630 unsigned history_seconds = strtoul(optarg, NULL, 0);
database/contexts/api_v2.c
+20 -7
@@ -167,6 +167,9 @@ struct function_v2_entry {
167 size_t used;
168 size_t *node_ids;
169 STRING *help;
170 + STRING *tags;
171 + HTTP_ACCESS access;
172 + int priority;
173 };
174
175 struct context_v2_entry {
@@ -913,8 +916,11 @@ static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool qu
916 .size = 1,
917 .node_ids = &ctl->nodes.ni,
918 .help = NULL,
919 + .tags = NULL,
920 + .access = HTTP_ACCESS_MEMBERS,
921 + .priority = RRDFUNCTIONS_PRIORITY_DEFAULT,
922 };
917 - host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help);
923 + host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help, &t.tags, &t.access, &t.priority);
924 }
925
926 if(ctl->mode & CONTEXTS_V2_NODES) {
@@ -2062,12 +2068,19 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
2068 struct function_v2_entry *t;
2069 dfe_start_read(ctl.functions.dict, t) {
2070 buffer_json_add_array_item_object(wb);
2065 - buffer_json_member_add_string(wb, "name", t_dfe.name);
2066 - buffer_json_member_add_string(wb, "help", string2str(t->help));
2067 - buffer_json_member_add_array(wb, "ni");
2068 - for (size_t i = 0; i < t->used; i++)
2069 - buffer_json_add_array_item_uint64(wb, t->node_ids[i]);
2070 - buffer_json_array_close(wb);
2071 + {
2072 + buffer_json_member_add_string(wb, "name", t_dfe.name);
2073 + buffer_json_member_add_string(wb, "help", string2str(t->help));
2074 + buffer_json_member_add_array(wb, "ni");
2075 + {
2076 + for (size_t i = 0; i < t->used; i++)
2077 + buffer_json_add_array_item_uint64(wb, t->node_ids[i]);
2078 + }
2079 + buffer_json_array_close(wb);
2080 + buffer_json_member_add_string(wb, "tags", string2str(t->tags));
2081 + buffer_json_member_add_string(wb, "access", http_id2access(t->access));
2082 + buffer_json_member_add_uint64(wb, "priority", t->priority);
2083 + }
2084 buffer_json_object_close(wb);
2085 }
2086 dfe_done(t);
database/contexts/rrdcontext.h
+2
@@ -299,6 +299,8 @@ typedef struct query_target_request {
299
300 qt_interrupt_callback_t interrupt_callback;
301 void *interrupt_callback_data;
302 +
303 + uuid_t *transaction;
304 } QUERY_TARGET_REQUEST;
305
306 #define GROUP_BY_MAX_LABEL_KEYS 10
database/rrd.h
+1
@@ -966,6 +966,7 @@ void rrdset_memory_file_update(RRDSET *st);
966 const char *rrdset_cache_filename(RRDSET *st);
967 bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE memory_mode);
968
969 +#include "rrdcollector.h"
970 #include "rrdfunctions.h"
971
972 // ----------------------------------------------------------------------------
database/rrdcollector.c new
+137
@@ -0,0 +1,137 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#define NETDATA_RRDCOLLECTOR_INTERNALS
4 +#include "rrdcollector.h"
5 +
6 +// Each function points to this collector structure
7 +// so that when the collector exits, all of them will
8 +// be invalidated (running == false)
9 +// The last function using this collector
10 +// frees the structure too (or when the collector calls
11 +// rrdset_collector_finished()).
12 +
13 +struct rrd_collector {
14 + int32_t refcount;
15 + int32_t refcount_dispatcher;
16 + pid_t tid;
17 + bool running;
18 +};
19 +
20 +// Each thread that adds RRDSET functions has to call
21 +// rrdset_collector_started() and rrdset_collector_finished()
22 +// to create the collector structure.
23 +
24 +__thread struct rrd_collector *thread_rrd_collector = NULL;
25 +
26 +inline bool rrd_collector_running(struct rrd_collector *rdc) {
27 + return __atomic_load_n(&rdc->running, __ATOMIC_RELAXED);
28 +}
29 +
30 +inline pid_t rrd_collector_tid(struct rrd_collector *rdc) {
31 + return rdc->tid;
32 +}
33 +
34 +bool rrd_collector_dispatcher_acquire(struct rrd_collector *rdc) {
35 + int32_t expected = __atomic_load_n(&rdc->refcount_dispatcher, __ATOMIC_RELAXED);
36 + int32_t wanted;
37 + do {
38 + if(expected < 0)
39 + return false;
40 +
41 + wanted = expected + 1;
42 + } while(!__atomic_compare_exchange_n(&rdc->refcount_dispatcher, &expected, wanted, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
43 +
44 + return true;
45 +}
46 +
47 +void rrd_collector_dispatcher_release(struct rrd_collector *rdc) {
48 + __atomic_sub_fetch(&rdc->refcount_dispatcher, 1, __ATOMIC_RELAXED);
49 +}
50 +
51 +static void rrd_collector_free(struct rrd_collector *rdc) {
52 + if(rdc->running)
53 + return;
54 +
55 + int32_t expected = 0;
56 + if(!__atomic_compare_exchange_n(&rdc->refcount, &expected, -1, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
57 + // the collector is still referenced by charts.
58 + // leave it hanging there, the last chart will actually free it.
59 + return;
60 + }
61 +
62 + // we can free it now
63 + freez(rdc);
64 +}
65 +
66 +// called once per collector
67 +void rrd_collector_started(void) {
68 + if(!thread_rrd_collector)
69 + thread_rrd_collector = callocz(1, sizeof(struct rrd_collector));
70 +
71 + thread_rrd_collector->tid = gettid();
72 + __atomic_store_n(&thread_rrd_collector->running, true, __ATOMIC_RELAXED);
73 +}
74 +
75 +// called once per collector
76 +void rrd_collector_finished(void) {
77 + if(!thread_rrd_collector)
78 + return;
79 +
80 + __atomic_store_n(&thread_rrd_collector->running, false, __ATOMIC_RELAXED);
81 +
82 + // wait for any cancellation requests to be dispatched;
83 + // the problem is that cancellation requests require a structure allocated by the collector,
84 + // so, while cancellation requests are being dispatched, this structure is accessed.
85 + // delaying the exit of the thread is required to avoid cleaning up this structure.
86 +
87 + int32_t expected = 0;
88 + while(!__atomic_compare_exchange_n(&thread_rrd_collector->refcount_dispatcher, &expected, -1, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
89 + expected = 0;
90 + sleep_usec(1 * USEC_PER_MS);
91 + }
92 +
93 + rrd_collector_free(thread_rrd_collector);
94 + thread_rrd_collector = NULL;
95 +}
96 +
97 +bool rrd_collector_acquire(struct rrd_collector *rdc) {
98 +
99 + int32_t expected = __atomic_load_n(&rdc->refcount, __ATOMIC_RELAXED), wanted = 0;
100 + do {
101 + if(expected < 0 || !rrd_collector_running(rdc))
102 + return false;
103 +
104 + wanted = expected + 1;
105 + } while(!__atomic_compare_exchange_n(&rdc->refcount, &expected, wanted, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
106 +
107 + return true;
108 +}
109 +
110 +struct rrd_collector *rrd_collector_acquire_current_thread(void) {
111 + rrd_collector_started();
112 +
113 + if(!rrd_collector_acquire(thread_rrd_collector))
114 + internal_fatal(true, "FUNCTIONS: Trying to acquire a the current thread collector, that is currently exiting.");
115 +
116 + return thread_rrd_collector;
117 +}
118 +
119 +void rrd_collector_release(struct rrd_collector *rdc) {
120 + if(unlikely(!rdc)) return;
121 +
122 + int32_t expected = __atomic_load_n(&rdc->refcount, __ATOMIC_RELAXED), wanted = 0;
123 + do {
124 + if(expected < 0)
125 + return;
126 +
127 + if(expected == 0) {
128 + internal_fatal(true, "FUNCTIONS: Trying to release a collector that is not acquired.");
129 + return;
130 + }
131 +
132 + wanted = expected - 1;
133 + } while(!__atomic_compare_exchange_n(&rdc->refcount, &expected, wanted, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED));
134 +
135 + if(wanted == 0)
136 + rrd_collector_free(rdc);
137 +}
database/rrdcollector.h new
+30
@@ -0,0 +1,30 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDCOLLECTOR_H
4 +#define NETDATA_RRDCOLLECTOR_H
5 +
6 +#include "rrd.h"
7 +
8 +#ifdef NETDATA_RRDCOLLECTOR_INTERNALS
9 +
10 +// ----------------------------------------------------------------------------
11 +// private API
12 +
13 +struct rrd_collector;
14 +struct rrd_collector *rrd_collector_acquire_current_thread(void);
15 +void rrd_collector_release(struct rrd_collector *rdc);
16 +extern __thread struct rrd_collector *thread_rrd_collector;
17 +bool rrd_collector_running(struct rrd_collector *rdc);
18 +pid_t rrd_collector_tid(struct rrd_collector *rdc);
19 +bool rrd_collector_dispatcher_acquire(struct rrd_collector *rdc);
20 +void rrd_collector_dispatcher_release(struct rrd_collector *rdc);
21 +
22 +#endif // NETDATA_RRDCOLLECTOR_INTERNALS
23 +
24 +// ----------------------------------------------------------------------------
25 +// public API
26 +
27 +void rrd_collector_started(void);
28 +void rrd_collector_finished(void);
29 +
30 +#endif //NETDATA_RRDCOLLECTOR_H
database/rrdfunctions.c
+417 -522
@@ -1,266 +1,68 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2 #define NETDATA_RRD_INTERNALS
3 +#define NETDATA_RRDCOLLECTOR_INTERNALS
4 +
5 #include "rrd.h"
6
7 #define MAX_FUNCTION_LENGTH (PLUGINSD_LINE_MAX - 512) // we need some space for the rest of the line
8
9 static unsigned char functions_allowed_chars[256] = {
8 - [0] = '\0', //
9 - [1] = '_', //
10 - [2] = '_', //
11 - [3] = '_', //
12 - [4] = '_', //
13 - [5] = '_', //
14 - [6] = '_', //
15 - [7] = '_', //
16 - [8] = '_', //
17 - [9] = ' ', // Horizontal Tab
18 - [10] = ' ', // Line Feed
19 - [11] = ' ', // Vertical Tab
20 - [12] = ' ', // Form Feed
21 - [13] = ' ', // Carriage Return
22 - [14] = '_', //
23 - [15] = '_', //
24 - [16] = '_', //
25 - [17] = '_', //
26 - [18] = '_', //
27 - [19] = '_', //
28 - [20] = '_', //
29 - [21] = '_', //
30 - [22] = '_', //
31 - [23] = '_', //
32 - [24] = '_', //
33 - [25] = '_', //
34 - [26] = '_', //
35 - [27] = '_', //
36 - [28] = '_', //
37 - [29] = '_', //
38 - [30] = '_', //
39 - [31] = '_', //
40 - [32] = ' ', // SPACE keep
41 - [33] = '!', // ! keep
42 - [34] = '"', // " keep
43 - [35] = '#', // # keep
44 - [36] = '$', // $ keep
45 - [37] = '%', // % keep
46 - [38] = '&', // & keep
47 - [39] = '\'', // ' keep
48 - [40] = '(', // ( keep
49 - [41] = ')', // ) keep
50 - [42] = '*', // * keep
51 - [43] = '+', // + keep
52 - [44] = ',', // , keep
53 - [45] = '-', // - keep
54 - [46] = '.', // . keep
55 - [47] = '/', // / keep
56 - [48] = '0', // 0 keep
57 - [49] = '1', // 1 keep
58 - [50] = '2', // 2 keep
59 - [51] = '3', // 3 keep
60 - [52] = '4', // 4 keep
61 - [53] = '5', // 5 keep
62 - [54] = '6', // 6 keep
63 - [55] = '7', // 7 keep
64 - [56] = '8', // 8 keep
65 - [57] = '9', // 9 keep
66 - [58] = ':', // : keep
67 - [59] = ';', // ; keep
68 - [60] = '<', // < keep
69 - [61] = '=', // = keep
70 - [62] = '>', // > keep
71 - [63] = '?', // ? keep
72 - [64] = '@', // @ keep
73 - [65] = 'A', // A keep
74 - [66] = 'B', // B keep
75 - [67] = 'C', // C keep
76 - [68] = 'D', // D keep
77 - [69] = 'E', // E keep
78 - [70] = 'F', // F keep
79 - [71] = 'G', // G keep
80 - [72] = 'H', // H keep
81 - [73] = 'I', // I keep
82 - [74] = 'J', // J keep
83 - [75] = 'K', // K keep
84 - [76] = 'L', // L keep
85 - [77] = 'M', // M keep
86 - [78] = 'N', // N keep
87 - [79] = 'O', // O keep
88 - [80] = 'P', // P keep
89 - [81] = 'Q', // Q keep
90 - [82] = 'R', // R keep
91 - [83] = 'S', // S keep
92 - [84] = 'T', // T keep
93 - [85] = 'U', // U keep
94 - [86] = 'V', // V keep
95 - [87] = 'W', // W keep
96 - [88] = 'X', // X keep
97 - [89] = 'Y', // Y keep
98 - [90] = 'Z', // Z keep
99 - [91] = '[', // [ keep
100 - [92] = '\\', // backslash keep
101 - [93] = ']', // ] keep
102 - [94] = '^', // ^ keep
103 - [95] = '_', // _ keep
104 - [96] = '`', // ` keep
105 - [97] = 'a', // a keep
106 - [98] = 'b', // b keep
107 - [99] = 'c', // c keep
108 - [100] = 'd', // d keep
109 - [101] = 'e', // e keep
110 - [102] = 'f', // f keep
111 - [103] = 'g', // g keep
112 - [104] = 'h', // h keep
113 - [105] = 'i', // i keep
114 - [106] = 'j', // j keep
115 - [107] = 'k', // k keep
116 - [108] = 'l', // l keep
117 - [109] = 'm', // m keep
118 - [110] = 'n', // n keep
119 - [111] = 'o', // o keep
120 - [112] = 'p', // p keep
121 - [113] = 'q', // q keep
122 - [114] = 'r', // r keep
123 - [115] = 's', // s keep
124 - [116] = 't', // t keep
125 - [117] = 'u', // u keep
126 - [118] = 'v', // v keep
127 - [119] = 'w', // w keep
128 - [120] = 'x', // x keep
129 - [121] = 'y', // y keep
130 - [122] = 'z', // z keep
131 - [123] = '{', // { keep
132 - [124] = '|', // | keep
133 - [125] = '}', // } keep
134 - [126] = '~', // ~ keep
135 - [127] = '_', //
136 - [128] = '_', //
137 - [129] = '_', //
138 - [130] = '_', //
139 - [131] = '_', //
140 - [132] = '_', //
141 - [133] = '_', //
142 - [134] = '_', //
143 - [135] = '_', //
144 - [136] = '_', //
145 - [137] = '_', //
146 - [138] = '_', //
147 - [139] = '_', //
148 - [140] = '_', //
149 - [141] = '_', //
150 - [142] = '_', //
151 - [143] = '_', //
152 - [144] = '_', //
153 - [145] = '_', //
154 - [146] = '_', //
155 - [147] = '_', //
156 - [148] = '_', //
157 - [149] = '_', //
158 - [150] = '_', //
159 - [151] = '_', //
160 - [152] = '_', //
161 - [153] = '_', //
162 - [154] = '_', //
163 - [155] = '_', //
164 - [156] = '_', //
165 - [157] = '_', //
166 - [158] = '_', //
167 - [159] = '_', //
168 - [160] = '_', //
169 - [161] = '_', //
170 - [162] = '_', //
171 - [163] = '_', //
172 - [164] = '_', //
173 - [165] = '_', //
174 - [166] = '_', //
175 - [167] = '_', //
176 - [168] = '_', //
177 - [169] = '_', //
178 - [170] = '_', //
179 - [171] = '_', //
180 - [172] = '_', //
181 - [173] = '_', //
182 - [174] = '_', //
183 - [175] = '_', //
184 - [176] = '_', //
185 - [177] = '_', //
186 - [178] = '_', //
187 - [179] = '_', //
188 - [180] = '_', //
189 - [181] = '_', //
190 - [182] = '_', //
191 - [183] = '_', //
192 - [184] = '_', //
193 - [185] = '_', //
194 - [186] = '_', //
195 - [187] = '_', //
196 - [188] = '_', //
197 - [189] = '_', //
198 - [190] = '_', //
199 - [191] = '_', //
200 - [192] = '_', //
201 - [193] = '_', //
202 - [194] = '_', //
203 - [195] = '_', //
204 - [196] = '_', //
205 - [197] = '_', //
206 - [198] = '_', //
207 - [199] = '_', //
208 - [200] = '_', //
209 - [201] = '_', //
210 - [202] = '_', //
211 - [203] = '_', //
212 - [204] = '_', //
213 - [205] = '_', //
214 - [206] = '_', //
215 - [207] = '_', //
216 - [208] = '_', //
217 - [209] = '_', //
218 - [210] = '_', //
219 - [211] = '_', //
220 - [212] = '_', //
221 - [213] = '_', //
222 - [214] = '_', //
223 - [215] = '_', //
224 - [216] = '_', //
225 - [217] = '_', //
226 - [218] = '_', //
227 - [219] = '_', //
228 - [220] = '_', //
229 - [221] = '_', //
230 - [222] = '_', //
231 - [223] = '_', //
232 - [224] = '_', //
233 - [225] = '_', //
234 - [226] = '_', //
235 - [227] = '_', //
236 - [228] = '_', //
237 - [229] = '_', //
238 - [230] = '_', //
239 - [231] = '_', //
240 - [232] = '_', //
241 - [233] = '_', //
242 - [234] = '_', //
243 - [235] = '_', //
244 - [236] = '_', //
245 - [237] = '_', //
246 - [238] = '_', //
247 - [239] = '_', //
248 - [240] = '_', //
249 - [241] = '_', //
250 - [242] = '_', //
251 - [243] = '_', //
252 - [244] = '_', //
253 - [245] = '_', //
254 - [246] = '_', //
255 - [247] = '_', //
256 - [248] = '_', //
257 - [249] = '_', //
258 - [250] = '_', //
259 - [251] = '_', //
260 - [252] = '_', //
261 - [253] = '_', //
262 - [254] = '_', //
263 - [255] = '_' //
10 + [0] = '\0', [1] = '_', [2] = '_', [3] = '_', [4] = '_', [5] = '_', [6] = '_', [7] = '_', [8] = '_',
11 +
12 + // control
13 + ['\t'] = ' ', ['\n'] = ' ', ['\v'] = ' ', [12] = ' ', ['\r'] = ' ',
14 +
15 + [14] = '_', [15] = '_', [16] = '_', [17] = '_', [18] = '_', [19] = '_', [20] = '_', [21] = '_',
16 + [22] = '_', [23] = '_', [24] = '_', [25] = '_', [26] = '_', [27] = '_', [28] = '_', [29] = '_',
17 + [30] = '_', [31] = '_',
18 +
19 + // symbols
20 + [' '] = ' ', ['!'] = '!', ['"'] = '"', ['#'] = '#', ['$'] = '$', ['%'] = '%', ['&'] = '&', ['\''] = '\'',
21 + ['('] = '(', [')'] = ')', ['*'] = '*', ['+'] = '+', [','] = ',', ['-'] = '-', ['.'] = '.', ['/'] = '/',
22 +
23 + // numbers
24 + ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7',
25 + ['8'] = '8', ['9'] = '9',
26 +
27 + // symbols
28 + [':'] = ':', [';'] = ';', ['<'] = '<', ['='] = '=', ['>'] = '>', ['?'] = '?', ['@'] = '@',
29 +
30 + // capitals
31 + ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H',
32 + ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P',
33 + ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X',
34 + ['Y'] = 'Y', ['Z'] = 'Z',
35 +
36 + // symbols
37 + ['['] = '[', ['\\'] = '\\', [']'] = ']', ['^'] = '^', ['_'] = '_', ['`'] = '`',
38 +
39 + // lower
40 + ['a'] = 'a', ['b'] = 'b', ['c'] = 'c', ['d'] = 'd', ['e'] = 'e', ['f'] = 'f', ['g'] = 'g', ['h'] = 'h',
41 + ['i'] = 'i', ['j'] = 'j', ['k'] = 'k', ['l'] = 'l', ['m'] = 'm', ['n'] = 'n', ['o'] = 'o', ['p'] = 'p',
42 + ['q'] = 'q', ['r'] = 'r', ['s'] = 's', ['t'] = 't', ['u'] = 'u', ['v'] = 'v', ['w'] = 'w', ['x'] = 'x',
43 + ['y'] = 'y', ['z'] = 'z',
44 +
45 + // symbols
46 + ['{'] = '{', ['|'] = '|', ['}'] = '}', ['~'] = '~',
47 +
48 + // rest
49 + [127] = '_', [128] = '_', [129] = '_', [130] = '_', [131] = '_', [132] = '_', [133] = '_', [134] = '_',
50 + [135] = '_', [136] = '_', [137] = '_', [138] = '_', [139] = '_', [140] = '_', [141] = '_', [142] = '_',
51 + [143] = '_', [144] = '_', [145] = '_', [146] = '_', [147] = '_', [148] = '_', [149] = '_', [150] = '_',
52 + [151] = '_', [152] = '_', [153] = '_', [154] = '_', [155] = '_', [156] = '_', [157] = '_', [158] = '_',
53 + [159] = '_', [160] = '_', [161] = '_', [162] = '_', [163] = '_', [164] = '_', [165] = '_', [166] = '_',
54 + [167] = '_', [168] = '_', [169] = '_', [170] = '_', [171] = '_', [172] = '_', [173] = '_', [174] = '_',
55 + [175] = '_', [176] = '_', [177] = '_', [178] = '_', [179] = '_', [180] = '_', [181] = '_', [182] = '_',
56 + [183] = '_', [184] = '_', [185] = '_', [186] = '_', [187] = '_', [188] = '_', [189] = '_', [190] = '_',
57 + [191] = '_', [192] = '_', [193] = '_', [194] = '_', [195] = '_', [196] = '_', [197] = '_', [198] = '_',
58 + [199] = '_', [200] = '_', [201] = '_', [202] = '_', [203] = '_', [204] = '_', [205] = '_', [206] = '_',
59 + [207] = '_', [208] = '_', [209] = '_', [210] = '_', [211] = '_', [212] = '_', [213] = '_', [214] = '_',
60 + [215] = '_', [216] = '_', [217] = '_', [218] = '_', [219] = '_', [220] = '_', [221] = '_', [222] = '_',
61 + [223] = '_', [224] = '_', [225] = '_', [226] = '_', [227] = '_', [228] = '_', [229] = '_', [230] = '_',
62 + [231] = '_', [232] = '_', [233] = '_', [234] = '_', [235] = '_', [236] = '_', [237] = '_', [238] = '_',
63 + [239] = '_', [240] = '_', [241] = '_', [242] = '_', [243] = '_', [244] = '_', [245] = '_', [246] = '_',
64 + [247] = '_', [248] = '_', [249] = '_', [250] = '_', [251] = '_', [252] = '_', [253] = '_', [254] = '_',
65 + [255] = '_'
66 };
67
68 static inline size_t sanitize_function_text(char *dst, const char *src, size_t dst_len) {
@@ -278,127 +80,96 @@ typedef enum __attribute__((packed)) {
80 // this is 8-bit
81 } RRD_FUNCTION_OPTIONS;
82
83 +// ----------------------------------------------------------------------------
84 +
85 struct rrd_host_function {
86 bool sync; // when true, the function is called synchronously
87 RRD_FUNCTION_OPTIONS options; // RRD_FUNCTION_OPTIONS
88 + HTTP_ACCESS access;
89 STRING *help;
90 + STRING *tags;
91 int timeout; // the default timeout of the function
92 + int priority;
93
94 rrd_function_execute_cb_t execute_cb;
288 -
95 void *execute_cb_data;
290 - struct rrd_collector *collector;
291 -};
96
293 -// Each function points to this collector structure
294 -// so that when the collector exits, all of them will
295 -// be invalidated (running == false)
296 -// The last function that is using this collector
297 -// frees the structure too (or when the collector calls
298 -// rrdset_collector_finished()).
299 -
300 -struct rrd_collector {
301 - int32_t refcount;
302 - int32_t refcount_canceller;
303 - pid_t tid;
304 - bool running;
97 + struct rrd_collector *collector;
98 };
99
307 -// Each thread that adds RRDSET functions, has to call
308 -// rrdset_collector_started() and rrdset_collector_finished()
309 -// to create the collector structure.
310 -
311 -static __thread struct rrd_collector *thread_rrd_collector = NULL;
312 -
313 -static void rrd_collector_free(struct rrd_collector *rdc) {
314 - if(rdc->running)
315 - return;
316 -
317 - int32_t expected = 0;
318 - if(!__atomic_compare_exchange_n(&rdc->refcount, &expected, -1, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
319 - // the collector is still referenced by charts.
320 - // leave it hanging there, the last chart will actually free it.
321 - return;
322 - }
323 -
324 - // we can free it now
325 - freez(rdc);
326 -}
327 -
328 -// called once per collector
329 -void rrd_collector_started(void) {
330 - if(!thread_rrd_collector)
331 - thread_rrd_collector = callocz(1, sizeof(struct rrd_collector));
332 -
333 - thread_rrd_collector->tid = gettid();
334 - __atomic_store_n(&thread_rrd_collector->running, true, __ATOMIC_RELAXED);
335 -}
336 -
337 -// called once per collector
338 -void rrd_collector_finished(void) {
339 - if(!thread_rrd_collector)
340 - return;
341 -
342 - __atomic_store_n(&thread_rrd_collector->running, false, __ATOMIC_RELAXED);
343 -
344 - int32_t expected = 0;
345 - while(!__atomic_compare_exchange_n(&thread_rrd_collector->refcount_canceller, &expected, -1, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
346 - expected = 0;
347 - sleep_usec(1 * USEC_PER_MS);
348 - }
349 -
350 - rrd_collector_free(thread_rrd_collector);
351 - thread_rrd_collector = NULL;
352 -}
100 +struct rrd_function_inflight {
101 + bool used;
102
354 -#define rrd_collector_running(c) __atomic_load_n(&(c)->running, __ATOMIC_RELAXED)
103 + RRDHOST *host;
104 + uuid_t transaction_uuid;
105 + const char *transaction;
106 + const char *cmd;
107 + const char *sanitized_cmd;
108 + size_t sanitized_cmd_length;
109 + int timeout;
110 + bool cancelled;
111 + usec_t stop_monotonic_ut;
112
356 -static struct rrd_collector *rrd_collector_acquire(void) {
357 - rrd_collector_started();
113 + const DICTIONARY_ITEM *host_function_acquired;
114
359 - int32_t expected = __atomic_load_n(&thread_rrd_collector->refcount, __ATOMIC_RELAXED), wanted = 0;
360 - do {
361 - if(expected < 0 || !rrd_collector_running(thread_rrd_collector)) {
362 - internal_fatal(true, "FUNCTIONS: Trying to acquire a collector that is exiting.");
363 - return thread_rrd_collector;
364 - }
115 + // the collector
116 + // we acquire this structure at the beginning,
117 + // and we release it at the end
118 + struct rrd_host_function *rdcf;
119
366 - wanted = expected + 1;
120 + struct {
121 + BUFFER *wb;
122
368 - } while(!__atomic_compare_exchange_n(&thread_rrd_collector->refcount, &expected, wanted, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
123 + // in async mode,
124 + // the function to call to send the result back
125 + rrd_function_result_callback_t cb;
126 + void *data;
127 + } result;
128
370 - return thread_rrd_collector;
371 -}
129 + struct {
130 + // to be called in sync mode
131 + // while the function is running
132 + // to check if the function has been canceled
133 + rrd_function_is_cancelled_cb_t cb;
134 + void *data;
135 + } is_cancelled;
136
373 -static void rrd_collector_release(struct rrd_collector *rdc) {
374 - if(unlikely(!rdc)) return;
137 + struct {
138 + // to be registered by the function itself
139 + // used to signal the function to cancel
140 + rrd_function_cancel_cb_t cb;
141 + void *data;
142 + } canceller;
143
376 - int32_t expected = __atomic_load_n(&rdc->refcount, __ATOMIC_RELAXED), wanted = 0;
377 - do {
378 - if(expected < 0) {
379 - internal_fatal(true, "FUNCTIONS: Trying to release a collector that is exiting.");
380 - return;
381 - }
144 + struct {
145 + // callback to receive progress reports from function
146 + rrd_function_progress_cb_t cb;
147 + void *data;
148 + } progress;
149
383 - if(expected == 0) {
384 - internal_fatal(true, "FUNCTIONS: Trying to release a collector that is not acquired.");
385 - return;
386 - }
150 + struct {
151 + // to be registered by the function itself
152 + // used to send progress requests to function
153 + rrd_function_progresser_cb_t cb;
154 + void *data;
155 + } progresser;
156 +};
157
388 - wanted = expected - 1;
158 +static DICTIONARY *rrd_functions_inflight_requests = NULL;
159
390 - } while(!__atomic_compare_exchange_n(&rdc->refcount, &expected, wanted, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED));
160 +static void rrd_function_cancel_inflight(struct rrd_function_inflight *r);
161
392 - if(wanted == 0)
393 - rrd_collector_free(rdc);
394 -}
162 +// ----------------------------------------------------------------------------
163
164 static void rrd_functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *rrdhost) {
165 RRDHOST *host = rrdhost; (void)host;
166 struct rrd_host_function *rdcf = func;
167
168 rrd_collector_started();
401 - rdcf->collector = rrd_collector_acquire();
169 + rdcf->collector = rrd_collector_acquire_current_thread();
170 +
171 + if(!rdcf->priority)
172 + rdcf->priority = RRDFUNCTIONS_PRIORITY_DEFAULT;
173
174 // internal_error(true, "FUNCTIONS: adding function '%s' on host '%s', collection tid %d, %s",
175 // dictionary_acquired_item_name(item), rrdhost_hostname(host),
@@ -422,26 +193,30 @@ static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_
193 bool changed = false;
194
195 if(rdcf->collector != thread_rrd_collector) {
425 - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed collector from %d to %d",
426 - dictionary_acquired_item_name(item), rrdhost_hostname(host), rdcf->collector->tid, thread_rrd_collector->tid);
196 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
197 + "FUNCTIONS: function '%s' of host '%s' changed collector from %d to %d",
198 + dictionary_acquired_item_name(item), rrdhost_hostname(host),
199 + rrd_collector_tid(rdcf->collector), rrd_collector_tid(thread_rrd_collector));
200
201 struct rrd_collector *old_rdc = rdcf->collector;
429 - rdcf->collector = rrd_collector_acquire();
202 + rdcf->collector = rrd_collector_acquire_current_thread();
203 rrd_collector_release(old_rdc);
204 changed = true;
205 }
206
207 if(rdcf->execute_cb != new_rdcf->execute_cb) {
435 - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed execute callback",
436 - dictionary_acquired_item_name(item), rrdhost_hostname(host));
208 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
209 + "FUNCTIONS: function '%s' of host '%s' changed execute callback",
210 + dictionary_acquired_item_name(item), rrdhost_hostname(host));
211
212 rdcf->execute_cb = new_rdcf->execute_cb;
213 changed = true;
214 }
215
216 if(rdcf->help != new_rdcf->help) {
443 - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed help text",
444 - dictionary_acquired_item_name(item), rrdhost_hostname(host));
217 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
218 + "FUNCTIONS: function '%s' of host '%s' changed help text",
219 + dictionary_acquired_item_name(item), rrdhost_hostname(host));
220
221 STRING *old = rdcf->help;
222 rdcf->help = new_rdcf->help;
@@ -451,25 +226,59 @@ static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_
226 else
227 string_freez(new_rdcf->help);
228
229 + if(rdcf->tags != new_rdcf->tags) {
230 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
231 + "FUNCTIONS: function '%s' of host '%s' changed tags",
232 + dictionary_acquired_item_name(item), rrdhost_hostname(host));
233 +
234 + STRING *old = rdcf->tags;
235 + rdcf->tags = new_rdcf->tags;
236 + string_freez(old);
237 + changed = true;
238 + }
239 + else
240 + string_freez(new_rdcf->tags);
241 +
242 if(rdcf->timeout != new_rdcf->timeout) {
455 - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed timeout",
456 - dictionary_acquired_item_name(item), rrdhost_hostname(host));
243 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
244 + "FUNCTIONS: function '%s' of host '%s' changed timeout",
245 + dictionary_acquired_item_name(item), rrdhost_hostname(host));
246
247 rdcf->timeout = new_rdcf->timeout;
248 changed = true;
249 }
250
251 + if(rdcf->priority != new_rdcf->priority) {
252 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
253 + "FUNCTIONS: function '%s' of host '%s' changed priority",
254 + dictionary_acquired_item_name(item), rrdhost_hostname(host));
255 +
256 + rdcf->priority = new_rdcf->priority;
257 + changed = true;
258 + }
259 +
260 + if(rdcf->access != new_rdcf->access) {
261 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
262 + "FUNCTIONS: function '%s' of host '%s' changed access level",
263 + dictionary_acquired_item_name(item), rrdhost_hostname(host));
264 +
265 + rdcf->access = new_rdcf->access;
266 + changed = true;
267 + }
268 +
269 if(rdcf->sync != new_rdcf->sync) {
463 - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed sync/async mode",
464 - dictionary_acquired_item_name(item), rrdhost_hostname(host));
270 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
271 + "FUNCTIONS: function '%s' of host '%s' changed sync/async mode",
272 + dictionary_acquired_item_name(item), rrdhost_hostname(host));
273
274 rdcf->sync = new_rdcf->sync;
275 changed = true;
276 }
277
278 if(rdcf->execute_cb_data != new_rdcf->execute_cb_data) {
471 - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed execute callback data",
472 - dictionary_acquired_item_name(item), rrdhost_hostname(host));
279 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
280 + "FUNCTIONS: function '%s' of host '%s' changed execute callback data",
281 + dictionary_acquired_item_name(item), rrdhost_hostname(host));
282
283 rdcf->execute_cb_data = new_rdcf->execute_cb_data;
284 changed = true;
@@ -497,12 +306,22 @@ void rrdfunctions_host_destroy(RRDHOST *host) {
306 dictionary_destroy(host->functions);
307 }
308
500 -void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, const char *help,
501 - bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data) {
309 +// ----------------------------------------------------------------------------
310 +
311 +void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority, const char *help, const char *tags,
312 + HTTP_ACCESS access, bool sync, rrd_function_execute_cb_t execute_cb,
313 + void *execute_cb_data) {
314
315 // RRDSET *st may be NULL in this function
316 // to create a GLOBAL function
317
318 + if(!tags || !*tags) {
319 + if(strcmp(name, "systemd-journal") == 0)
320 + tags = "logs";
321 + else
322 + tags = "top";
323 + }
324 +
325 if(st && !st->functions_view)
326 st->functions_view = dictionary_create_view(host->functions);
327
@@ -513,9 +332,12 @@ void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout,
332 .sync = sync,
333 .timeout = timeout,
334 .options = (st)?RRD_FUNCTION_LOCAL:RRD_FUNCTION_GLOBAL,
335 + .access = access,
336 .execute_cb = execute_cb,
337 .execute_cb_data = execute_cb_data,
338 .help = string_strdupz(help),
339 + .tags = string_strdupz(tags),
340 + .priority = priority,
341 };
342 const DICTIONARY_ITEM *item = dictionary_set_and_acquire_item(host->functions, key, &tmp, sizeof(tmp));
343
@@ -534,10 +356,13 @@ void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb) {
356 struct rrd_host_function *tmp;
357 dfe_start_read(st->functions_view, tmp) {
358 buffer_sprintf(wb
537 - , PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\"\n"
359 + , PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\" \"%s\" \"%s\" %d\n"
360 , tmp_dfe.name
361 , tmp->timeout
362 , string2str(tmp->help)
363 + , string2str(tmp->tags)
364 + , http_id2access(tmp->access)
365 + , tmp->priority
366 );
367 }
368 dfe_done(tmp);
@@ -552,10 +377,13 @@ void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb) {
377 continue;
378
379 buffer_sprintf(wb
555 - , PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n"
380 + , PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"%s\" \"%s\" %d\n"
381 , tmp_dfe.name
382 , tmp->timeout
383 , string2str(tmp->help)
384 + , string2str(tmp->tags)
385 + , http_id2access(tmp->access)
386 + , tmp->priority
387 );
388 }
389 dfe_done(tmp);
@@ -632,7 +460,7 @@ static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, s
460 }
461 }
462
635 - // if s == NULL, set it to the end of the buffer
463 + // if s == NULL, set it to the end of the buffer;
464 // this should happen only the first time
465 if (unlikely(!s))
466 s = &buffer[key_length - 1];
@@ -663,51 +491,6 @@ static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, s
491
492 // ----------------------------------------------------------------------------
493
666 -struct rrd_function_inflight {
667 - bool used;
668 -
669 - RRDHOST *host;
670 - const char *transaction;
671 - const char *cmd;
672 - const char *sanitized_cmd;
673 - size_t sanitized_cmd_length;
674 - int timeout;
675 - bool cancelled;
676 -
677 - const DICTIONARY_ITEM *host_function_acquired;
678 -
679 - // the collector
680 - // we acquire this structure at the beginning,
681 - // and we release it at the end
682 - struct rrd_host_function *rdcf;
683 -
684 - struct {
685 - BUFFER *wb;
686 -
687 - // in async mode,
688 - // the function to call to send the result back
689 - rrd_function_result_callback_t cb;
690 - void *data;
691 - } result;
692 -
693 - struct {
694 - // to be called in sync mode
695 - // while the function is running
696 - // to check if the function has been cancelled
697 - rrd_function_is_cancelled_cb_t cb;
698 - void *data;
699 - } is_cancelled;
700 -
701 - struct {
702 - // to be registered by the function itself
703 - // used to signal the function to cancel
704 - rrd_function_canceller_cb_t cb;
705 - void *data;
706 - } canceller;
707 -};
708 -
709 -static DICTIONARY *rrd_functions_inflight_requests = NULL;
710 -
494 static void rrd_functions_inflight_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
495 struct rrd_function_inflight *r = value;
496
@@ -736,12 +519,18 @@ void rrd_functions_inflight_destroy(void) {
519 rrd_functions_inflight_requests = NULL;
520 }
521
739 -static void rrd_inflight_async_function_register_canceller_cb(void *register_canceller_cb_data, rrd_function_canceller_cb_t canceller_cb, void *canceller_cb_data) {
522 +static void rrd_inflight_async_function_register_canceller_cb(void *register_canceller_cb_data, rrd_function_cancel_cb_t canceller_cb, void *canceller_cb_data) {
523 struct rrd_function_inflight *r = register_canceller_cb_data;
524 r->canceller.cb = canceller_cb;
525 r->canceller.data = canceller_cb_data;
526 }
527
528 +static void rrd_inflight_async_function_register_progresser_cb(void *register_progresser_cb_data, rrd_function_progresser_cb_t progresser_cb, void *progresser_cb_data) {
529 + struct rrd_function_inflight *r = register_progresser_cb_data;
530 + r->progresser.cb = progresser_cb;
531 + r->progresser.data = progresser_cb_data;
532 +}
533 +
534 // ----------------------------------------------------------------------------
535 // waiting for async function completion
536
@@ -758,14 +547,13 @@ struct rrd_function_call_wait {
547 };
548
549 static void rrd_inflight_function_cleanup(RRDHOST *host __maybe_unused,
761 - const DICTIONARY_ITEM *host_function_acquired __maybe_unused,
550 const char *transaction) {
551 dictionary_del(rrd_functions_inflight_requests, transaction);
552 dictionary_garbage_collect(rrd_functions_inflight_requests);
553 }
554
555 static void rrd_function_call_wait_free(struct rrd_function_call_wait *tmp) {
768 - rrd_inflight_function_cleanup(tmp->host, tmp->host_function_acquired, tmp->transaction);
556 + rrd_inflight_function_cleanup(tmp->host, tmp->transaction);
557 freez(tmp->transaction);
558
559 pthread_cond_destroy(&tmp->cond);
@@ -805,7 +593,7 @@ static void rrd_inflight_async_function_nowait_finished(BUFFER *wb, int code, vo
593 if(r->result.cb)
594 r->result.cb(wb, code, r->result.data);
595
808 - rrd_inflight_function_cleanup(r->host, r->host_function_acquired, r->transaction);
596 + rrd_inflight_function_cleanup(r->host, r->transaction);
597 }
598
599 static bool rrd_inflight_async_function_is_cancelled(void *data) {
@@ -814,27 +602,25 @@ static bool rrd_inflight_async_function_is_cancelled(void *data) {
602 }
603
604 static inline int rrd_call_function_async_and_dont_wait(struct rrd_function_inflight *r) {
817 - int code = r->rdcf->execute_cb(r->result.wb, r->timeout, r->sanitized_cmd, r->rdcf->execute_cb_data,
605 + int code = r->rdcf->execute_cb(&r->transaction_uuid, r->result.wb,
606 + &r->stop_monotonic_ut, r->sanitized_cmd, r->rdcf->execute_cb_data,
607 rrd_inflight_async_function_nowait_finished, r,
608 + r->progress.cb, r->progress.data,
609 rrd_inflight_async_function_is_cancelled, r,
820 - rrd_inflight_async_function_register_canceller_cb, r);
610 + rrd_inflight_async_function_register_canceller_cb, r,
611 + rrd_inflight_async_function_register_progresser_cb, r);
612
613 if(code != HTTP_RESP_OK) {
614 if (!buffer_strlen(r->result.wb))
615 rrd_call_function_error(r->result.wb, "Failed to send request to the collector.", code);
616
826 - rrd_inflight_function_cleanup(r->host, r->host_function_acquired, r->transaction);
617 + rrd_inflight_function_cleanup(r->host, r->transaction);
618 }
619
620 return code;
621 }
622
623 static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) {
833 - struct timespec tp;
834 - clock_gettime(CLOCK_REALTIME, &tp);
835 - usec_t now_ut = tp.tv_sec * USEC_PER_SEC + tp.tv_nsec / NSEC_PER_USEC;
836 - usec_t end_ut = now_ut + r->timeout * USEC_PER_SEC + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
837 -
624 struct rrd_function_call_wait *tmp = mallocz(sizeof(struct rrd_function_call_wait));
625 tmp->free_with_signal = false;
626 tmp->data_are_ready = false;
@@ -844,19 +630,22 @@ static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) {
630 netdata_mutex_init(&tmp->mutex);
631 pthread_cond_init(&tmp->cond, NULL);
632
847 - // we need a temporary BUFFER, because we may time out and the caller supplied one may vanish
848 - // so, we create a new one we guarantee will survive until the collector finishes...
633 + // we need a temporary BUFFER, because we may time out and the caller supplied one may vanish,
634 + // so we create a new one we guarantee will survive until the collector finishes...
635
636 bool we_should_free = true;
637 BUFFER *temp_wb = buffer_create(PLUGINSD_LINE_MAX + 1, &netdata_buffers_statistics.buffers_functions); // we need it because we may give up on it
638 temp_wb->content_type = r->result.wb->content_type;
639
854 - int code = r->rdcf->execute_cb(temp_wb, r->timeout, r->sanitized_cmd, r->rdcf->execute_cb_data,
855 - // we overwrite the result callbacks,
856 - // so that we can clean up the allocations made
640 + int code = r->rdcf->execute_cb(&r->transaction_uuid, temp_wb, &r->stop_monotonic_ut,
641 + r->sanitized_cmd, r->rdcf->execute_cb_data,
642 + // we overwrite the result callbacks,
643 + // so that we can clean up the allocations made
644 rrd_async_function_signal_when_ready, tmp,
645 + r->progress.cb, r->progress.data,
646 rrd_inflight_async_function_is_cancelled, r,
859 - rrd_inflight_async_function_register_canceller_cb, r);
647 + rrd_inflight_async_function_register_canceller_cb, r,
648 + rrd_inflight_async_function_register_progresser_cb, r);
649
650 if (code == HTTP_RESP_OK) {
651 netdata_mutex_lock(&tmp->mutex);
@@ -864,14 +653,16 @@ static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) {
653 bool cancelled = false;
654 int rc = 0;
655 while (rc == 0 && !cancelled && !tmp->data_are_ready) {
867 - clock_gettime(CLOCK_REALTIME, &tp);
868 - now_ut = tp.tv_sec * USEC_PER_SEC + tp.tv_nsec / NSEC_PER_USEC;
869 -
870 - if(now_ut >= end_ut) {
656 + usec_t now_mono_ut = now_monotonic_usec();
657 + usec_t stop_mono_ut = __atomic_load_n(&r->stop_monotonic_ut, __ATOMIC_RELAXED) + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
658 + if(now_mono_ut > stop_mono_ut) {
659 rc = ETIMEDOUT;
660 break;
661 }
662
663 + // wait for 10ms, and loop again...
664 + struct timespec tp;
665 + clock_gettime(CLOCK_REALTIME, &tp);
666 tp.tv_nsec += 10 * NSEC_PER_MSEC;
667 if(tp.tv_nsec > (long)(1 * NSEC_PER_SEC)) {
668 tp.tv_sec++;
@@ -883,14 +674,15 @@ static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) {
674 // the mutex is again ours
675
676 if(rc == ETIMEDOUT) {
677 + // 10ms have passed
678 +
679 rc = 0;
680 if (!tmp->data_are_ready && r->is_cancelled.cb &&
681 r->is_cancelled.cb(r->is_cancelled.data)) {
682 // internal_error(true, "FUNCTIONS: transaction '%s' is cancelled while waiting for response",
683 // r->transaction);
891 - rc = 0;
684 cancelled = true;
893 - rrd_function_cancel(r->transaction);
685 + rrd_function_cancel_inflight(r);
686 break;
687 }
688 }
@@ -955,9 +747,10 @@ static inline int rrd_call_function_async(struct rrd_function_inflight *r, bool
747 void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data);
748 // ----------------------------------------------------------------------------
749
958 -int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *cmd,
750 +int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, HTTP_ACCESS access, const char *cmd,
751 bool wait, const char *transaction,
752 rrd_function_result_callback_t result_cb, void *result_cb_data,
753 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
754 rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload) {
755
756 int code;
@@ -980,42 +773,31 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *
773
774 struct rrd_host_function *rdcf = dictionary_acquired_item_value(host_function_acquired);
775
983 - if(timeout <= 0)
984 - timeout = rdcf->timeout;
776 + if(access != HTTP_ACCESS_ADMINS && rdcf->access != HTTP_ACCESS_ANY && access > rdcf->access) {
777 + rrd_call_function_error(result_wb, "This function requires a higher access level.", code);
778 + dictionary_acquired_item_release(host->functions, host_function_acquired);
779 + return HTTP_RESP_PRECOND_FAIL;
780 + }
781
782 + if(timeout_s <= 0)
783 + timeout_s = rdcf->timeout;
784
785 // ------------------------------------------------------------------------
988 - // the function can only be executed in sync mode
786 + // validate and parse the transaction, or generate a new transaction id
787
990 - if(rdcf->sync) {
991 - // the caller has to wait
992 -
993 - code = rdcf->execute_cb(result_wb, timeout, sanitized_cmd, rdcf->execute_cb_data,
994 - result_cb, result_cb_data,
995 - is_cancelled_cb, is_cancelled_cb_data, // it is ok to pass these, we block the caller
996 - NULL, NULL); // no need to pass, we will wait
997 -
998 - if (code != HTTP_RESP_OK && !buffer_strlen(result_wb))
999 - rrd_call_function_error(result_wb, "Collector reported error.", code);
788 + char uuid_str[UUID_COMPACT_STR_LEN];
789 + uuid_t uuid;
790
1001 - dictionary_acquired_item_release(host->functions, host_function_acquired);
1002 - return code;
1003 - }
791 + if(!transaction || !*transaction || uuid_parse_flexi(transaction, uuid) != 0)
792 + uuid_generate_random(uuid);
793
794 + uuid_unparse_lower_compact(uuid, uuid_str);
795 + transaction = uuid_str;
796
797 // ------------------------------------------------------------------------
798 // the function can only be executed in async mode
799 // put the function into the inflight requests
800
1010 - char uuid_str[UUID_COMPACT_STR_LEN];
1011 - if(!transaction) {
1012 - uuid_t uuid;
1013 - uuid_generate_random(uuid);
1014 - uuid_unparse_lower_compact(uuid, uuid_str);
1015 - transaction = uuid_str;
1016 - }
1017 -
1018 - // put the request into the inflight requests
801 struct rrd_function_inflight t = {
802 .used = false,
803 .host = host,
@@ -1023,8 +805,9 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *
805 .sanitized_cmd = strdupz(sanitized_cmd),
806 .sanitized_cmd_length = sanitized_cmd_length,
807 .transaction = strdupz(transaction),
1026 - .timeout = timeout,
808 + .timeout = timeout_s,
809 .cancelled = false,
810 + .stop_monotonic_ut = now_monotonic_usec() + timeout_s * USEC_PER_SEC,
811 .host_function_acquired = host_function_acquired,
812 .rdcf = rdcf,
813 .result = {
@@ -1035,11 +818,20 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *
818 .is_cancelled = {
819 .cb = is_cancelled_cb,
820 .data = is_cancelled_cb_data,
1038 - }
821 + },
822 + .progress = {
823 + .cb = progress_cb,
824 + .data = progress_cb_data,
825 + },
826 };
827 + uuid_copy(t.transaction_uuid, uuid);
828 +
829 struct rrd_function_inflight *r = dictionary_set(rrd_functions_inflight_requests, transaction, &t, sizeof(t));
830 if(r->used) {
1042 - netdata_log_info("FUNCTIONS: duplicate transaction '%s', function: '%s'", t.transaction, t.cmd);
831 + nd_log(NDLS_DAEMON, NDLP_NOTICE,
832 + "FUNCTIONS: duplicate transaction '%s', function: '%s'",
833 + t.transaction, t.cmd);
834 +
835 code = rrd_call_function_error(result_wb, "duplicate transaction", HTTP_RESP_BAD_REQUEST);
836 freez((void *)t.transaction);
837 freez((void *)t.cmd);
@@ -1050,51 +842,105 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *
842 r->used = true;
843 // internal_error(true, "FUNCTIONS: transaction '%s' started", r->transaction);
844
845 + if(r->rdcf->sync) {
846 + // the caller has to wait
847 + code = r->rdcf->execute_cb(&r->transaction_uuid, r->result.wb,
848 + &r->stop_monotonic_ut, r->sanitized_cmd, r->rdcf->execute_cb_data,
849 + r->result.cb, r->result.data,
850 + r->progress.cb, r->progress.data,
851 + r->is_cancelled.cb, r->is_cancelled.data, // it is ok to pass these, we block the caller
852 + NULL, NULL, // no need to register canceller, we will wait
853 + NULL, NULL // ?? do we need a progresser in this case?
854 + );
855 +
856 + if(code != HTTP_RESP_OK && !buffer_strlen(result_wb))
857 + rrd_call_function_error(result_wb, "Collector reported error.", code);
858 +
859 + rrd_inflight_function_cleanup(host, r->transaction);
860 + return code;
861 + }
862 +
863 return rrd_call_function_async(r, wait);
864 }
865
866 +static void rrd_function_cancel_inflight(struct rrd_function_inflight *r) {
867 + if(!r)
868 + return;
869 +
870 + bool cancelled = __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED);
871 + if(cancelled) {
872 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
873 + "FUNCTIONS: received a CANCEL request for transaction '%s', but it is already cancelled.",
874 + r->transaction);
875 + return;
876 + }
877 +
878 + __atomic_store_n(&r->cancelled, true, __ATOMIC_RELAXED);
879 +
880 + if(!rrd_collector_dispatcher_acquire(r->rdcf->collector)) {
881 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
882 + "FUNCTIONS: received a CANCEL request for transaction '%s', but the collector is not running.",
883 + r->transaction);
884 + return;
885 + }
886 +
887 + if(r->canceller.cb)
888 + r->canceller.cb(r->canceller.data);
889 +
890 + rrd_collector_dispatcher_release(r->rdcf->collector);
891 +}
892 +
893 void rrd_function_cancel(const char *transaction) {
894 // internal_error(true, "FUNCTIONS: request to cancel transaction '%s'", transaction);
895
896 const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction);
897 if(!item) {
1061 - netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but the transaction is not running.",
1062 - transaction);
898 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
899 + "FUNCTIONS: received a CANCEL request for transaction '%s', but the transaction is not running.",
900 + transaction);
901 return;
902 }
903
904 struct rrd_function_inflight *r = dictionary_acquired_item_value(item);
905 + rrd_function_cancel_inflight(r);
906 + dictionary_acquired_item_release(rrd_functions_inflight_requests, item);
907 +}
908
1068 - bool cancelled = __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED);
1069 - if(cancelled) {
1070 - netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but it is already cancelled.",
1071 - transaction);
1072 - goto cleanup;
909 +void rrd_function_progress(const char *transaction) {
910 + const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction);
911 + if(!item) {
912 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
913 + "FUNCTIONS: received a PROGRESS request for transaction '%s', but the transaction is not running.",
914 + transaction);
915 + return;
916 }
917
1075 - __atomic_store_n(&r->cancelled, true, __ATOMIC_RELAXED);
918 + struct rrd_function_inflight *r = dictionary_acquired_item_value(item);
919
1077 - int32_t expected = __atomic_load_n(&r->rdcf->collector->refcount_canceller, __ATOMIC_RELAXED);
1078 - int32_t wanted;
1079 - do {
1080 - if(expected < 0) {
1081 - netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but the collector is not running.",
1082 - transaction);
1083 - goto cleanup;
1084 - }
920 + if(!rrd_collector_dispatcher_acquire(r->rdcf->collector)) {
921 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
922 + "FUNCTIONS: received a PROGRESS request for transaction '%s', but the collector is not running.",
923 + transaction);
924 + goto cleanup;
925 + }
926
1086 - wanted = expected + 1;
1087 - } while(!__atomic_compare_exchange_n(&r->rdcf->collector->refcount_canceller, &expected, wanted, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
927 + functions_stop_monotonic_update_on_progress(&r->stop_monotonic_ut);
928
1089 - if(r->canceller.cb)
1090 - r->canceller.cb(r->canceller.data);
929 + if(r->progresser.cb)
930 + r->progresser.cb(r->progresser.data);
931
1092 - __atomic_sub_fetch(&r->rdcf->collector->refcount_canceller, 1, __ATOMIC_RELAXED);
932 + rrd_collector_dispatcher_release(r->rdcf->collector);
933
934 cleanup:
935 dictionary_acquired_item_release(rrd_functions_inflight_requests, item);
936 }
937
938 +void rrd_function_call_progresser(uuid_t *transaction) {
939 + char str[UUID_COMPACT_STR_LEN];
940 + uuid_unparse_lower_compact(*transaction, str);
941 + rrd_function_progress(str);
942 +}
943 +
944 // ----------------------------------------------------------------------------
945
946 static void functions2json(DICTIONARY *functions, BUFFER *wb)
@@ -1106,18 +952,23 @@ static void functions2json(DICTIONARY *functions, BUFFER *wb)
952 continue;
953
954 buffer_json_member_add_object(wb, t_dfe.name);
1109 - buffer_json_member_add_string_or_empty(wb, "help", string2str(t->help));
1110 - buffer_json_member_add_int64(wb, "timeout", (int64_t)t->timeout);
1111 -
1112 - char options[65];
1113 - snprintfz(
1114 - options,
1115 - 64,
1116 - "%s%s",
1117 - (t->options & RRD_FUNCTION_LOCAL) ? "LOCAL " : "",
1118 - (t->options & RRD_FUNCTION_GLOBAL) ? "GLOBAL" : "");
1119 -
1120 - buffer_json_member_add_string_or_empty(wb, "options", options);
955 + {
956 + buffer_json_member_add_string_or_empty(wb, "help", string2str(t->help));
957 + buffer_json_member_add_int64(wb, "timeout", (int64_t) t->timeout);
958 +
959 + char options[65];
960 + snprintfz(
961 + options, 64
962 + , "%s%s"
963 + , (t->options & RRD_FUNCTION_LOCAL) ? "LOCAL " : ""
964 + , (t->options & RRD_FUNCTION_GLOBAL) ? "GLOBAL" : ""
965 + );
966 +
967 + buffer_json_member_add_string_or_empty(wb, "options", options);
968 + buffer_json_member_add_string_or_empty(wb, "tags", string2str(t->tags));
969 + buffer_json_member_add_string(wb, "access", http_id2access(t->access));
970 + buffer_json_member_add_uint64(wb, "priority", t->priority);
971 + }
972 buffer_json_object_close(wb);
973 }
974 dfe_done(t);
@@ -1139,14 +990,21 @@ void host_functions2json(RRDHOST *host, BUFFER *wb) {
990 if(!rrd_collector_running(t->collector)) continue;
991
992 buffer_json_member_add_object(wb, t_dfe.name);
1142 - buffer_json_member_add_string(wb, "help", string2str(t->help));
1143 - buffer_json_member_add_int64(wb, "timeout", t->timeout);
1144 - buffer_json_member_add_array(wb, "options");
1145 - if(t->options & RRD_FUNCTION_GLOBAL)
1146 - buffer_json_add_array_item_string(wb, "GLOBAL");
1147 - if(t->options & RRD_FUNCTION_LOCAL)
1148 - buffer_json_add_array_item_string(wb, "LOCAL");
1149 - buffer_json_array_close(wb);
993 + {
994 + buffer_json_member_add_string(wb, "help", string2str(t->help));
995 + buffer_json_member_add_int64(wb, "timeout", t->timeout);
996 + buffer_json_member_add_array(wb, "options");
997 + {
998 + if (t->options & RRD_FUNCTION_GLOBAL)
999 + buffer_json_add_array_item_string(wb, "GLOBAL");
1000 + if (t->options & RRD_FUNCTION_LOCAL)
1001 + buffer_json_add_array_item_string(wb, "LOCAL");
1002 + }
1003 + buffer_json_array_close(wb);
1004 + buffer_json_member_add_string(wb, "tags", string2str(t->tags));
1005 + buffer_json_member_add_string(wb, "access", http_id2access(t->access));
1006 + buffer_json_member_add_uint64(wb, "priority", t->priority);
1007 + }
1008 buffer_json_object_close(wb);
1009 }
1010 dfe_done(t);
@@ -1166,7 +1024,7 @@ void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst,
1024 dfe_done(t);
1025 }
1026
1169 -void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help) {
1027 +void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help, STRING **tags, HTTP_ACCESS *access, int *priority) {
1028 if(!host || !host->functions || !dictionary_entries(host->functions) || !dst) return;
1029
1030 struct rrd_host_function *t;
@@ -1176,6 +1034,15 @@ void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t
1034 if(help)
1035 *help = t->help;
1036
1037 + if(tags)
1038 + *tags = t->tags;
1039 +
1040 + if(access)
1041 + *access = t->access;
1042 +
1043 + if(priority)
1044 + *priority = t->priority;
1045 +
1046 dictionary_set(dst, t_dfe.name, value, value_size);
1047 }
1048 dfe_done(t);
@@ -1183,12 +1050,40 @@ void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t
1050
1051 // ----------------------------------------------------------------------------
1052
1186 -int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused,
1053 +int rrdhost_function_progress(uuid_t *transaction __maybe_unused, BUFFER *wb,
1054 + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
1055 + void *collector_data __maybe_unused,
1056 + rrd_function_result_callback_t result_cb, void *result_cb_data,
1057 + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
1058 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
1059 + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
1060 + void *register_canceller_cb_data __maybe_unused,
1061 + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
1062 + void *register_progresser_cb_data __maybe_unused) {
1063 +
1064 + int response = progress_function_result(wb, rrdhost_hostname(localhost));
1065 +
1066 + if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) {
1067 + buffer_flush(wb);
1068 + response = HTTP_RESP_CLIENT_CLOSED_REQUEST;
1069 + }
1070 +
1071 + if(result_cb)
1072 + result_cb(wb, response, result_cb_data);
1073 +
1074 + return response;
1075 +}
1076 +
1077 +int rrdhost_function_streaming(uuid_t *transaction __maybe_unused, BUFFER *wb,
1078 + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused,
1079 void *collector_data __maybe_unused,
1080 rrd_function_result_callback_t result_cb, void *result_cb_data,
1081 + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused,
1082 rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
1083 rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
1191 - void *register_canceller_cb_data __maybe_unused) {
1084 + void *register_canceller_cb_data __maybe_unused,
1085 + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused,
1086 + void *register_progresser_cb_data __maybe_unused) {
1087
1088 time_t now = now_realtime_sec();
1089
@@ -1389,19 +1284,19 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1284
1285 buffer_rrdf_table_add_field(wb, field_id++, "dbMetrics", "Time-series Metrics in the DB",
1286 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1392 - 0, NULL, max_db_metrics, RRDF_FIELD_SORT_DESCENDING, NULL,
1287 + 0, NULL, (double)max_db_metrics, RRDF_FIELD_SORT_DESCENDING, NULL,
1288 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1289 RRDF_FIELD_OPTS_VISIBLE, NULL);
1290
1291 buffer_rrdf_table_add_field(wb, field_id++, "dbInstances", "Instances in the DB",
1292 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1398 - 0, NULL, max_db_instances, RRDF_FIELD_SORT_DESCENDING, NULL,
1293 + 0, NULL, (double)max_db_instances, RRDF_FIELD_SORT_DESCENDING, NULL,
1294 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1295 RRDF_FIELD_OPTS_VISIBLE, NULL);
1296
1297 buffer_rrdf_table_add_field(wb, field_id++, "dbContexts", "Contexts in the DB",
1298 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1404 - 0, NULL, max_db_contexts, RRDF_FIELD_SORT_DESCENDING, NULL,
1299 + 0, NULL, (double)max_db_contexts, RRDF_FIELD_SORT_DESCENDING, NULL,
1300 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1301 RRDF_FIELD_OPTS_VISIBLE, NULL);
1302
@@ -1460,7 +1355,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1355
1356 buffer_rrdf_table_add_field(wb, field_id++, "InReplInstances", "Inbound Replicating Instances",
1357 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1463 - 0, "instances", max_collection_replication_instances, RRDF_FIELD_SORT_DESCENDING,
1358 + 0, "instances", (double)max_collection_replication_instances, RRDF_FIELD_SORT_DESCENDING,
1359 NULL,
1360 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1361 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1535,7 +1430,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1430
1431 buffer_rrdf_table_add_field(wb, field_id++, "OutReplInstances", "Outbound Replicating Instances",
1432 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1538 - 0, "instances", max_streaming_replication_instances, RRDF_FIELD_SORT_DESCENDING,
1433 + 0, "instances", (double)max_streaming_replication_instances, RRDF_FIELD_SORT_DESCENDING,
1434 NULL,
1435 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1436 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1584,7 +1479,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1479
1480 buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficData", "Outbound Metric Data Traffic",
1481 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1587 - 0, "bytes", max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA],
1482 + 0, "bytes", (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA],
1483 RRDF_FIELD_SORT_DESCENDING, NULL,
1484 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1485 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1592,7 +1487,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1487 buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficMetadata", "Outbound Metric Metadata Traffic",
1488 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1489 0, "bytes",
1595 - max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA],
1490 + (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA],
1491 RRDF_FIELD_SORT_DESCENDING, NULL,
1492 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1493 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1600,7 +1495,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1495 buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficReplication", "Outbound Metric Replication Traffic",
1496 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1497 0, "bytes",
1603 - max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION],
1498 + (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION],
1499 RRDF_FIELD_SORT_DESCENDING, NULL,
1500 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1501 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1608,7 +1503,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1503 buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficFunctions", "Outbound Metric Functions Traffic",
1504 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1505 0, "bytes",
1611 - max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS],
1506 + (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS],
1507 RRDF_FIELD_SORT_DESCENDING, NULL,
1508 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1509 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1639,7 +1534,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1534 buffer_rrdf_table_add_field(wb, field_id++, "MlAnomalous", "Number of Anomalous Metrics",
1535 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1536 0, "metrics",
1642 - max_ml_anomalous,
1537 + (double)max_ml_anomalous,
1538 RRDF_FIELD_SORT_DESCENDING, NULL,
1539 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1540 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1647,7 +1542,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1542 buffer_rrdf_table_add_field(wb, field_id++, "MlNormal", "Number of Not Anomalous Metrics",
1543 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1544 0, "metrics",
1650 - max_ml_normal,
1545 + (double)max_ml_normal,
1546 RRDF_FIELD_SORT_DESCENDING, NULL,
1547 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1548 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1655,7 +1550,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1550 buffer_rrdf_table_add_field(wb, field_id++, "MlTrained", "Number of Trained Metrics",
1551 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1552 0, "metrics",
1658 - max_ml_trained,
1553 + (double)max_ml_trained,
1554 RRDF_FIELD_SORT_DESCENDING, NULL,
1555 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1556 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1663,7 +1558,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1558 buffer_rrdf_table_add_field(wb, field_id++, "MlPending", "Number of Pending Metrics",
1559 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1560 0, "metrics",
1666 - max_ml_pending,
1561 + (double)max_ml_pending,
1562 RRDF_FIELD_SORT_DESCENDING, NULL,
1563 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1564 RRDF_FIELD_OPTS_NONE, NULL);
@@ -1671,7 +1566,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
1566 buffer_rrdf_table_add_field(wb, field_id++, "MlSilenced", "Number of Silenced Metrics",
1567 RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
1568 0, "metrics",
1674 - max_ml_silenced,
1569 + (double)max_ml_silenced,
1570 RRDF_FIELD_SORT_DESCENDING, NULL,
1571 RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
1572 RRDF_FIELD_OPTS_NONE, NULL);
database/rrdfunctions.h
+36 -13
@@ -6,53 +6,76 @@
6
7 #include "rrd.h"
8
9 +#define RRDFUNCTIONS_PRIORITY_DEFAULT 100
10 +
11 #define RRDFUNCTIONS_TIMEOUT_EXTENSION_UT (1 * USEC_PER_SEC)
12
13 typedef void (*rrd_function_result_callback_t)(BUFFER *wb, int code, void *result_cb_data);
14 typedef bool (*rrd_function_is_cancelled_cb_t)(void *is_cancelled_cb_data);
13 -typedef void (*rrd_function_canceller_cb_t)(void *data);
14 -typedef void (*rrd_function_register_canceller_cb_t)(void *register_cancel_cb_data, rrd_function_canceller_cb_t cancel_cb, void *cancel_cb_data);
15 -typedef int (*rrd_function_execute_cb_t)(BUFFER *wb, int timeout, const char *function, void *collector_data,
15 +typedef void (*rrd_function_cancel_cb_t)(void *data);
16 +typedef void (*rrd_function_register_canceller_cb_t)(void *register_cancel_cb_data, rrd_function_cancel_cb_t cancel_cb, void *cancel_cb_data);
17 +typedef void (*rrd_function_progress_cb_t)(void *data, size_t done, size_t all);
18 +typedef void (*rrd_function_progresser_cb_t)(void *data);
19 +typedef void (*rrd_function_register_progresser_cb_t)(void *register_progresser_cb_data, rrd_function_progresser_cb_t progresser_cb, void *progresser_cb_data);
20 +
21 +typedef int (*rrd_function_execute_cb_t)(uuid_t *transaction, BUFFER *wb,
22 + usec_t *stop_monotonic_ut, const char *function, void *collector_data,
23 rrd_function_result_callback_t result_cb, void *result_cb_data,
24 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
25 rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
18 - rrd_function_register_canceller_cb_t register_cancel_cb, void *register_cancel_db_data);
26 + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
27 + rrd_function_register_progresser_cb_t register_progresser_cb, void *register_progresser_cb_data);
28
29 void rrd_functions_inflight_init(void);
30 void rrdfunctions_host_init(RRDHOST *host);
31 void rrdfunctions_host_destroy(RRDHOST *host);
32
24 -void rrd_collector_started(void);
25 -void rrd_collector_finished(void);
26 -
33 // add a function, to be run from the collector
28 -void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, const char *help,
29 - bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data);
34 +void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority, const char *help, const char *tags,
35 + HTTP_ACCESS access, bool sync, rrd_function_execute_cb_t execute_cb,
36 + void *execute_cb_data);
37
38 // call a function, to be run from anywhere
32 -int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *cmd,
39 +int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, HTTP_ACCESS access, const char *cmd,
40 bool wait, const char *transaction,
41 rrd_function_result_callback_t result_cb, void *result_cb_data,
42 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
43 rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload);
44
45 // cancel a running function, to be run from anywhere
46 void rrd_function_cancel(const char *transaction);
47 +void rrd_function_progress(const char *transaction);
48 +void rrd_function_call_progresser(uuid_t *transaction);
49
50 void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb);
51 void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb);
52
53 void chart_functions2json(RRDSET *st, BUFFER *wb);
54 void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst, void *value, size_t value_size);
45 -void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help);
55 +void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help, STRING **tags, HTTP_ACCESS *access, int *priority);
56 void host_functions2json(RRDHOST *host, BUFFER *wb);
57
58 uint8_t functions_format_to_content_type(const char *format);
59 const char *functions_content_type_to_format(HTTP_CONTENT_TYPE content_type);
60 int rrd_call_function_error(BUFFER *wb, const char *msg, int code);
61
52 -int rrdhost_function_streaming(BUFFER *wb, int timeout, const char *function, void *collector_data,
62 +int rrdhost_function_progress(uuid_t *transaction, BUFFER *wb,
63 + usec_t *stop_monotonic_ut, const char *function, void *collector_data,
64 + rrd_function_result_callback_t result_cb, void *result_cb_data,
65 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
66 + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
67 + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
68 + rrd_function_register_progresser_cb_t register_progresser_cb,
69 + void *register_progresser_cb_data);
70 +
71 +int rrdhost_function_streaming(uuid_t *transaction, BUFFER *wb,
72 + usec_t *stop_monotonic_ut, const char *function, void *collector_data,
73 rrd_function_result_callback_t result_cb, void *result_cb_data,
74 + rrd_function_progress_cb_t progress_cb, void *progress_cb_data,
75 rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
55 - rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data);
76 + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data,
77 + rrd_function_register_progresser_cb_t register_progresser_cb,
78 + void *register_progresser_cb_data);
79
80 #define RRDFUNCTIONS_STREAMING_HELP "Streaming status for parents and children."
81
database/rrdhost.c
+11 -5
@@ -1112,23 +1112,29 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unitt
1112 , 0
1113 );
1114
1115 - if (unlikely(!localhost)) {
1115 + if (unlikely(!localhost))
1116 return 1;
1117 - }
1117
1118 // we register this only on localhost
1119 // for the other nodes, the origin server should register it
1120 rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
1122 - rrd_function_add(localhost, NULL, "streaming", 10,
1123 - RRDFUNCTIONS_STREAMING_HELP, true,
1121 + rrd_function_add(localhost, NULL, "streaming", 10, RRDFUNCTIONS_PRIORITY_DEFAULT + 1,
1122 + RRDFUNCTIONS_STREAMING_HELP, "top",
1123 + HTTP_ACCESS_MEMBERS, true,
1124 rrdhost_function_streaming, NULL);
1125
1126 + rrd_function_add(localhost, NULL, "netdata-api-calls", 10, RRDFUNCTIONS_PRIORITY_DEFAULT + 2,
1127 + RRDFUNCTIONS_PROGRESS_HELP, "top",
1128 + HTTP_ACCESS_MEMBERS, true,
1129 + rrdhost_function_progress, NULL);
1130 +
1131 if (likely(system_info)) {
1132 migrate_localhost(&localhost->host_uuid);
1133 sql_aclk_sync_init();
1134 web_client_api_v1_management_init();
1135 }
1131 - return localhost==NULL;
1136 +
1137 + return 0;
1138 }
1139
1140 // ----------------------------------------------------------------------------
libnetdata/buffer/buffer.c
+10 -10
@@ -259,23 +259,23 @@ void buffer_free(BUFFER *b) {
259 void buffer_increase(BUFFER *b, size_t free_size_required) {
260 buffer_overflow_check(b);
261
262 - size_t left = b->size - b->len;
263 - if(left >= free_size_required) return;
262 + size_t remaining = b->size - b->len;
263 + if(remaining >= free_size_required) return;
264
265 - size_t wanted = free_size_required - left;
266 - size_t minimum = WEB_DATA_LENGTH_INCREASE_STEP;
267 - if(minimum > wanted) wanted = minimum;
265 + size_t increase = free_size_required - remaining;
266 + size_t minimum = 128;
267 + if(minimum > increase) increase = minimum;
268
269 size_t optimal = (b->size > 5*1024*1024) ? b->size / 2 : b->size;
270 - if(optimal > wanted) wanted = optimal;
270 + if(optimal > increase) increase = optimal;
271
272 - netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + wanted);
272 + netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + increase);
273
274 - b->buffer = reallocz(b->buffer, b->size + wanted + sizeof(BUFFER_OVERFLOW_EOF) + 2);
275 - b->size += wanted;
274 + b->buffer = reallocz(b->buffer, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2);
275 + b->size += increase;
276
277 if(b->statistics)
278 - __atomic_add_fetch(b->statistics, wanted, __ATOMIC_RELAXED);
278 + __atomic_add_fetch(b->statistics, increase, __ATOMIC_RELAXED);
279
280 buffer_overflow_init(b);
281 buffer_overflow_check(b);
libnetdata/buffer/buffer.h
+20 -2
@@ -10,8 +10,6 @@
10 #include "h2o/memory.h"
11 #endif
12
13 -#define WEB_DATA_LENGTH_INCREASE_STEP 1024
14 -
13 #define BUFFER_JSON_MAX_DEPTH 32 // max is 255
14
15 extern const char hex_digits[16];
@@ -871,6 +869,26 @@ static inline void buffer_json_add_array_item_string(BUFFER *wb, const char *val
869 wb->json.stack[wb->json.depth].count++;
870 }
871
872 +static inline void buffer_json_add_array_item_uuid(BUFFER *wb, uuid_t *value) {
873 + if(value && !uuid_is_null(*value)) {
874 + char uuid[GUID_LEN + 1];
875 + uuid_unparse_lower(*value, uuid);
876 + buffer_json_add_array_item_string(wb, uuid);
877 + }
878 + else
879 + buffer_json_add_array_item_string(wb, NULL);
880 +}
881 +
882 +static inline void buffer_json_add_array_item_uuid_compact(BUFFER *wb, uuid_t *value) {
883 + if(value && !uuid_is_null(*value)) {
884 + char uuid[GUID_LEN + 1];
885 + uuid_unparse_lower_compact(*value, uuid);
886 + buffer_json_add_array_item_string(wb, uuid);
887 + }
888 + else
889 + buffer_json_add_array_item_string(wb, NULL);
890 +}
891 +
892 static inline void buffer_json_add_array_item_double(BUFFER *wb, NETDATA_DOUBLE value) {
893 buffer_print_json_comma_newline_spacing(wb);
894
libnetdata/dictionary/dictionary.h
+1 -1
@@ -294,7 +294,7 @@ typedef DICTFE_CONST struct dictionary_foreach {
294 DICTFE value ## _dfe = {}; \
295 (void)(value); /* needed to avoid warning when looping without using this */ \
296 for((value) = dictionary_foreach_start_rw(&value ## _dfe, (dict), (mode)); \
297 - (value ## _dfe.item) ; \
297 + (value ## _dfe.item) || (value) ; \
298 (value) = dictionary_foreach_next(&value ## _dfe)) \
299 {
300
libnetdata/dyn_conf/dyn_conf.c
+12 -12
@@ -620,7 +620,7 @@ void freez_dyncfg(void *ptr) {
620 #ifdef NETDATA_TEST_DYNCFG
621 static void handle_dyncfg_root(DICTIONARY *plugins_dict, struct uni_http_response *resp, int method)
622 {
623 - if (method != HTTP_METHOD_GET) {
623 + if (method != HTTP_REQUEST_MODE_GET) {
624 resp->content = "method not allowed";
625 resp->content_length = strlen(resp->content);
626 resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
@@ -640,7 +640,7 @@ static void handle_dyncfg_root(DICTIONARY *plugins_dict, struct uni_http_respons
640 static void handle_plugin_root(struct uni_http_response *resp, int method, struct configurable_plugin *plugin, void *post_payload, size_t post_payload_size)
641 {
642 switch(method) {
643 - case HTTP_METHOD_GET:
643 + case HTTP_REQUEST_MODE_GET:
644 {
645 dyncfg_config_t cfg = plugin->get_config_cb(plugin->cb_usr_ctx, plugin->name);
646 resp->content = mallocz(cfg.data_size);
@@ -650,7 +650,7 @@ static void handle_plugin_root(struct uni_http_response *resp, int method, struc
650 resp->content_length = cfg.data_size;
651 return;
652 }
653 - case HTTP_METHOD_PUT:
653 + case HTTP_REQUEST_MODE_PUT:
654 {
655 char *response;
656 if (post_payload == NULL) {
@@ -696,7 +696,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi
696 return;
697 }
698 if (strncmp(module, DYN_CONF_MODULE_LIST, sizeof(DYN_CONF_MODULE_LIST)) == 0) {
699 - if (method != HTTP_METHOD_GET) {
699 + if (method != HTTP_REQUEST_MODE_GET) {
700 resp->content = "method not allowed (only GET)";
701 resp->content_length = strlen(resp->content);
702 resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
@@ -720,7 +720,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi
720 resp->status = HTTP_RESP_NOT_FOUND;
721 return;
722 }
723 - if (method == HTTP_METHOD_GET) {
723 + if (method == HTTP_REQUEST_MODE_GET) {
724 dyncfg_config_t cfg = mod->get_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name);
725 resp->content = mallocz(cfg.data_size);
726 memcpy(resp->content, cfg.data, cfg.data_size);
@@ -728,7 +728,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi
728 resp->content_free = freez_dyncfg;
729 resp->content_length = cfg.data_size;
730 return;
731 - } else if (method == HTTP_METHOD_PUT) {
731 + } else if (method == HTTP_REQUEST_MODE_PUT) {
732 char *response;
733 if (post_payload == NULL) {
734 resp->content = "no payload";
@@ -759,7 +759,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi
759
760 static inline void _handle_job_root(struct uni_http_response *resp, int method, struct module *mod, const char *job_id, void *post_payload, size_t post_payload_size, struct job *job)
761 {
762 - if (method == HTTP_METHOD_POST) {
762 + if (method == HTTP_REQUEST_MODE_POST) {
763 if (job != NULL) {
764 resp->content = "can't POST, job already exists (use PUT to update?)";
765 resp->content_length = strlen(resp->content);
@@ -794,7 +794,7 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method,
794 return;
795 }
796 switch (method) {
797 - case HTTP_METHOD_GET:
797 + case HTTP_REQUEST_MODE_GET:
798 {
799 dyncfg_config_t cfg = mod->get_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job->name);
800 resp->content = mallocz(cfg.data_size);
@@ -804,7 +804,7 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method,
804 resp->content_length = cfg.data_size;
805 return;
806 }
807 - case HTTP_METHOD_PUT:
807 + case HTTP_REQUEST_MODE_PUT:
808 {
809 if (post_payload == NULL) {
810 resp->content = "missing payload";
@@ -828,7 +828,7 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method,
828 resp->content_length = strlen(resp->content);
829 return;
830 }
831 - case HTTP_METHOD_DELETE:
831 + case HTTP_REQUEST_MODE_DELETE:
832 {
833 if (!remove_job(mod, job)) {
834 resp->content = "failed to remove job";
@@ -876,7 +876,7 @@ void handle_job_root(struct uni_http_response *resp, int method, struct module *
876 resp->status = HTTP_RESP_NOT_FOUND;
877 return;
878 }
879 - if (method != HTTP_METHOD_GET) {
879 + if (method != HTTP_REQUEST_MODE_GET) {
880 resp->content = "method not allowed (only GET)";
881 resp->content_length = strlen(resp->content);
882 resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
@@ -913,7 +913,7 @@ struct uni_http_response dyn_conf_process_http_request(
913 struct uni_http_response resp = {
914 .status = HTTP_RESP_INTERNAL_SERVER_ERROR,
915 .content_type = CT_TEXT_PLAIN,
916 - .content = HTTP_RESP_INTERNAL_SERVER_ERROR_STR,
916 + .content = (char *) http_response_code2string(HTTP_RESP_INTERNAL_SERVER_ERROR),
917 .content_free = NULL,
918 .content_length = 0
919 };
libnetdata/facets/facets.c
+6 -12
@@ -102,10 +102,7 @@ static inline bool is_valid_string_hash(const char *s) {
102 // hashtable for FACET_VALUE
103
104 // cleanup hashtable defines
105 -#undef SIMPLE_HASHTABLE_SORT_FUNCTION
106 -#undef SIMPLE_HASHTABLE_VALUE_TYPE
107 -#undef SIMPLE_HASHTABLE_NAME
108 -#undef NETDATA_SIMPLE_HASHTABLE_H
105 +#include "../../libnetdata/simple_hashtable_undef.h"
106
107 struct facet_value;
108 // #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value
@@ -117,10 +114,7 @@ struct facet_value;
114 // hashtable for FACET_KEY
115
116 // cleanup hashtable defines
120 -#undef SIMPLE_HASHTABLE_SORT_FUNCTION
121 -#undef SIMPLE_HASHTABLE_VALUE_TYPE
122 -#undef SIMPLE_HASHTABLE_NAME
123 -#undef NETDATA_SIMPLE_HASHTABLE_H
117 +#include "../../libnetdata/simple_hashtable_undef.h"
118
119 struct facet_key;
120 // #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key
@@ -439,12 +433,12 @@ static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY *k, FACET_VALUE *v, const
433 }
434
435 static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH hash) {
442 - SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, true);
436 + SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, NULL, true);
437 return SIMPLE_HASHTABLE_SLOT_DATA(slot);
438 }
439
440 static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) {
447 - SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, true);
441 + SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, NULL, true);
442
443 if(SIMPLE_HASHTABLE_SLOT_DATA(slot)) {
444 // already exists
@@ -634,7 +628,7 @@ static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) {
628 }
629
630 static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH hash) {
637 - SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true);
631 + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, NULL, true);
632 return SIMPLE_HASHTABLE_SLOT_DATA(slot);
633 }
634
@@ -714,7 +708,7 @@ static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, con
708 static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) {
709 facets->operations.keys.registered++;
710
717 - SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true);
711 + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, NULL, true);
712
713 if(unlikely(!SIMPLE_HASHTABLE_SLOT_DATA(slot))) {
714 // we have to add it
libnetdata/functions_evloop/functions_evloop.c
+28 -9
@@ -8,6 +8,7 @@ struct functions_evloop_worker_job {
8 bool used;
9 bool running;
10 bool cancelled;
11 + usec_t stop_monotonic_ut;
12 char *cmd;
13 const char *transaction;
14 time_t timeout;
@@ -72,7 +73,7 @@ static void *rrd_functions_worker_globals_worker_main(void *arg) {
73
74 last_acquired = true;
75 j = dictionary_acquired_item_value(acquired);
75 - j->cb(j->transaction, j->cmd, j->timeout, &j->cancelled);
76 + j->cb(j->transaction, j->cmd, &j->stop_monotonic_ut, &j->cancelled);
77 dictionary_del(wg->worker_queue, j->transaction);
78 dictionary_acquired_item_release(wg->worker_queue, acquired);
79 dictionary_garbage_collect(wg->worker_queue);
@@ -102,7 +103,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {
103 char *function = get_word(words, num_words, 3);
104
105 if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
105 - netdata_log_error("Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
106 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
107 keyword,
108 transaction?transaction:"(unset)",
109 timeout_s?timeout_s:"(unset)",
@@ -111,24 +112,30 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {
112 else {
113 int timeout = str2i(timeout_s);
114
115 + const char *msg = "No function with this name found";
116 bool found = false;
117 struct rrd_functions_expectation *we;
118 for(we = wg->expectations; we ;we = we->next) {
119 if(strncmp(function, we->function, we->function_length) == 0) {
120 + if(timeout <= 0)
121 + timeout = (int)we->default_timeout;
122 +
123 struct functions_evloop_worker_job t = {
124 .cmd = strdupz(function),
125 .transaction = strdupz(transaction),
126 .running = false,
127 .cancelled = false,
123 - .timeout = timeout > 0 ? timeout : we->default_timeout,
128 + .timeout = timeout,
129 + .stop_monotonic_ut = now_monotonic_usec() + (timeout * USEC_PER_SEC),
130 .used = false,
131 .cb = we->cb,
132 };
133 struct functions_evloop_worker_job *j = dictionary_set(wg->worker_queue, transaction, &t, sizeof(t));
134 if(j->used) {
129 - netdata_log_error("Received duplicate function transaction '%s'", transaction);
135 + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Received duplicate function transaction '%s'. Ignoring it.", transaction);
136 freez((void *)t.cmd);
137 freez((void *)t.transaction);
138 + msg = "Duplicate function transaction. Ignoring it.";
139 }
140 else {
141 found = true;
@@ -140,8 +147,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {
147
148 if(!found) {
149 netdata_mutex_lock(wg->stdout_mutex);
143 - pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND,
144 - "No function with this name found.");
150 + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, msg);
151 netdata_mutex_unlock(wg->stdout_mutex);
152 }
153 }
@@ -157,15 +163,28 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {
163 dictionary_garbage_collect(wg->worker_queue);
164 }
165 else
160 - netdata_log_error("Received CANCEL for transaction '%s', but it not available here", transaction);
166 + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received CANCEL for transaction '%s', but it not available here", transaction);
167 + }
168 + else if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION_PROGRESS) == 0) {
169 + char *transaction = get_word(words, num_words, 1);
170 + const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction);
171 + if(acquired) {
172 + struct functions_evloop_worker_job *j = dictionary_acquired_item_value(acquired);
173 +
174 + functions_stop_monotonic_update_on_progress(&j->stop_monotonic_ut);
175 +
176 + dictionary_acquired_item_release(wg->worker_queue, acquired);
177 + }
178 + else
179 + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received PROGRESS for transaction '%s', but it not available here", transaction);
180 }
181 else
163 - netdata_log_error("Received unknown command: %s", keyword?keyword:"(unset)");
182 + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword?keyword:"(unset)");
183 }
184
185 if(!s || feof(stdin) || ferror(stdin)) {
186 *wg->plugin_should_exit = true;
168 - netdata_log_error("Received error on stdin.");
187 + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Received error on stdin.");
188 }
189
190 exit(1);
libnetdata/functions_evloop/functions_evloop.h
+22 -1
@@ -7,6 +7,7 @@
7
8 #define PLUGINSD_KEYWORD_CHART "CHART"
9 #define PLUGINSD_KEYWORD_CHART_DEFINITION_END "CHART_DEFINITION_END"
10 +
11 #define PLUGINSD_KEYWORD_DIMENSION "DIMENSION"
12 #define PLUGINSD_KEYWORD_BEGIN "BEGIN"
13 #define PLUGINSD_KEYWORD_SET "SET"
@@ -18,8 +19,12 @@
19 #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
20 #define PLUGINSD_KEYWORD_CLABEL "CLABEL"
21 #define PLUGINSD_KEYWORD_CLABEL_COMMIT "CLABEL_COMMIT"
22 +
23 #define PLUGINSD_KEYWORD_FUNCTION "FUNCTION"
24 +#define PLUGINSD_KEYWORD_FUNCTION_PAYLOAD "FUNCTION_PAYLOAD"
25 +#define PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END "FUNCTION_PAYLOAD_END"
26 #define PLUGINSD_KEYWORD_FUNCTION_CANCEL "FUNCTION_CANCEL"
27 +#define PLUGINSD_KEYWORD_FUNCTION_PROGRESS "FUNCTION_PROGRESS"
28 #define PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN "FUNCTION_RESULT_BEGIN"
29 #define PLUGINSD_KEYWORD_FUNCTION_RESULT_END "FUNCTION_RESULT_END"
30
@@ -50,12 +55,22 @@
55
56 #define PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT 10 // seconds
57
53 -typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, int timeout, bool *cancelled);
58 +typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled);
59 struct functions_evloop_worker_job;
60 struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, const char *tag, netdata_mutex_t *stdout_mutex, bool *plugin_should_exit);
61 void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout);
62 void functions_evloop_cancel_threads(struct functions_evloop_globals *wg);
63
64 +#define FUNCTIONS_EXTENDED_TIME_ON_PROGRESS_UT (10 * USEC_PER_SEC)
65 +static inline void functions_stop_monotonic_update_on_progress(usec_t *stop_monotonic_ut) {
66 + usec_t now_ut = now_monotonic_usec();
67 + if(now_ut + FUNCTIONS_EXTENDED_TIME_ON_PROGRESS_UT > *stop_monotonic_ut) {
68 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Extending function timeout due to PROGRESS update...");
69 + __atomic_store_n(stop_monotonic_ut, now_ut + FUNCTIONS_EXTENDED_TIME_ON_PROGRESS_UT, __ATOMIC_RELAXED);
70 + }
71 + else
72 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Received PROGRESS update...");
73 +}
74
75 #define pluginsd_function_result_begin_to_buffer(wb, transaction, code, content_type, expires) \
76 buffer_sprintf(wb \
@@ -98,4 +113,10 @@ static inline void pluginsd_function_result_to_stdout(const char *transaction, i
113 fflush(stdout);
114 }
115
116 +static inline void pluginsd_function_progress_to_stdout(const char *transaction, size_t done, size_t all) {
117 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION_PROGRESS " '%s' %zu %zu\n",
118 + transaction, done, all);
119 + fflush(stdout);
120 +}
121 +
122 #endif //NETDATA_FUNCTIONS_EVLOOP_H
libnetdata/http/http_access.c new
+38
@@ -0,0 +1,38 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "../libnetdata.h"
4 +
5 +static struct {
6 + HTTP_ACCESS access;
7 + const char *name;
8 +} rrd_function_access_levels[] = {
9 + { .access = HTTP_ACCESS_NONE, .name = "none" },
10 + { .access = HTTP_ACCESS_MEMBERS, .name = "members" },
11 + { .access = HTTP_ACCESS_ADMINS, .name = "admins" },
12 + { .access = HTTP_ACCESS_ANY, .name = "any" },
13 +};
14 +
15 +HTTP_ACCESS http_access2id(const char *access) {
16 + if(!access || !*access)
17 + return HTTP_ACCESS_MEMBERS;
18 +
19 + size_t entries = sizeof(rrd_function_access_levels) / sizeof(rrd_function_access_levels[0]);
20 + for(size_t i = 0; i < entries ;i++) {
21 + if(strcmp(rrd_function_access_levels[i].name, access) == 0)
22 + return rrd_function_access_levels[i].access;
23 + }
24 +
25 + nd_log(NDLS_DAEMON, NDLP_WARNING, "HTTP access level '%s' is not valid", access);
26 + return HTTP_ACCESS_MEMBERS;
27 +}
28 +
29 +const char *http_id2access(HTTP_ACCESS access) {
30 + size_t entries = sizeof(rrd_function_access_levels) / sizeof(rrd_function_access_levels[0]);
31 + for(size_t i = 0; i < entries ;i++) {
32 + if(access == rrd_function_access_levels[i].access)
33 + return rrd_function_access_levels[i].name;
34 + }
35 +
36 + nd_log(NDLS_DAEMON, NDLP_WARNING, "HTTP access level %d is not valid", access);
37 + return "members";
38 +}
libnetdata/http/http_access.h new
+56
@@ -0,0 +1,56 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_HTTP_ACCESS_H
4 +#define NETDATA_HTTP_ACCESS_H
5 +
6 +typedef enum __attribute__((packed)) {
7 + HTTP_ACCESS_NONE = 0,
8 + HTTP_ACCESS_ADMINS = 1,
9 + HTTP_ACCESS_MEMBERS = 2,
10 + HTTP_ACCESS_ANY = 3,
11 +
12 + // keep this list so that lower numbers are more strict access levels
13 +} HTTP_ACCESS;
14 +
15 +const char *http_id2access(HTTP_ACCESS access);
16 +HTTP_ACCESS http_access2id(const char *access);
17 +
18 +typedef enum __attribute__((packed)) {
19 + HTTP_ACL_NONE = (0),
20 + HTTP_ACL_NOCHECK = (1 << 0), // Don't check anything - this should work on all channels
21 + HTTP_ACL_DASHBOARD = (1 << 1),
22 + HTTP_ACL_REGISTRY = (1 << 2),
23 + HTTP_ACL_BADGE = (1 << 3),
24 + HTTP_ACL_MGMT = (1 << 4),
25 + HTTP_ACL_STREAMING = (1 << 5),
26 + HTTP_ACL_NETDATACONF = (1 << 6),
27 + HTTP_ACL_SSL_OPTIONAL = (1 << 7),
28 + HTTP_ACL_SSL_FORCE = (1 << 8),
29 + HTTP_ACL_SSL_DEFAULT = (1 << 9),
30 + HTTP_ACL_ACLK = (1 << 10),
31 + HTTP_ACL_WEBRTC = (1 << 11),
32 + HTTP_ACL_BEARER_IF_PROTECTED = (1 << 12), // allow unprotected access if bearer is not enabled in netdata
33 + HTTP_ACL_BEARER_REQUIRED = (1 << 13), // allow access only if a valid bearer is used
34 + HTTP_ACL_BEARER_OPTIONAL = (1 << 14), // the call may or may not need a bearer - will be determined later
35 +} HTTP_ACL;
36 +
37 +#define HTTP_ACL_DASHBOARD_ACLK_WEBRTC (HTTP_ACL_DASHBOARD | HTTP_ACL_ACLK | HTTP_ACL_WEBRTC | HTTP_ACL_BEARER_IF_PROTECTED)
38 +#define HTTP_ACL_ACLK_WEBRTC_DASHBOARD_WITH_OPTIONAL_BEARER (HTTP_ACL_DASHBOARD | HTTP_ACL_ACLK | HTTP_ACL_WEBRTC | HTTP_ACL_BEARER_OPTIONAL)
39 +
40 +#ifdef NETDATA_DEV_MODE
41 +#define ACL_DEV_OPEN_ACCESS HTTP_ACL_NOCHECK
42 +#else
43 +#define ACL_DEV_OPEN_ACCESS 0
44 +#endif
45 +
46 +#define http_can_access_dashboard(w) ((w)->acl & HTTP_ACL_DASHBOARD)
47 +#define http_can_access_registry(w) ((w)->acl & HTTP_ACL_REGISTRY)
48 +#define http_can_access_badges(w) ((w)->acl & HTTP_ACL_BADGE)
49 +#define http_can_access_mgmt(w) ((w)->acl & HTTP_ACL_MGMT)
50 +#define http_can_access_stream(w) ((w)->acl & HTTP_ACL_STREAMING)
51 +#define http_can_access_netdataconf(w) ((w)->acl & HTTP_ACL_NETDATACONF)
52 +#define http_is_using_ssl_optional(w) ((w)->port_acl & HTTP_ACL_SSL_OPTIONAL)
53 +#define http_is_using_ssl_force(w) ((w)->port_acl & HTTP_ACL_SSL_FORCE)
54 +#define http_is_using_ssl_default(w) ((w)->port_acl & HTTP_ACL_SSL_DEFAULT)
55 +
56 +#endif //NETDATA_HTTP_ACCESS_H
libnetdata/http/http_defs.c new
+255
@@ -0,0 +1,255 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "../libnetdata.h"
4 +
5 +const char *http_request_method2string(HTTP_REQUEST_MODE mode) {
6 + switch(mode) {
7 + case HTTP_REQUEST_MODE_OPTIONS:
8 + return "OPTIONS";
9 +
10 + case HTTP_REQUEST_MODE_GET:
11 + return "GET";
12 +
13 + case HTTP_REQUEST_MODE_FILECOPY:
14 + return "FILECOPY";
15 +
16 + case HTTP_REQUEST_MODE_POST:
17 + return "POST";
18 +
19 + case HTTP_REQUEST_MODE_PUT:
20 + return "PUT";
21 +
22 + case HTTP_REQUEST_MODE_DELETE:
23 + return "DELETE";
24 +
25 + case HTTP_REQUEST_MODE_STREAM:
26 + return "STREAM";
27 +
28 + default:
29 + return "UNKNOWN";
30 + }
31 +}
32 +
33 +const char *http_response_code2string(int code) {
34 + switch(code) {
35 + case 100:
36 + return "Continue";
37 + case 101:
38 + return "Switching Protocols";
39 + case 102:
40 + return "Processing";
41 + case 103:
42 + return "Early Hints";
43 +
44 + case 200:
45 + return "OK";
46 + case 201:
47 + return "Created";
48 + case 202:
49 + return "Accepted";
50 + case 203:
51 + return "Non-Authoritative Information";
52 + case 204:
53 + return "No Content";
54 + case 205:
55 + return "Reset Content";
56 + case 206:
57 + return "Partial Content";
58 + case 207:
59 + return "Multi-Status";
60 + case 208:
61 + return "Already Reported";
62 + case 226:
63 + return "IM Used";
64 +
65 + case 300:
66 + return "Multiple Choices";
67 + case 301:
68 + return "Moved Permanently";
69 + case 302:
70 + return "Found";
71 + case 303:
72 + return "See Other";
73 + case 304:
74 + return "Not Modified";
75 + case 305:
76 + return "Use Proxy";
77 + case 306:
78 + return "Switch Proxy";
79 + case 307:
80 + return "Temporary Redirect";
81 + case 308:
82 + return "Permanent Redirect";
83 +
84 + case 400:
85 + return "Bad Request";
86 + case 401:
87 + return "Unauthorized";
88 + case 402:
89 + return "Payment Required";
90 + case 403:
91 + return "Forbidden";
92 + case 404:
93 + return "Not Found";
94 + case 405:
95 + return "Method Not Allowed";
96 + case 406:
97 + return "Not Acceptable";
98 + case 407:
99 + return "Proxy Authentication Required";
100 + case 408:
101 + return "Request Timeout";
102 + case 409:
103 + return "Conflict";
104 + case 410:
105 + return "Gone";
106 + case 411:
107 + return "Length Required";
108 + case 412:
109 + return "Precondition Failed";
110 + case 413:
111 + return "Payload Too Large";
112 + case 414:
113 + return "URI Too Long";
114 + case 415:
115 + return "Unsupported Media Type";
116 + case 416:
117 + return "Range Not Satisfiable";
118 + case 417:
119 + return "Expectation Failed";
120 + case 418:
121 + return "I'm a teapot";
122 + case 421:
123 + return "Misdirected Request";
124 + case 422:
125 + return "Unprocessable Entity";
126 + case 423:
127 + return "Locked";
128 + case 424:
129 + return "Failed Dependency";
130 + case 425:
131 + return "Too Early";
132 + case 426:
133 + return "Upgrade Required";
134 + case 428:
135 + return "Precondition Required";
136 + case 429:
137 + return "Too Many Requests";
138 + case 431:
139 + return "Request Header Fields Too Large";
140 + case 451:
141 + return "Unavailable For Legal Reasons";
142 + case 499: // nginx's extension to the standard
143 + return "Client Closed Request";
144 +
145 + case 500:
146 + return "Internal Server Error";
147 + case 501:
148 + return "Not Implemented";
149 + case 502:
150 + return "Bad Gateway";
151 + case 503:
152 + return "Service Unavailable";
153 + case 504:
154 + return "Gateway Timeout";
155 + case 505:
156 + return "HTTP Version Not Supported";
157 + case 506:
158 + return "Variant Also Negotiates";
159 + case 507:
160 + return "Insufficient Storage";
161 + case 508:
162 + return "Loop Detected";
163 + case 510:
164 + return "Not Extended";
165 + case 511:
166 + return "Network Authentication Required";
167 +
168 + default:
169 + if(code >= 100 && code < 200)
170 + return "Informational";
171 +
172 + if(code >= 200 && code < 300)
173 + return "Successful";
174 +
175 + if(code >= 300 && code < 400)
176 + return "Redirection";
177 +
178 + if(code >= 400 && code < 500)
179 + return "Client Error";
180 +
181 + if(code >= 500 && code < 600)
182 + return "Server Error";
183 +
184 + return "Undefined Error";
185 + }
186 +}
187 +
188 +
189 +static struct {
190 + const char *extension;
191 + uint32_t hash;
192 + HTTP_CONTENT_TYPE contenttype;
193 +} mime_types[] = {
194 + { "html" , 0 , CT_TEXT_HTML }
195 + , { "js" , 0 , CT_APPLICATION_X_JAVASCRIPT }
196 + , { "css" , 0 , CT_TEXT_CSS }
197 + , { "xml" , 0 , CT_TEXT_XML }
198 + , { "xsl" , 0 , CT_TEXT_XSL }
199 + , { "txt" , 0 , CT_TEXT_PLAIN }
200 + , { "svg" , 0 , CT_IMAGE_SVG_XML }
201 + , { "ttf" , 0 , CT_APPLICATION_X_FONT_TRUETYPE }
202 + , { "otf" , 0 , CT_APPLICATION_X_FONT_OPENTYPE }
203 + , { "woff2", 0 , CT_APPLICATION_FONT_WOFF2 }
204 + , { "woff" , 0 , CT_APPLICATION_FONT_WOFF }
205 + , { "eot" , 0 , CT_APPLICATION_VND_MS_FONTOBJ }
206 + , { "png" , 0 , CT_IMAGE_PNG }
207 + , { "jpg" , 0 , CT_IMAGE_JPG }
208 + , { "jpeg" , 0 , CT_IMAGE_JPG }
209 + , { "gif" , 0 , CT_IMAGE_GIF }
210 + , { "bmp" , 0 , CT_IMAGE_BMP }
211 + , { "ico" , 0 , CT_IMAGE_XICON }
212 + , { "icns" , 0 , CT_IMAGE_ICNS }
213 + , { NULL , 0 , 0 }
214 +};
215 +
216 +HTTP_CONTENT_TYPE contenttype_for_filename(const char *filename) {
217 + // netdata_log_info("checking filename '%s'", filename);
218 +
219 + static int initialized = 0;
220 + int i;
221 +
222 + if(unlikely(!initialized)) {
223 + for (i = 0; mime_types[i].extension; i++)
224 + mime_types[i].hash = simple_hash(mime_types[i].extension);
225 +
226 + initialized = 1;
227 + }
228 +
229 + const char *s = filename, *last_dot = NULL;
230 +
231 + // find the last dot
232 + while(*s) {
233 + if(unlikely(*s == '.')) last_dot = s;
234 + s++;
235 + }
236 +
237 + if(unlikely(!last_dot || !*last_dot || !last_dot[1])) {
238 + // netdata_log_info("no extension for filename '%s'", filename);
239 + return CT_APPLICATION_OCTET_STREAM;
240 + }
241 + last_dot++;
242 +
243 + // netdata_log_info("extension for filename '%s' is '%s'", filename, last_dot);
244 +
245 + uint32_t hash = simple_hash(last_dot);
246 + for(i = 0; mime_types[i].extension ; i++) {
247 + if(unlikely(hash == mime_types[i].hash && !strcmp(last_dot, mime_types[i].extension))) {
248 + // netdata_log_info("matched extension for filename '%s': '%s'", filename, last_dot);
249 + return mime_types[i].contenttype;
250 + }
251 + }
252 +
253 + // netdata_log_info("not matched extension for filename '%s': '%s'", filename, last_dot);
254 + return CT_APPLICATION_OCTET_STREAM;
255 +}
libnetdata/http/http_defs.h
+14 -6
@@ -26,7 +26,6 @@
26 #define HTTP_RESP_FORBIDDEN 403
27 #define HTTP_RESP_NOT_FOUND 404
28 #define HTTP_RESP_METHOD_NOT_ALLOWED 405
29 -#define HTTP_RESP_METHOD_NOT_ALLOWED_STR "Method Not Allowed"
29 #define HTTP_RESP_CONFLICT 409
30 #define HTTP_RESP_PRECOND_FAIL 412
31 #define HTTP_RESP_CONTENT_TOO_LONG 413
@@ -34,14 +33,23 @@
33
34 // HTTP_CODES 5XX Server Errors
35 #define HTTP_RESP_INTERNAL_SERVER_ERROR 500
37 -#define HTTP_RESP_INTERNAL_SERVER_ERROR_STR "Internal Server Error"
36 #define HTTP_RESP_SERVICE_UNAVAILABLE 503
37 #define HTTP_RESP_GATEWAY_TIMEOUT 504
38 #define HTTP_RESP_BACKEND_RESPONSE_INVALID 591
39
42 -#define HTTP_METHOD_GET (1)
43 -#define HTTP_METHOD_POST (2)
44 -#define HTTP_METHOD_PUT (3)
45 -#define HTTP_METHOD_DELETE (4)
40 +typedef enum __attribute__((__packed__)) {
41 + HTTP_REQUEST_MODE_NONE = 0,
42 + HTTP_REQUEST_MODE_GET = 1,
43 + HTTP_REQUEST_MODE_POST = 2,
44 + HTTP_REQUEST_MODE_PUT = 3,
45 + HTTP_REQUEST_MODE_DELETE = 4,
46 + HTTP_REQUEST_MODE_FILECOPY = 5,
47 + HTTP_REQUEST_MODE_OPTIONS = 6,
48 + HTTP_REQUEST_MODE_STREAM = 7,
49 +} HTTP_REQUEST_MODE;
50 +
51 +const char *http_request_method2string(HTTP_REQUEST_MODE mode);
52 +const char *http_response_code2string(int code);
53 +HTTP_CONTENT_TYPE contenttype_for_filename(const char *filename);
54
55 #endif /* NETDATA_HTTP_DEFS_H */
libnetdata/libnetdata.h
+2 -1
@@ -697,7 +697,7 @@ extern char *netdata_configured_host_prefix;
697 #include "xxhash.h"
698
699 #include "uuid/uuid.h"
700 -
700 +#include "http/http_access.h"
701 #include "libjudy/src/Judy.h"
702 #include "july/july.h"
703 #include "os.h"
@@ -745,6 +745,7 @@ extern char *netdata_configured_host_prefix;
745 #include "facets/facets.h"
746 #include "dyn_conf/dyn_conf.h"
747 #include "functions_evloop/functions_evloop.h"
748 +#include "query_progress/progress.h"
749
750 // BEWARE: this exists in alarm-notify.sh
751 #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
libnetdata/log/README.md
+2
@@ -138,6 +138,8 @@ Netdata exposes the following fields to its logs:
138 | `ND_SRC_TRANSPORT` | `src_transport` | `src_transport` | when the event happened during a request, this is the request transport |
139 | `ND_SRC_IP` | `src_ip` | `src_ip` | when the event happened during an inbound request, this is the IP the request came from |
140 | `ND_SRC_PORT` | `src_port` | `src_port` | when the event happened during an inbound request, this is the port the request came from |
141 +| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `src_forwarded_host` | the contents of the HTTP header `X-Forwarded-Host` |
142 +| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `src_forwarded_for` | the contents of the HTTP header `X-Forwarded-For` |
143 | `ND_SRC_CAPABILITIES` | `src_capabilities` | `src_capabilities` | when the request came from a child, this is the communication capabilities of the child |
144 | `ND_DST_TRANSPORT` | `dst_transport` | `dst_transport` | when the event happened during an outbound request, this is the outbound request transport |
145 | `ND_DST_IP` | `dst_ip` | `dst_ip` | when the event happened during an outbound request, this is the IP the request destination |
libnetdata/log/log.c
+8
@@ -1121,6 +1121,14 @@ static __thread struct log_field thread_log_fields[_NDF_MAX] = {
1121 .journal = "ND_SRC_IP",
1122 .logfmt = "src_ip",
1123 },
1124 + [NDF_SRC_FORWARDED_HOST] = {
1125 + .journal = "ND_SRC_FORWARDED_HOST",
1126 + .logfmt = "src_forwarded_host",
1127 + },
1128 + [NDF_SRC_FORWARDED_FOR] = {
1129 + .journal = "ND_SRC_FORWARDED_FOR",
1130 + .logfmt = "src_forwarded_for",
1131 + },
1132 [NDF_SRC_PORT] = {
1133 .journal = "ND_SRC_PORT",
1134 .logfmt = "src_port",
libnetdata/log/log.h
+2
@@ -66,6 +66,8 @@ typedef enum __attribute__((__packed__)) {
66 // web server and stream receiver
67 NDF_SRC_IP, // the streaming / web server source IP
68 NDF_SRC_PORT, // the streaming / web server source Port
69 + NDF_SRC_FORWARDED_HOST,
70 + NDF_SRC_FORWARDED_FOR,
71 NDF_SRC_CAPABILITIES, // the stream receiver capabilities
72
73 // stream sender (established links)
libnetdata/query_progress/README.md
libnetdata/query_progress/progress.c new
+654
@@ -0,0 +1,654 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "progress.h"
4 +
5 +#define PROGRESS_CACHE_SIZE 200
6 +
7 +// ----------------------------------------------------------------------------
8 +// hashtable for HASHED_KEY
9 +
10 +// cleanup hashtable defines
11 +#include "../simple_hashtable_undef.h"
12 +
13 +struct query;
14 +#define SIMPLE_HASHTABLE_VALUE_TYPE struct query
15 +#define SIMPLE_HASHTABLE_KEY_TYPE uuid_t
16 +#define SIMPLE_HASHTABLE_NAME _QUERY
17 +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION query_transaction
18 +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION query_compare_keys
19 +#include "../simple_hashtable.h"
20 +
21 +// ----------------------------------------------------------------------------
22 +
23 +typedef struct query {
24 + uuid_t transaction;
25 +
26 + BUFFER *query;
27 + BUFFER *payload;
28 + BUFFER *client;
29 +
30 + usec_t started_ut;
31 + usec_t finished_ut;
32 +
33 + HTTP_REQUEST_MODE mode;
34 + HTTP_ACL acl;
35 +
36 + uint32_t sent_size;
37 + uint32_t response_size;
38 + short response_code;
39 +
40 + bool indexed;
41 +
42 + uint32_t updates;
43 +
44 + usec_t duration_ut;
45 + size_t all;
46 + size_t done;
47 +
48 + struct query *prev, *next;
49 +} QUERY_PROGRESS;
50 +
51 +static inline uuid_t *query_transaction(QUERY_PROGRESS *qp) {
52 + return qp ? &qp->transaction : NULL;
53 +}
54 +
55 +static inline bool query_compare_keys(uuid_t *t1, uuid_t *t2) {
56 + if(t1 == t2 || (t1 && t2 && memcmp(t1, t2, sizeof(uuid_t)) == 0))
57 + return true;
58 +
59 + return false;
60 +}
61 +
62 +static struct progress {
63 + SPINLOCK spinlock;
64 + bool initialized;
65 +
66 + struct {
67 + size_t available;
68 + QUERY_PROGRESS *list;
69 + } cache;
70 +
71 + SIMPLE_HASHTABLE_QUERY hashtable;
72 +
73 +} progress = {
74 + .initialized = false,
75 +};
76 +
77 +SIMPLE_HASHTABLE_HASH query_hash(uuid_t *transaction) {
78 + struct uuid_hi_lo_t {
79 + uint64_t hi;
80 + uint64_t lo;
81 + } *parts = (struct uuid_hi_lo_t *)transaction;
82 +
83 + return parts->lo;
84 +}
85 +
86 +static void query_progress_init_unsafe(void) {
87 + if(!progress.initialized) {
88 + memset(&progress, 0, sizeof(progress));
89 + simple_hashtable_init_QUERY(&progress.hashtable, PROGRESS_CACHE_SIZE * 4);
90 + progress.initialized = true;
91 + }
92 +}
93 +
94 +// ----------------------------------------------------------------------------
95 +
96 +static inline QUERY_PROGRESS *query_progress_find_in_hashtable_unsafe(uuid_t *transaction) {
97 + SIMPLE_HASHTABLE_HASH hash = query_hash(transaction);
98 + SIMPLE_HASHTABLE_SLOT_QUERY *slot = simple_hashtable_get_slot_QUERY(&progress.hashtable, hash, transaction, true);
99 + QUERY_PROGRESS *qp = SIMPLE_HASHTABLE_SLOT_DATA(slot);
100 +
101 + assert(!qp || qp->indexed);
102 +
103 + return qp;
104 +}
105 +
106 +static inline void query_progress_add_to_hashtable_unsafe(QUERY_PROGRESS *qp) {
107 + assert(!qp->indexed);
108 +
109 + SIMPLE_HASHTABLE_HASH hash = query_hash(&qp->transaction);
110 + SIMPLE_HASHTABLE_SLOT_QUERY *slot =
111 + simple_hashtable_get_slot_QUERY(&progress.hashtable, hash, &qp->transaction, true);
112 +
113 + internal_fatal(SIMPLE_HASHTABLE_SLOT_DATA(slot) != NULL && SIMPLE_HASHTABLE_SLOT_DATA(slot) != qp,
114 + "Attempt to overwrite a progress slot, with another value");
115 +
116 + simple_hashtable_set_slot_QUERY(&progress.hashtable, slot, hash, qp);
117 +
118 + qp->indexed = true;
119 +}
120 +
121 +static inline void query_progress_remove_from_hashtable_unsafe(QUERY_PROGRESS *qp) {
122 + assert(qp->indexed);
123 +
124 + SIMPLE_HASHTABLE_HASH hash = query_hash(&qp->transaction);
125 + SIMPLE_HASHTABLE_SLOT_QUERY *slot =
126 + simple_hashtable_get_slot_QUERY(&progress.hashtable, hash, &qp->transaction, true);
127 +
128 + if(SIMPLE_HASHTABLE_SLOT_DATA(slot) == qp)
129 + simple_hashtable_del_slot_QUERY(&progress.hashtable, slot);
130 + else
131 + internal_fatal(SIMPLE_HASHTABLE_SLOT_DATA(slot) != NULL,
132 + "Attempt to remove from the hashtable a progress slot with a different value");
133 +
134 + qp->indexed = false;
135 +}
136 +
137 +// ----------------------------------------------------------------------------
138 +
139 +static QUERY_PROGRESS *query_progress_alloc(uuid_t *transaction) {
140 + QUERY_PROGRESS *qp;
141 + qp = callocz(1, sizeof(*qp));
142 + uuid_copy(qp->transaction, *transaction);
143 + qp->query = buffer_create(0, NULL);
144 + qp->payload = buffer_create(0, NULL);
145 + qp->client = buffer_create(0, NULL);
146 + return qp;
147 +}
148 +
149 +static void query_progress_free(QUERY_PROGRESS *qp) {
150 + if(!qp) return;
151 +
152 + buffer_free(qp->query);
153 + buffer_free(qp->payload);
154 + buffer_free(qp->client);
155 + freez(qp);
156 +}
157 +
158 +static void query_progress_cleanup_to_reuse(QUERY_PROGRESS *qp, uuid_t *transaction) {
159 + assert(qp && qp->prev == NULL && qp->next == NULL);
160 + assert(!transaction || !qp->indexed);
161 +
162 + buffer_flush(qp->query);
163 + buffer_flush(qp->payload);
164 + buffer_flush(qp->client);
165 + qp->started_ut = qp->finished_ut = qp->duration_ut = 0;
166 + qp->all = qp->done = qp->updates = 0;
167 + qp->acl = 0;
168 + qp->next = qp->prev = NULL;
169 + qp->response_size = qp->sent_size = 0;
170 + qp->response_code = 0;
171 +
172 + if(transaction)
173 + uuid_copy(qp->transaction, *transaction);
174 +}
175 +
176 +static inline void query_progress_update(QUERY_PROGRESS *qp, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client) {
177 + qp->mode = mode;
178 + qp->acl = acl;
179 + qp->started_ut = started_ut ? started_ut : now_realtime_usec();
180 + qp->finished_ut = 0;
181 + qp->duration_ut = 0;
182 + qp->response_size = 0;
183 + qp->sent_size = 0;
184 + qp->response_code = 0;
185 +
186 + if(query && *query && !buffer_strlen(qp->query))
187 + buffer_strcat(qp->query, query);
188 +
189 + if(payload && *payload && !buffer_strlen(qp->payload))
190 + buffer_strcat(qp->payload, payload);
191 +
192 + if(client && *client && !buffer_strlen(qp->client))
193 + buffer_strcat(qp->client, client);
194 +}
195 +
196 +// ----------------------------------------------------------------------------
197 +
198 +static inline void query_progress_link_to_cache_unsafe(QUERY_PROGRESS *qp) {
199 + assert(!qp->prev && !qp->next);
200 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(progress.cache.list, qp, prev, next);
201 + progress.cache.available++;
202 +}
203 +
204 +static inline void query_progress_unlink_from_cache_unsafe(QUERY_PROGRESS *qp) {
205 + assert(qp->prev);
206 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(progress.cache.list, qp, prev, next);
207 + progress.cache.available--;
208 +}
209 +
210 +// ----------------------------------------------------------------------------
211 +// Progress API
212 +
213 +void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client) {
214 + if(!transaction)
215 + return;
216 +
217 + spinlock_lock(&progress.spinlock);
218 + query_progress_init_unsafe();
219 +
220 + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction);
221 + if(qp) {
222 + // the transaction is already there
223 + if(qp->prev) {
224 + // reusing a finished transaction
225 + query_progress_unlink_from_cache_unsafe(qp);
226 + query_progress_cleanup_to_reuse(qp, NULL);
227 + }
228 + }
229 + else if (progress.cache.available >= PROGRESS_CACHE_SIZE && progress.cache.list) {
230 + // transaction is not found - get the first available, if any.
231 + qp = progress.cache.list;
232 + query_progress_unlink_from_cache_unsafe(qp);
233 +
234 + query_progress_remove_from_hashtable_unsafe(qp);
235 + query_progress_cleanup_to_reuse(qp, transaction);
236 + }
237 + else {
238 + qp = query_progress_alloc(transaction);
239 + }
240 +
241 + query_progress_update(qp, started_ut, mode, acl, query, payload, client);
242 +
243 + if(!qp->indexed)
244 + query_progress_add_to_hashtable_unsafe(qp);
245 +
246 + spinlock_unlock(&progress.spinlock);
247 +}
248 +
249 +void query_progress_set_finish_line(uuid_t *transaction, size_t all) {
250 + if(!transaction)
251 + return;
252 +
253 + spinlock_lock(&progress.spinlock);
254 + query_progress_init_unsafe();
255 +
256 + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction);
257 + if(qp) {
258 + qp->updates++;
259 +
260 + if(all > qp->all)
261 + qp->all = all;
262 + }
263 +
264 + spinlock_unlock(&progress.spinlock);
265 +}
266 +
267 +void query_progress_done_step(uuid_t *transaction, size_t done) {
268 + if(!transaction)
269 + return;
270 +
271 + spinlock_lock(&progress.spinlock);
272 + query_progress_init_unsafe();
273 +
274 + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction);
275 + if(qp) {
276 + qp->updates++;
277 + qp->done += done;
278 + }
279 +
280 + spinlock_unlock(&progress.spinlock);
281 +}
282 +
283 +void query_progress_finished(uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size) {
284 + if(!transaction)
285 + return;
286 +
287 + spinlock_lock(&progress.spinlock);
288 + query_progress_init_unsafe();
289 +
290 + // find this transaction to update it
291 + {
292 + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction);
293 + if(qp) {
294 + qp->sent_size = sent_size;
295 + qp->response_size = response_size;
296 + qp->response_code = response_code;
297 + qp->duration_ut = duration_ut;
298 + qp->finished_ut = finished_ut ? finished_ut : now_realtime_usec();
299 +
300 + if(qp->prev)
301 + query_progress_unlink_from_cache_unsafe(qp);
302 +
303 + query_progress_link_to_cache_unsafe(qp);
304 + }
305 + }
306 +
307 + // find an item to free
308 + {
309 + QUERY_PROGRESS *qp_to_free = NULL;
310 + if(progress.cache.available > PROGRESS_CACHE_SIZE && progress.cache.list) {
311 + qp_to_free = progress.cache.list;
312 + query_progress_unlink_from_cache_unsafe(qp_to_free);
313 + query_progress_remove_from_hashtable_unsafe(qp_to_free);
314 + }
315 +
316 + spinlock_unlock(&progress.spinlock);
317 +
318 + query_progress_free(qp_to_free);
319 + }
320 +}
321 +
322 +void query_progress_functions_update(uuid_t *transaction, size_t done, size_t all) {
323 + // functions send to the total 'done', not the increment
324 +
325 + if(!transaction)
326 + return;
327 +
328 + spinlock_lock(&progress.spinlock);
329 + query_progress_init_unsafe();
330 +
331 + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction);
332 +
333 + if(qp) {
334 + if(all)
335 + qp->all = all;
336 +
337 + if(done)
338 + qp->done = done;
339 +
340 + qp->updates++;
341 + }
342 +
343 + spinlock_unlock(&progress.spinlock);
344 +}
345 +
346 +// ----------------------------------------------------------------------------
347 +// /api/v2/progress - to get the progress of a transaction
348 +
349 +int web_api_v2_report_progress(uuid_t *transaction, BUFFER *wb) {
350 + buffer_flush(wb);
351 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
352 +
353 + if(!transaction) {
354 + buffer_json_member_add_uint64(wb, "status", 400);
355 + buffer_json_member_add_string(wb, "message", "No transaction given");
356 + buffer_json_finalize(wb);
357 + return 400;
358 + }
359 +
360 + spinlock_lock(&progress.spinlock);
361 + query_progress_init_unsafe();
362 +
363 + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction);
364 + if(!qp) {
365 + spinlock_unlock(&progress.spinlock);
366 + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_NOT_FOUND);
367 + buffer_json_member_add_string(wb, "message", "Transaction not found");
368 + buffer_json_finalize(wb);
369 + return HTTP_RESP_NOT_FOUND;
370 + }
371 +
372 + buffer_json_member_add_uint64(wb, "status", 200);
373 +
374 + if(qp->finished_ut) {
375 + buffer_json_member_add_double(wb, "progress", 100.0);
376 + buffer_json_member_add_uint64(wb, "age_ut", qp->finished_ut - qp->started_ut);
377 + }
378 + else {
379 + buffer_json_member_add_uint64(wb, "age_ut", now_realtime_usec() - qp->started_ut);
380 +
381 + if (qp->all)
382 + buffer_json_member_add_double(wb, "progress", (double) qp->done * 100.0 / (double) qp->all);
383 + else
384 + buffer_json_member_add_uint64(wb, "working", qp->done);
385 + }
386 +
387 + buffer_json_finalize(wb);
388 +
389 + spinlock_unlock(&progress.spinlock);
390 +
391 + return 200;
392 +}
393 +
394 +// ----------------------------------------------------------------------------
395 +// function to show the progress of all current queries
396 +// and the recent few completed queries
397 +
398 +int progress_function_result(BUFFER *wb, const char *hostname) {
399 + buffer_flush(wb);
400 + wb->content_type = CT_APPLICATION_JSON;
401 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
402 +
403 + buffer_json_member_add_string(wb, "hostname", hostname);
404 + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
405 + buffer_json_member_add_string(wb, "type", "table");
406 + buffer_json_member_add_time_t(wb, "update_every", 1);
407 + buffer_json_member_add_string(wb, "help", RRDFUNCTIONS_PROGRESS_HELP);
408 + buffer_json_member_add_array(wb, "data");
409 +
410 + spinlock_lock(&progress.spinlock);
411 + query_progress_init_unsafe();
412 +
413 + usec_t now_ut = now_realtime_usec();
414 + usec_t max_duration_ut = 0;
415 + size_t max_size = 0, max_sent = 0;
416 + size_t archived = 0, running = 0;
417 + SIMPLE_HASHTABLE_FOREACH_READ_ONLY(&progress.hashtable, sl, _QUERY) {
418 + QUERY_PROGRESS *qp = SIMPLE_HASHTABLE_FOREACH_READ_ONLY_VALUE(sl);
419 + if(unlikely(!qp)) continue; // not really needed, just for completeness
420 +
421 + if(qp->prev)
422 + archived++;
423 + else
424 + running++;
425 +
426 + bool finished = qp->finished_ut ? true : false;
427 + usec_t duration_ut = finished ? qp->duration_ut : now_ut - qp->started_ut;
428 + if(duration_ut > max_duration_ut)
429 + max_duration_ut = duration_ut;
430 +
431 + if(finished) {
432 + if(qp->response_size > max_size)
433 + max_size = qp->response_size;
434 +
435 + if(qp->sent_size > max_sent)
436 + max_sent = qp->sent_size;
437 + }
438 +
439 + buffer_json_add_array_item_array(wb); // row
440 +
441 + buffer_json_add_array_item_uuid_compact(wb, &qp->transaction);
442 + buffer_json_add_array_item_uint64(wb, qp->started_ut);
443 + buffer_json_add_array_item_string(wb, http_request_method2string(qp->mode));
444 + buffer_json_add_array_item_string(wb, buffer_tostring(qp->query));
445 +
446 + if(!buffer_strlen(qp->client)) {
447 + if(qp->acl & HTTP_ACL_ACLK)
448 + buffer_json_add_array_item_string(wb, "ACLK");
449 + else if(qp->acl & HTTP_ACL_WEBRTC)
450 + buffer_json_add_array_item_string(wb, "WEBRTC");
451 + else
452 + buffer_json_add_array_item_string(wb, "unknown");
453 + }
454 + else
455 + buffer_json_add_array_item_string(wb, buffer_tostring(qp->client));
456 +
457 + if(finished) {
458 + buffer_json_add_array_item_string(wb, "finished");
459 + buffer_json_add_array_item_string(wb, "100.00 %%");
460 + }
461 + else {
462 + char buf[50];
463 +
464 + buffer_json_add_array_item_string(wb, "in-progress");
465 +
466 + if (qp->all)
467 + snprintfz(buf, sizeof(buf), "%0.2f %%", (double) qp->done * 100.0 / (double) qp->all);
468 + else
469 + snprintfz(buf, sizeof(buf), "%zu", qp->done);
470 +
471 + buffer_json_add_array_item_string(wb, buf);
472 + }
473 +
474 + buffer_json_add_array_item_double(wb, (double)duration_ut / USEC_PER_MS);
475 +
476 + if(finished) {
477 + buffer_json_add_array_item_uint64(wb, qp->response_code);
478 + buffer_json_add_array_item_uint64(wb, qp->response_size);
479 + buffer_json_add_array_item_uint64(wb, qp->sent_size);
480 + }
481 + else {
482 + buffer_json_add_array_item_string(wb, NULL);
483 + buffer_json_add_array_item_string(wb, NULL);
484 + buffer_json_add_array_item_string(wb, NULL);
485 + }
486 +
487 + buffer_json_add_array_item_object(wb); // row options
488 + {
489 + char *severity = "notice";
490 + if(finished) {
491 + if(qp->response_code == HTTP_RESP_NOT_MODIFIED ||
492 + qp->response_code == HTTP_RESP_CLIENT_CLOSED_REQUEST ||
493 + qp->response_code == HTTP_RESP_CONFLICT)
494 + severity = "debug";
495 + else if(qp->response_code >= 500 && qp->response_code <= 599)
496 + severity = "error";
497 + else if(qp->response_code >= 400 && qp->response_code <= 499)
498 + severity = "warning";
499 + else if(qp->response_code >= 300 && qp->response_code <= 399)
500 + severity = "notice";
501 + else
502 + severity = "normal";
503 + }
504 + buffer_json_member_add_string(wb, "severity", severity);
505 + }
506 + buffer_json_object_close(wb); // row options
507 +
508 + buffer_json_array_close(wb); // row
509 + }
510 +
511 + assert(archived == progress.cache.available);
512 +
513 + spinlock_unlock(&progress.spinlock);
514 +
515 + buffer_json_array_close(wb); // data
516 + buffer_json_member_add_object(wb, "columns");
517 + {
518 + size_t field_id = 0;
519 +
520 + // transaction
521 + buffer_rrdf_table_add_field(wb, field_id++, "Transaction", "Transaction ID",
522 + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
523 + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
524 + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,
525 + RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY,
526 + NULL);
527 +
528 + // timestamp
529 + buffer_rrdf_table_add_field(wb, field_id++, "Started", "Query Start Timestamp",
530 + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_USEC,
531 + 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
532 + RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_NONE,
533 + RRDF_FIELD_OPTS_VISIBLE, NULL);
534 +
535 + // request method
536 + buffer_rrdf_table_add_field(wb, field_id++, "Method", "Request Method",
537 + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
538 + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
539 + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
540 + RRDF_FIELD_OPTS_VISIBLE, NULL);
541 +
542 + // query
543 + buffer_rrdf_table_add_field(wb, field_id++, "Query", "Query",
544 + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
545 + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
546 + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,
547 + RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP, NULL);
548 +
549 + // client
550 + buffer_rrdf_table_add_field(wb, field_id++, "Client", "Client",
551 + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
552 + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
553 + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
554 + RRDF_FIELD_OPTS_VISIBLE, NULL);
555 +
556 + // status
557 + buffer_rrdf_table_add_field(wb, field_id++, "Status", "Query Status",
558 + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
559 + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
560 + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
561 + RRDF_FIELD_OPTS_VISIBLE, NULL);
562 +
563 + // progress
564 + buffer_rrdf_table_add_field(wb, field_id++, "Progress", "Query Progress",
565 + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
566 + 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
567 + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,
568 + RRDF_FIELD_OPTS_VISIBLE, NULL);
569 +
570 + // duration
571 + buffer_rrdf_table_add_field(wb, field_id++, "Duration", "Query Duration",
572 + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER,
573 + 2, "ms", (double)max_duration_ut / USEC_PER_MS, RRDF_FIELD_SORT_DESCENDING, NULL,
574 + RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE,
575 + RRDF_FIELD_OPTS_VISIBLE, NULL);
576 +
577 + // response code
578 + buffer_rrdf_table_add_field(wb, field_id++, "Response", "Query Response Code",
579 + RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
580 + 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
581 + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT,
582 + RRDF_FIELD_OPTS_VISIBLE, NULL);
583 +
584 + // response size
585 + buffer_rrdf_table_add_field(wb, field_id++, "Size", "Query Response Size",
586 + RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
587 + 0, "bytes", (double)max_size, RRDF_FIELD_SORT_DESCENDING, NULL,
588 + RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
589 + RRDF_FIELD_OPTS_NONE, NULL);
590 +
591 + // sent size
592 + buffer_rrdf_table_add_field(wb, field_id++, "Sent", "Query Response Final Size",
593 + RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE,
594 + 0, "bytes", (double)max_sent, RRDF_FIELD_SORT_DESCENDING, NULL,
595 + RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE,
596 + RRDF_FIELD_OPTS_NONE, NULL);
597 +
598 + // row options
599 + buffer_rrdf_table_add_field(wb, field_id++, "rowOptions", "rowOptions",
600 + RRDF_FIELD_TYPE_NONE, RRDR_FIELD_VISUAL_ROW_OPTIONS, RRDF_FIELD_TRANSFORM_NONE,
601 + 0, NULL, NAN, RRDF_FIELD_SORT_FIXED, NULL,
602 + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE,
603 + RRDF_FIELD_OPTS_DUMMY, NULL);
604 + }
605 +
606 + buffer_json_object_close(wb); // columns
607 + buffer_json_member_add_string(wb, "default_sort_column", "Started");
608 +
609 + buffer_json_member_add_time_t(wb, "expires", (time_t)((now_ut / USEC_PER_SEC) + 1));
610 + buffer_json_finalize(wb);
611 +
612 + return 200;
613 +}
614 +
615 +
616 +// ----------------------------------------------------------------------------
617 +
618 +int progress_unittest(void) {
619 + size_t permanent = 100;
620 + uuid_t valid[permanent];
621 +
622 + usec_t started = now_monotonic_usec();
623 +
624 + for(size_t i = 0; i < permanent ;i++) {
625 + uuid_generate_random(valid[i]);
626 + query_progress_start_or_update(&valid[i], 0, HTTP_REQUEST_MODE_GET, HTTP_ACL_ACLK, "permanent", NULL, "test");
627 + }
628 +
629 + for(size_t n = 0; n < 5000000 ;n++) {
630 + uuid_t t;
631 + uuid_generate_random(t);
632 + query_progress_start_or_update(&t, 0, HTTP_REQUEST_MODE_OPTIONS, HTTP_ACL_WEBRTC, "ephemeral", NULL, "test");
633 + query_progress_finished(&t, 0, 200, 1234, 123, 12);
634 +
635 + QUERY_PROGRESS *qp;
636 + for(size_t i = 0; i < permanent ;i++) {
637 + qp = query_progress_find_in_hashtable_unsafe(&valid[i]);
638 + assert(qp);
639 + }
640 + }
641 +
642 + usec_t ended = now_monotonic_usec();
643 + usec_t duration = ended - started;
644 +
645 + printf("progress hashtable resizes: %zu, size: %zu, used: %zu, deleted: %zu, searches: %zu, collisions: %zu, additions: %zu, deletions: %zu\n",
646 + progress.hashtable.resizes,
647 + progress.hashtable.size, progress.hashtable.used, progress.hashtable.deleted,
648 + progress.hashtable.searches, progress.hashtable.collisions, progress.hashtable.additions, progress.hashtable.deletions);
649 +
650 + double d = (double)duration / USEC_PER_SEC;
651 + printf("hashtable ops: %0.2f / sec\n", (double)progress.hashtable.searches / d);
652 +
653 + return 0;
654 +}
libnetdata/query_progress/progress.h new
+19
@@ -0,0 +1,19 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_QUERY_PROGRESS_H
4 +#define NETDATA_QUERY_PROGRESS_H 1
5 +
6 +#include "../libnetdata.h"
7 +
8 +void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client);
9 +void query_progress_done_step(uuid_t *transaction, size_t done);
10 +void query_progress_set_finish_line(uuid_t *transaction, size_t all);
11 +void query_progress_finished(uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size);
12 +void query_progress_functions_update(uuid_t *transaction, size_t done, size_t all);
13 +
14 +int web_api_v2_report_progress(uuid_t *transaction, BUFFER *wb);
15 +
16 +#define RRDFUNCTIONS_PROGRESS_HELP "View the progress on the running and latest Netdata API Requests"
17 +int progress_function_result(BUFFER *wb, const char *hostname);
18 +
19 +#endif // NETDATA_QUERY_PROGRESS_H
libnetdata/simple_hashtable.h
+203 -55
@@ -3,14 +3,47 @@
3 #ifndef NETDATA_SIMPLE_HASHTABLE_H
4 #define NETDATA_SIMPLE_HASHTABLE_H
5
6 -#ifndef XXH_INLINE_ALL
7 -#define XXH_INLINE_ALL
8 -#endif
9 -#include "xxhash.h"
10 -
6 typedef uint64_t SIMPLE_HASHTABLE_HASH;
7 #define SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS 32
8
9 +/*
10 + * CONFIGURATION
11 + *
12 + * SIMPLE_HASHTABLE_NAME
13 + * The name of the hashtable - all functions and defines will have this name appended
14 + * Example: #define SIMPLE_HASHTABLE_NAME _FACET_KEY
15 + *
16 + * SIMPLE_HASHTABLE_VALUE_TYPE and SIMPLE_HASHTABLE_KEY_TYPE
17 + * The data types of values and keys - optional - setting them will enable strict type checking by the compiler.
18 + * If undefined, they both default to void.
19 + *
20 + * SIMPLE_HASHTABLE_SORT_FUNCTION
21 + * A function name that accepts 2x values and compares them for sorting (returning -1, 0, 1).
22 + * When set, the hashtable will maintain an always sorted array of the values in the hashtable.
23 + * Do not use this for non-static hashtables. So, if your data is changing all the time, this can make the
24 + * hashtable quite slower (it memmove()s an array of pointers to keep it sorted, on every single change).
25 + *
26 + * SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION and SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION
27 + * The hashtable can either compare just hashes (the default), or hashes and keys (when these are set).
28 + * Both need to be set for this feature to be enabled.
29 + *
30 + * - SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION
31 + * The name of a function accepting SIMPLE_HASHTABLE_VALUE_TYPE pointer.
32 + * It should return a pointer to SIMPLE_HASHTABLE_KEY_TYPE.
33 + * This function is called prior to SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION to extract the key from a value.
34 + * It is also called during hashtable resize, to rehash all values in the hashtable.
35 + *
36 + * - SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION
37 + * The name of a function accepting 2x SIMPLE_HASHTABLE_KEY_TYPE pointers.
38 + * It should return true when the keys match.
39 + * This function is only called when the hashes match, to verify that the keys also match.
40 + *
41 + * SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION
42 + * If defined, 3x functions will be injected for easily working with the hashtable.
43 + *
44 + */
45 +
46 +
47 #ifndef SIMPLE_HASHTABLE_NAME
48 #define SIMPLE_HASHTABLE_NAME
49 #endif
@@ -19,6 +52,22 @@ typedef uint64_t SIMPLE_HASHTABLE_HASH;
52 #define SIMPLE_HASHTABLE_VALUE_TYPE void
53 #endif
54
55 +#ifndef SIMPLE_HASHTABLE_KEY_TYPE
56 +#define SIMPLE_HASHTABLE_KEY_TYPE void
57 +#endif
58 +
59 +#ifndef SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION
60 +#undef SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION
61 +#endif
62 +
63 +#if defined(SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION)
64 +static inline SIMPLE_HASHTABLE_KEY_TYPE *SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION(SIMPLE_HASHTABLE_VALUE_TYPE *);
65 +#endif
66 +
67 +#if defined(SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION)
68 +static inline bool SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION(SIMPLE_HASHTABLE_KEY_TYPE *, SIMPLE_HASHTABLE_KEY_TYPE *);
69 +#endif
70 +
71 // First layer of macro for token concatenation
72 #define CONCAT_INTERNAL(a, b) a ## b
73 // Second layer of macro, which ensures proper expansion
@@ -33,6 +82,7 @@ typedef uint64_t SIMPLE_HASHTABLE_HASH;
82 #define simple_hashtable_named CONCAT(simple_hashtable, SIMPLE_HASHTABLE_NAME)
83 #define SIMPLE_HASHTABLE_NAMED CONCAT(SIMPLE_HASHTABLE, SIMPLE_HASHTABLE_NAME)
84 #define simple_hashtable_resize_named CONCAT(simple_hashtable_resize, SIMPLE_HASHTABLE_NAME)
85 +#define simple_hashtable_can_use_slot_named CONCAT(simple_hashtable_keys_match, SIMPLE_HASHTABLE_NAME)
86 #define simple_hashtable_get_slot_named CONCAT(simple_hashtable_get_slot, SIMPLE_HASHTABLE_NAME)
87 #define simple_hashtable_del_slot_named CONCAT(simple_hashtable_del_slot, SIMPLE_HASHTABLE_NAME)
88 #define simple_hashtable_set_slot_named CONCAT(simple_hashtable_set_slot, SIMPLE_HASHTABLE_NAME)
@@ -55,10 +105,12 @@ typedef struct simple_hashtable_named {
105 size_t resizes;
106 size_t searches;
107 size_t collisions;
108 + size_t additions;
109 size_t deletions;
110 size_t deleted;
111 size_t used;
112 size_t size;
113 + bool needs_cleanup;
114 SIMPLE_HASHTABLE_SLOT_NAMED *hashtable;
115
116 #ifdef SIMPLE_HASHTABLE_SORT_FUNCTION
@@ -210,19 +262,53 @@ static void simple_hashtable_destroy_named(SIMPLE_HASHTABLE_NAMED *ht) {
262
263 static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht);
264
213 -#define SHTS_DATA_UNSET ((void *)NULL)
214 -#define SHTS_DATA_DELETED ((void *)0x01)
215 -#define SHTS_DATA_USERNULL ((void *)0x02)
216 -#define SHTS_IS_UNSET(sl) ((sl)->data == SHTS_DATA_UNSET)
217 -#define SHTS_IS_DELETED(sl) ((sl)->data == SHTS_DATA_DELETED)
218 -#define SHTS_IS_USERNULL(sl) ((sl)->data == SHTS_DATA_USERNULL)
219 -#define SIMPLE_HASHTABLE_SLOT_DATA(sl) ((SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl) || SHTS_IS_USERNULL(sl)) ? NULL : (sl)->data)
220 -#define SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) ((SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl)) ? NULL : (sl)->data)
265 +#define simple_hashtable_data_unset ((void *)NULL)
266 +#define simple_hashtable_data_deleted ((void *)UINT64_MAX)
267 +#define simple_hashtable_data_usernull ((void *)(UINT64_MAX - 1))
268 +#define simple_hashtable_is_slot_unset(sl) ((sl)->data == simple_hashtable_data_unset)
269 +#define simple_hashtable_is_slot_deleted(sl) ((sl)->data == simple_hashtable_data_deleted)
270 +#define simple_hashtable_is_slot_usernull(sl) ((sl)->data == simple_hashtable_data_usernull)
271 +#define SIMPLE_HASHTABLE_SLOT_DATA(sl) ((simple_hashtable_is_slot_unset(sl) || simple_hashtable_is_slot_deleted(sl) || simple_hashtable_is_slot_usernull(sl)) ? NULL : (sl)->data)
272
222 -// IMPORTANT
223 -// The pointer returned by this call is valid up to the next call of this function (or the resize one)
273 +static inline bool simple_hashtable_can_use_slot_named(
274 + SIMPLE_HASHTABLE_SLOT_NAMED *sl, SIMPLE_HASHTABLE_HASH hash,
275 + SIMPLE_HASHTABLE_KEY_TYPE *key __maybe_unused) {
276 +
277 + if(simple_hashtable_is_slot_unset(sl))
278 + return true;
279 +
280 + if(simple_hashtable_is_slot_deleted(sl))
281 + return false;
282 +
283 + if(sl->hash == hash) {
284 +#if defined(SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION) && defined(SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION)
285 + return SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION(SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION(SIMPLE_HASHTABLE_SLOT_DATA(sl)), key);
286 +#else
287 + return true;
288 +#endif
289 + }
290 +
291 + return false;
292 +}
293 +
294 +#define SIMPLE_HASHTABLE_NEEDS_RESIZE(ht) ((ht)->size <= ((ht)->used - (ht)->deleted) << 1 || (ht)->used >= (ht)->size)
295 +
296 +// IMPORTANT: the pointer returned by this call is valid up to the next call of this function (or the resize one).
297 // If you need to cache something, cache the hash, not the slot pointer.
225 -static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_get_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_HASH hash, bool resize) {
298 +static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_get_slot_named(
299 + SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_HASH hash,
300 + SIMPLE_HASHTABLE_KEY_TYPE *key, bool resize) {
301 +
302 + // This function finds the requested hash and key in the hashtable.
303 + // It uses a second version of the hash in case of collisions, and then linear probing.
304 + // It may resize the hashtable if it is more than 50% full.
305 +
306 + // Deleted items remain in the hashtable, but they are marked as DELETED.
307 + // Reuse of DELETED slots happens only if the slot to be returned is UNSET.
308 + // So, when looking up for an item, it tries to find it, assuming DELETED
309 + // slots are occupied. If the item to be returned is UNSET, and it has
310 + // encountered a DELETED slot, it returns the DELETED one instead of the UNSET.
311 +
312 ht->searches++;
313
314 size_t slot;
@@ -231,41 +317,68 @@ static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_get_slot_named(SIMPL
317
318 slot = hash % ht->size;
319 sl = &ht->hashtable[slot];
234 - deleted = SHTS_IS_DELETED(sl) ? sl : NULL;
235 - if(likely(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) || sl->hash == hash))
236 - return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
320 + deleted = simple_hashtable_is_slot_deleted(sl) ? sl : NULL;
321 + if(likely(simple_hashtable_can_use_slot_named(sl, hash, key)))
322 + return (simple_hashtable_is_slot_unset(sl) && deleted) ? deleted : sl;
323
324 ht->collisions++;
325
240 - if(unlikely(resize && (ht->size <= (ht->used << 1) || ht->used >= ht->size))) {
326 + if(unlikely(resize && (ht->needs_cleanup || SIMPLE_HASHTABLE_NEEDS_RESIZE(ht)))) {
327 simple_hashtable_resize_named(ht);
328 + deleted = NULL; // our deleted pointer is not valid anymore
329
330 slot = hash % ht->size;
331 sl = &ht->hashtable[slot];
245 - deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
246 - if(likely(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) || sl->hash == hash))
247 - return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
332 + if(likely(simple_hashtable_can_use_slot_named(sl, hash, key)))
333 + return sl;
334
335 ht->collisions++;
336 }
337
338 slot = ((hash >> SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS) + 1) % ht->size;
339 sl = &ht->hashtable[slot];
254 - deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
340 + deleted = (!deleted && simple_hashtable_is_slot_deleted(sl)) ? sl : deleted;
341
342 // Linear probing until we find it
257 - while (SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) && sl->hash != hash) {
343 + SIMPLE_HASHTABLE_SLOT_NAMED *sl_started = sl;
344 + size_t collisions_started = ht->collisions;
345 + while (!simple_hashtable_can_use_slot_named(sl, hash, key)) {
346 slot = (slot + 1) % ht->size; // Wrap around if necessary
347 sl = &ht->hashtable[slot];
260 - deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
348 + deleted = (!deleted && simple_hashtable_is_slot_deleted(sl)) ? sl : deleted;
349 ht->collisions++;
350 +
351 + if(sl == sl_started) {
352 + if(deleted) {
353 + // we looped through all items, and we didn't find a free slot,
354 + // but we have found a deleted slot, so return it.
355 + return deleted;
356 + }
357 + else if(resize) {
358 + // the hashtable is full, without any deleted slots.
359 + // we need to resize it now.
360 + simple_hashtable_resize_named(ht);
361 + return simple_hashtable_get_slot_named(ht, hash, key, false);
362 + }
363 + else {
364 + // the hashtable is full, but resize is false.
365 + // this should never happen.
366 + assert(sl != sl_started);
367 + }
368 + }
369 }
370
264 - return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
371 + if((ht->collisions - collisions_started) > (ht->size / 2) && ht->deleted >= (ht->size / 3)) {
372 + // we traversed through half of the hashtable to find a slot,
373 + // but we have more than 1/3 deleted items
374 + ht->needs_cleanup = true;
375 + }
376 +
377 + return (simple_hashtable_is_slot_unset(sl) && deleted) ? deleted : sl;
378 }
379
380 static inline bool simple_hashtable_del_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl) {
268 - if(SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl))
381 + if(simple_hashtable_is_slot_unset(sl) || simple_hashtable_is_slot_deleted(sl))
382 return false;
383
384 ht->deletions++;
@@ -273,25 +386,28 @@ static inline bool simple_hashtable_del_slot_named(SIMPLE_HASHTABLE_NAMED *ht, S
386
387 simple_hashtable_del_value_sorted_named(ht, SIMPLE_HASHTABLE_SLOT_DATA(sl));
388
276 - sl->data = SHTS_DATA_DELETED;
389 + sl->data = simple_hashtable_data_deleted;
390 return true;
391 }
392
280 -static inline void simple_hashtable_set_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl, SIMPLE_HASHTABLE_HASH hash, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
393 +static inline void simple_hashtable_set_slot_named(
394 + SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl,
395 + SIMPLE_HASHTABLE_HASH hash, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
396 +
397 if(data == NULL)
282 - data = SHTS_DATA_USERNULL;
398 + data = simple_hashtable_data_usernull;
399
284 - if(unlikely(data == SHTS_DATA_UNSET || data == SHTS_DATA_DELETED)) {
400 + if(unlikely(data == simple_hashtable_data_unset || data == simple_hashtable_data_deleted)) {
401 simple_hashtable_del_slot_named(ht, sl);
402 return;
403 }
404
289 - if(likely(SHTS_IS_UNSET(sl))) {
405 + if(likely(simple_hashtable_is_slot_unset(sl))) {
406 simple_hashtable_add_value_sorted_named(ht, data);
407 ht->used++;
408 }
409
294 - else if(unlikely(SHTS_IS_DELETED(sl))) {
410 + else if(unlikely(simple_hashtable_is_slot_deleted(sl))) {
411 ht->deleted--;
412 }
413
@@ -300,6 +416,7 @@ static inline void simple_hashtable_set_slot_named(SIMPLE_HASHTABLE_NAMED *ht, S
416
417 sl->hash = hash;
418 sl->data = data;
419 + ht->additions++;
420 }
421
422 // IMPORTANT
@@ -308,19 +425,38 @@ static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht) {
425 SIMPLE_HASHTABLE_SLOT_NAMED *old = ht->hashtable;
426 size_t old_size = ht->size;
427
428 + size_t new_size = ht->size;
429 +
430 + if(SIMPLE_HASHTABLE_NEEDS_RESIZE(ht))
431 + new_size = (ht->size << 1) - ((ht->size > 16) ? 1 : 0);
432 +
433 ht->resizes++;
312 - ht->size = (ht->size << 1) - ((ht->size > 16) ? 1 : 0);
313 - ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
314 - ht->used = ht->deleted = 0;
434 + ht->size = new_size;
435 + ht->hashtable = callocz(new_size, sizeof(*ht->hashtable));
436 + size_t used = 0;
437 for(size_t i = 0 ; i < old_size ; i++) {
316 - if(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(&old[i]))
438 + SIMPLE_HASHTABLE_SLOT_NAMED *slot = &old[i];
439 + if(simple_hashtable_is_slot_unset(slot) || simple_hashtable_is_slot_deleted(slot))
440 continue;
441
319 - SIMPLE_HASHTABLE_SLOT_NAMED *slot = simple_hashtable_get_slot_named(ht, old[i].hash, false);
320 - *slot = old[i];
321 - ht->used++;
442 + SIMPLE_HASHTABLE_KEY_TYPE *key = NULL;
443 +
444 +#if defined(SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION) && defined(SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION)
445 + SIMPLE_HASHTABLE_VALUE_TYPE *value = SIMPLE_HASHTABLE_SLOT_DATA(slot);
446 + key = SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION(value);
447 +#endif
448 +
449 + SIMPLE_HASHTABLE_SLOT_NAMED *slot2 = simple_hashtable_get_slot_named(ht, slot->hash, key, false);
450 + *slot2 = *slot;
451 + used++;
452 }
453
454 + assert(used == ht->used - ht->deleted);
455 +
456 + ht->used = used;
457 + ht->deleted = 0;
458 + ht->needs_cleanup = false;
459 +
460 freez(old);
461 }
462
@@ -329,9 +465,9 @@ static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht) {
465 // the hashtable should not be modified while the traversal is taking place
466
467 static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_first_read_only_named(SIMPLE_HASHTABLE_NAMED *ht) {
332 - for(size_t i = 0; i < ht->used ;i++) {
468 + for(size_t i = 0; i < ht->size ;i++) {
469 SIMPLE_HASHTABLE_SLOT_NAMED *sl = &ht->hashtable[i];
334 - if(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl))
470 + if(!simple_hashtable_is_slot_unset(sl) && !simple_hashtable_is_slot_deleted(sl))
471 return sl;
472 }
473
@@ -342,12 +478,12 @@ static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_next_read_only_named
478 if (!last) return NULL;
479
480 // Calculate the current position in the array
345 - size_t currentIndex = last - ht->hashtable;
481 + size_t index = last - ht->hashtable;
482
483 // Iterate over the hashtable starting from the next element
348 - for (size_t i = currentIndex + 1; i < ht->size; i++) {
484 + for (size_t i = index + 1; i < ht->size; i++) {
485 SIMPLE_HASHTABLE_SLOT_NAMED *sl = &ht->hashtable[i];
350 - if (!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl)) {
486 + if (!simple_hashtable_is_slot_unset(sl) && !simple_hashtable_is_slot_deleted(sl)) {
487 return sl;
488 }
489 }
@@ -368,29 +504,41 @@ static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_next_read_only_named
504
505 #ifdef SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION
506
507 +#ifndef XXH_INLINE_ALL
508 +#define XXH_INLINE_ALL
509 +#endif
510 +#include "xxhash.h"
511 +
512 #define simple_hashtable_set_named CONCAT(simple_hashtable_set, SIMPLE_HASHTABLE_NAME)
513 #define simple_hashtable_get_named CONCAT(simple_hashtable_get, SIMPLE_HASHTABLE_NAME)
514 #define simple_hashtable_del_named CONCAT(simple_hashtable_del, SIMPLE_HASHTABLE_NAME)
515
375 -static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_set_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
376 - XXH64_hash_t hash = XXH3_64bits(key, key_len);
377 - SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
516 +static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_set_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
517 + XXH64_hash_t hash = XXH3_64bits((void *)key, key_len);
518 + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true);
519 simple_hashtable_set_slot_named(ht, sl, hash, data);
520 return SIMPLE_HASHTABLE_SLOT_DATA(sl);
521 }
522
382 -static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
383 - XXH64_hash_t hash = XXH3_64bits(key, key_len);
384 - SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
523 +static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
524 + XXH64_hash_t hash = XXH3_64bits((void *)key, key_len);
525 + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true);
526 return SIMPLE_HASHTABLE_SLOT_DATA(sl);
527 }
528
388 -static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
389 - XXH64_hash_t hash = XXH3_64bits(key, key_len);
390 - SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
529 +static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
530 + XXH64_hash_t hash = XXH3_64bits((void *)key, key_len);
531 + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true);
532 return simple_hashtable_del_slot_named(ht, sl);
533 }
534
535 #endif // SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION
536
537 +// ----------------------------------------------------------------------------
538 +// Clear the preprocessor defines of simple_hashtable.h
539 +// allowing simple_hashtable.h to be included multiple times
540 +// with different configuration each time.
541 +
542 +#include "simple_hashtable_undef.h"
543 +
544 #endif //NETDATA_SIMPLE_HASHTABLE_H
libnetdata/simple_hashtable_undef.h new
+35
@@ -0,0 +1,35 @@
1 +
2 +// this file clears the preprocessor defines of simple_hashtable.h
3 +// allowing simple_hashtable.h to be included multiple times
4 +// with different configuration each time.
5 +
6 +#undef SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS
7 +
8 +#undef simple_hashtable_init_named
9 +#undef simple_hashtable_destroy_named
10 +#undef simple_hashtable_slot_named
11 +#undef SIMPLE_HASHTABLE_SLOT_NAMED
12 +#undef simple_hashtable_named
13 +#undef SIMPLE_HASHTABLE_NAMED
14 +#undef simple_hashtable_resize_named
15 +#undef simple_hashtable_can_use_slot_named
16 +#undef simple_hashtable_get_slot_named
17 +#undef simple_hashtable_del_slot_named
18 +#undef simple_hashtable_set_slot_named
19 +#undef simple_hashtable_first_read_only_named
20 +#undef simple_hashtable_next_read_only_named
21 +#undef simple_hashtable_sorted_binary_search_named
22 +#undef simple_hashtable_add_value_sorted_named
23 +#undef simple_hashtable_del_value_sorted_named
24 +#undef simple_hashtable_replace_value_sorted_named
25 +#undef simple_hashtable_sorted_array_first_read_only_named
26 +#undef simple_hashtable_sorted_array_next_read_only_named
27 +
28 +#undef SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION
29 +#undef SIMPLE_HASHTABLE_SORT_FUNCTION
30 +#undef SIMPLE_HASHTABLE_VALUE_TYPE
31 +#undef SIMPLE_HASHTABLE_KEY_TYPE
32 +#undef SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION
33 +#undef SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION
34 +#undef SIMPLE_HASHTABLE_NAME
35 +#undef NETDATA_SIMPLE_HASHTABLE_H
libnetdata/socket/socket.c
+18 -18
@@ -500,7 +500,7 @@ void listen_sockets_close(LISTEN_SOCKETS *sockets) {
500 *
501 * @param acl is the acl given by the user.
502 */
503 -WEB_CLIENT_ACL socket_ssl_acl(char *acl) {
503 +HTTP_ACL socket_ssl_acl(char *acl) {
504 char *ssl = strchr(acl,'^');
505 if(ssl) {
506 //Due the format of the SSL command it is always the last command,
@@ -511,34 +511,34 @@ WEB_CLIENT_ACL socket_ssl_acl(char *acl) {
511 if (!strncmp("SSL=",ssl,4)) {
512 ssl += 4;
513 if (!strcmp(ssl,"optional")) {
514 - return WEB_CLIENT_ACL_SSL_OPTIONAL;
514 + return HTTP_ACL_SSL_OPTIONAL;
515 }
516 else if (!strcmp(ssl,"force")) {
517 - return WEB_CLIENT_ACL_SSL_FORCE;
517 + return HTTP_ACL_SSL_FORCE;
518 }
519 }
520 #endif
521 }
522
523 - return WEB_CLIENT_ACL_NONE;
523 + return HTTP_ACL_NONE;
524 }
525
526 -WEB_CLIENT_ACL read_acl(char *st) {
527 - WEB_CLIENT_ACL ret = socket_ssl_acl(st);
526 +HTTP_ACL read_acl(char *st) {
527 + HTTP_ACL ret = socket_ssl_acl(st);
528
529 - if (!strcmp(st,"dashboard")) ret |= WEB_CLIENT_ACL_DASHBOARD;
530 - if (!strcmp(st,"registry")) ret |= WEB_CLIENT_ACL_REGISTRY;
531 - if (!strcmp(st,"badges")) ret |= WEB_CLIENT_ACL_BADGE;
532 - if (!strcmp(st,"management")) ret |= WEB_CLIENT_ACL_MGMT;
533 - if (!strcmp(st,"streaming")) ret |= WEB_CLIENT_ACL_STREAMING;
534 - if (!strcmp(st,"netdata.conf")) ret |= WEB_CLIENT_ACL_NETDATACONF;
529 + if (!strcmp(st,"dashboard")) ret |= HTTP_ACL_DASHBOARD;
530 + if (!strcmp(st,"registry")) ret |= HTTP_ACL_REGISTRY;
531 + if (!strcmp(st,"badges")) ret |= HTTP_ACL_BADGE;
532 + if (!strcmp(st,"management")) ret |= HTTP_ACL_MGMT;
533 + if (!strcmp(st,"streaming")) ret |= HTTP_ACL_STREAMING;
534 + if (!strcmp(st,"netdata.conf")) ret |= HTTP_ACL_NETDATACONF;
535
536 return ret;
537 }
538
539 static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, uint16_t default_port, int listen_backlog) {
540 int added = 0;
541 - WEB_CLIENT_ACL acl_flags = WEB_CLIENT_ACL_NONE;
541 + HTTP_ACL acl_flags = HTTP_ACL_NONE;
542
543 struct addrinfo hints;
544 struct addrinfo *result = NULL, *rp = NULL;
@@ -578,7 +578,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition,
578
579 sockets->failed++;
580 } else {
581 - acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT;
581 + acl_flags = HTTP_ACL_DASHBOARD | HTTP_ACL_REGISTRY | HTTP_ACL_BADGE | HTTP_ACL_MGMT | HTTP_ACL_NETDATACONF | HTTP_ACL_STREAMING | HTTP_ACL_SSL_DEFAULT;
582 listen_sockets_add(sockets, fd, AF_UNIX, socktype, protocol_str, path, 0, acl_flags);
583 added++;
584 }
@@ -628,13 +628,13 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition,
628 }
629 acl_flags |= read_acl(portconfig);
630 } else {
631 - acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT;
631 + acl_flags = HTTP_ACL_DASHBOARD | HTTP_ACL_REGISTRY | HTTP_ACL_BADGE | HTTP_ACL_MGMT | HTTP_ACL_NETDATACONF | HTTP_ACL_STREAMING | HTTP_ACL_SSL_DEFAULT;
632 }
633
634 //Case the user does not set the option SSL in the "bind to", but he has
635 //the certificates, I must redirect, so I am assuming here the default option
636 - if(!(acl_flags & WEB_CLIENT_ACL_SSL_OPTIONAL) && !(acl_flags & WEB_CLIENT_ACL_SSL_FORCE)) {
637 - acl_flags |= WEB_CLIENT_ACL_SSL_DEFAULT;
636 + if(!(acl_flags & HTTP_ACL_SSL_OPTIONAL) && !(acl_flags & HTTP_ACL_SSL_FORCE)) {
637 + acl_flags |= HTTP_ACL_SSL_DEFAULT;
638 }
639
640 uint32_t scope_id = 0;
@@ -1463,7 +1463,7 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien
1463 inline POLLINFO *poll_add_fd(POLLJOB *p
1464 , int fd
1465 , int socktype
1466 - , WEB_CLIENT_ACL port_acl
1466 + , HTTP_ACL port_acl
1467 , uint32_t flags
1468 , const char *client_ip
1469 , const char *client_port
libnetdata/socket/socket.h
+3 -42
@@ -9,45 +9,6 @@
9 #define MAX_LISTEN_FDS 50
10 #endif
11
12 -typedef enum web_client_acl {
13 - WEB_CLIENT_ACL_NONE = (0),
14 - WEB_CLIENT_ACL_NOCHECK = (1 << 0), // Don't check anything - this should work on all channels
15 - WEB_CLIENT_ACL_DASHBOARD = (1 << 1),
16 - WEB_CLIENT_ACL_REGISTRY = (1 << 2),
17 - WEB_CLIENT_ACL_BADGE = (1 << 3),
18 - WEB_CLIENT_ACL_MGMT = (1 << 4),
19 - WEB_CLIENT_ACL_STREAMING = (1 << 5),
20 - WEB_CLIENT_ACL_NETDATACONF = (1 << 6),
21 - WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 7),
22 - WEB_CLIENT_ACL_SSL_FORCE = (1 << 8),
23 - WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 9),
24 - WEB_CLIENT_ACL_ACLK = (1 << 10),
25 - WEB_CLIENT_ACL_WEBRTC = (1 << 11),
26 - WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 12), // allow unprotected access if bearer is not enabled in netdata
27 - WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 13), // allow access only if a valid bearer is used
28 -} WEB_CLIENT_ACL;
29 -
30 -#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_OPTIONAL)
31 -#define WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_REQUIRED)
32 -
33 -#ifdef NETDATA_DEV_MODE
34 -#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_NOCHECK
35 -#else
36 -#define ACL_DEV_OPEN_ACCESS 0
37 -#endif
38 -
39 -#define WEB_CLIENT_ACL_ALL 0xFFFF
40 -
41 -#define web_client_can_access_dashboard(w) ((w)->acl & WEB_CLIENT_ACL_DASHBOARD)
42 -#define web_client_can_access_registry(w) ((w)->acl & WEB_CLIENT_ACL_REGISTRY)
43 -#define web_client_can_access_badges(w) ((w)->acl & WEB_CLIENT_ACL_BADGE)
44 -#define web_client_can_access_mgmt(w) ((w)->acl & WEB_CLIENT_ACL_MGMT)
45 -#define web_client_can_access_stream(w) ((w)->acl & WEB_CLIENT_ACL_STREAMING)
46 -#define web_client_can_access_netdataconf(w) ((w)->acl & WEB_CLIENT_ACL_NETDATACONF)
47 -#define web_client_is_using_ssl_optional(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_OPTIONAL)
48 -#define web_client_is_using_ssl_force(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_FORCE)
49 -#define web_client_is_using_ssl_default(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_DEFAULT)
50 -
12 typedef struct listen_sockets {
13 struct config *config; // the config file to use
14 const char *config_section; // the netdata configuration section to read settings from
@@ -61,7 +22,7 @@ typedef struct listen_sockets {
22 char *fds_names[MAX_LISTEN_FDS]; // descriptions for the open sockets
23 int fds_types[MAX_LISTEN_FDS]; // the socktype for the open sockets (SOCK_STREAM, SOCK_DGRAM)
24 int fds_families[MAX_LISTEN_FDS]; // the family of the open sockets (AF_UNIX, AF_INET, AF_INET6)
64 - WEB_CLIENT_ACL fds_acl_flags[MAX_LISTEN_FDS]; // the acl to apply to the open sockets (dashboard, badges, streaming, netdata.conf, management)
25 + HTTP_ACL fds_acl_flags[MAX_LISTEN_FDS]; // the acl to apply to the open sockets (dashboard, badges, streaming, netdata.conf, management)
26 } LISTEN_SOCKETS;
27
28 char *strdup_client_description(int family, const char *protocol, const char *ip, uint16_t port);
@@ -128,7 +89,7 @@ typedef struct pollinfo {
89
90 int fd; // the file descriptor
91 int socktype; // the client socket type
131 - WEB_CLIENT_ACL port_acl; // the access lists permitted on this web server port (it's -1 for client sockets)
92 + HTTP_ACL port_acl; // the access lists permitted on this web server port (it's -1 for client sockets)
93 char *client_ip; // Max INET6_ADDRSTRLEN bytes
94 char *client_port; // Max NI_MAXSERV bytes
95 char *client_host; // Max NI_MAXHOST bytes
@@ -196,7 +157,7 @@ void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data);
157 POLLINFO *poll_add_fd(POLLJOB *p
158 , int fd
159 , int socktype
199 - , WEB_CLIENT_ACL port_acl
160 + , HTTP_ACL port_acl
161 , uint32_t flags
162 , const char *client_ip
163 , const char *client_port
logsmanagement/functions.c
+15 -52
@@ -25,8 +25,6 @@
25 #define LOGS_MANAG_FUNC_PARAM_DATA_ONLY "data_only"
26 #define LOGS_MANAG_FUNC_PARAM_SOURCE "source"
27 #define LOGS_MANAG_FUNC_PARAM_INFO "info"
28 -#define LOGS_MANAG_FUNC_PARAM_ID "id"
29 -#define LOGS_MANAG_FUNC_PARAM_PROGRESS "progress"
28 #define LOGS_MANAG_FUNC_PARAM_SLICE "slice"
29 #define LOGS_MANAG_FUNC_PARAM_DELTA "delta"
30 #define LOGS_MANAG_FUNC_PARAM_TAIL "tail"
@@ -116,9 +114,7 @@ static DICTIONARY *used_hashes_registry = NULL;
114
115 typedef struct function_query_status {
116 bool *cancelled; // a pointer to the cancelling boolean
119 - usec_t stop_monotonic_ut;
120 -
121 - usec_t started_monotonic_ut;
117 + usec_t *stop_monotonic_ut;
118
119 // request
120 STRING *source;
@@ -164,7 +160,7 @@ typedef struct function_query_status {
160 "|message" \
161 ""
162
167 -static void logsmanagement_function_facets(const char *transaction, char *function, int timeout, bool *cancelled){
163 +static void logsmanagement_function_facets(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled){
164
165 struct rusage start, end;
166 getrusage(RUSAGE_THREAD, &start);
@@ -175,11 +171,9 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
171 buffer_flush(wb);
172 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
173
178 - usec_t now_monotonic_ut = now_monotonic_usec();
174 FUNCTION_QUERY_STATUS tmp_fqs = {
175 .cancelled = cancelled,
181 - .started_monotonic_ut = now_monotonic_ut,
182 - .stop_monotonic_ut = now_monotonic_ut + (timeout * USEC_PER_SEC),
176 + .stop_monotonic_ut = stop_monotonic_ut,
177 };
178 FUNCTION_QUERY_STATUS *fqs = NULL;
179 const DICTIONARY_ITEM *fqs_item = NULL;
@@ -201,8 +195,6 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
195 facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_HISTOGRAM);
196 facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_IF_MODIFIED_SINCE);
197 facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_DATA_ONLY);
204 - // facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_ID);
205 - // facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_PROGRESS);
198 facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_DELTA);
199 // facets_accepted_param(facets, JOURNAL_PARAMETER_TAIL);
200
@@ -235,7 +227,6 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
227
228 bool info = false,
229 data_only = false,
238 - progress = false,
230 /* slice = true, */
231 delta = false,
232 tail = false;
@@ -247,8 +238,7 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
238 const char *query = NULL;
239 const char *chart = NULL;
240 const char *source = NULL;
250 - const char *progress_id = NULL;
251 - // size_t filters = 0;
241 + // size_t filters = 0;
242
243 buffer_json_member_add_object(wb, "_request");
244
@@ -265,20 +255,17 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
255
256
257 if(!strcmp(keyword, LOGS_MANAG_FUNC_PARAM_HELP)){
268 - BUFFER *wb = buffer_create(0, NULL);
269 - buffer_sprintf(wb, FUNCTION_LOGSMANAGEMENT_HELP_LONG);
258 + BUFFER *tmp = buffer_create(0, NULL);
259 + buffer_sprintf(tmp, FUNCTION_LOGSMANAGEMENT_HELP_LONG);
260 netdata_mutex_lock(&stdout_mut);
271 - pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb);
261 + pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, tmp);
262 netdata_mutex_unlock(&stdout_mut);
273 - buffer_free(wb);
263 + buffer_free(tmp);
264 goto cleanup;
265 }
266 else if(!strcmp(keyword, LOGS_MANAG_FUNC_PARAM_INFO)){
267 info = true;
268 }
279 - else if(!strcmp(keyword, LOGS_MANAG_FUNC_PARAM_PROGRESS)){
280 - progress = true;
281 - }
269 else if(strncmp(keyword, LOGS_MANAG_FUNC_PARAM_DELTA ":", sizeof(LOGS_MANAG_FUNC_PARAM_DELTA ":") - 1) == 0) {
270 char *v = &keyword[sizeof(LOGS_MANAG_FUNC_PARAM_DELTA ":") - 1];
271
@@ -314,13 +301,6 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
301 // else
302 // slice = true;
303 // }
317 - else if(strncmp(keyword, LOGS_MANAG_FUNC_PARAM_ID ":", sizeof(LOGS_MANAG_FUNC_PARAM_ID ":") - 1) == 0) {
318 - char *id = &keyword[sizeof(LOGS_MANAG_FUNC_PARAM_ID ":") - 1];
319 -
320 - if(*id)
321 - progress_id = id;
322 - }
323 -
304 else if(strncmp(keyword, LOGS_MANAG_FUNC_PARAM_SOURCE ":", sizeof(LOGS_MANAG_FUNC_PARAM_SOURCE ":") - 1) == 0) {
305 source = !strcmp("all", &keyword[sizeof(LOGS_MANAG_FUNC_PARAM_SOURCE ":") - 1]) ?
306 NULL : &keyword[sizeof(LOGS_MANAG_FUNC_PARAM_SOURCE ":") - 1];
@@ -393,18 +373,8 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
373 }
374 }
375
396 - // ------------------------------------------------------------------------
397 - // put this request into the progress db
398 -
399 - if(progress_id && *progress_id) {
400 - fqs_item = dictionary_set_and_acquire_item(function_query_status_dict, progress_id, &tmp_fqs, sizeof(tmp_fqs));
401 - fqs = dictionary_acquired_item_value(fqs_item);
402 - }
403 - else {
404 - // no progress id given, proceed without registering our progress in the dictionary
405 - fqs = &tmp_fqs;
406 - fqs_item = NULL;
407 - }
376 + fqs = &tmp_fqs;
377 + fqs_item = NULL;
378
379 // ------------------------------------------------------------------------
380 // validate parameters
@@ -508,10 +478,8 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
478 buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_INFO, false);
479 buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_SLICE, fqs->slice);
480 buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_DATA_ONLY, fqs->data_only);
511 - buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_PROGRESS, false);
481 buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_DELTA, fqs->delta);
482 buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_TAIL, fqs->tail);
514 - buffer_json_member_add_string(wb, LOGS_MANAG_FUNC_PARAM_ID, progress_id);
483 buffer_json_member_add_string(wb, LOGS_MANAG_FUNC_PARAM_SOURCE, string2str(fqs->source));
484 buffer_json_member_add_uint64(wb, LOGS_MANAG_FUNC_PARAM_AFTER, fqs->after_ut / USEC_PER_SEC);
485 buffer_json_member_add_uint64(wb, LOGS_MANAG_FUNC_PARAM_BEFORE, fqs->before_ut / USEC_PER_SEC);
@@ -556,13 +524,7 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
524 goto output;
525 }
526
559 - if(progress) {
560 - // TODO: Add progress function
561 - // function_logsmanagement_progress(wb, transaction, progress_id);
562 - goto cleanup;
563 - }
564 -
565 - if(!req_quota)
527 + if(!req_quota)
528 query_params.quota = LOGS_MANAG_QUERY_QUOTA_DEFAULT;
529 else if(req_quota > LOGS_MANAG_QUERY_QUOTA_MAX)
530 query_params.quota = LOGS_MANAG_QUERY_QUOTA_MAX;
@@ -590,7 +552,7 @@ static void logsmanagement_function_facets(const char *transaction, char *functi
552 }
553
554 query_params.cancelled = cancelled;
593 - query_params.stop_monotonic_ut = now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC;
555 + query_params.stop_monotonic_ut = stop_monotonic_ut;
556 query_params.results_buff = buffer_create(query_params.quota, NULL);
557
558 facets_rows_begin(facets);
@@ -736,10 +698,11 @@ struct functions_evloop_globals *logsmanagement_func_facets_init(bool *p_logsman
698 used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
699
700 netdata_mutex_lock(&stdout_mut);
739 - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n",
701 + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" \"members\" %d\n",
702 LOGS_MANAG_FUNC_NAME,
703 LOGS_MANAG_QUERY_TIMEOUT_DEFAULT,
742 - FUNCTION_LOGSMANAGEMENT_HELP_SHORT);
704 + FUNCTION_LOGSMANAGEMENT_HELP_SHORT,
705 + RRDFUNCTIONS_PRIORITY_DEFAULT + 1);
706 netdata_mutex_unlock(&stdout_mut);
707
708 struct functions_evloop_globals *wg = functions_evloop_init(1, "LGSMNGM",
logsmanagement/query.c
+3 -4
@@ -6,7 +6,9 @@
6 * logs management querying API.
7 */
8
9 +#ifndef _GNU_SOURCE
10 #define _GNU_SOURCE
11 +#endif
12
13 #include "query.h"
14 #include <uv.h>
@@ -107,7 +109,7 @@ bool terminate_logs_manag_query(logs_query_params_t *const p_query_params){
109 return true;
110 }
111
110 - if(now_monotonic_usec() > p_query_params->stop_monotonic_ut)
112 + if(now_monotonic_usec() > __atomic_load_n(p_query_params->stop_monotonic_ut, __ATOMIC_RELAXED))
113 return true;
114
115 return false;
@@ -173,9 +175,6 @@ const logs_qry_res_err_t *execute_logs_manag_query(logs_query_params_t *p_query_
175 p_query_params->keyword = sanitise_string(p_query_params->keyword); // freez(p_query_params->keyword) in this case
176 }
177
176 - if(p_query_params->stop_monotonic_ut == 0)
177 - p_query_params->stop_monotonic_ut = now_monotonic_usec() + (LOGS_MANAG_QUERY_TIMEOUT_DEFAULT - 1) * USEC_PER_SEC;
178 -
178 struct rusage ru_start, ru_end;
179 getrusage(RUSAGE_THREAD, &ru_start);
180
logsmanagement/query.h
+1 -1
@@ -113,7 +113,7 @@ typedef struct logs_query_params {
113 int order_by_asc;
114 unsigned long quota;
115 bool *cancelled;
116 - usec_t stop_monotonic_ut;
116 + usec_t *stop_monotonic_ut;
117 char *chartname[LOGS_MANAG_MAX_COMPOUND_QUERY_SOURCES];
118 char *filename[LOGS_MANAG_MAX_COMPOUND_QUERY_SOURCES];
119 char *keyword;
logsmanagement/unit_test/unit_test.c
+4
@@ -7,7 +7,11 @@
7 #include "unit_test.h"
8 #include <stdlib.h>
9 #include <stdio.h>
10 +
11 +#ifndef __USE_XOPEN_EXTENDED
12 #define __USE_XOPEN_EXTENDED
13 +#endif
14 +
15 #include <ftw.h>
16 #include <unistd.h>
17 #include "../circular_buffer.h"
streaming/rrdpush.c
+4 -3
@@ -208,14 +208,14 @@ int configured_as_parent() {
208 // chart labels
209 static int send_clabels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
210 BUFFER *wb = (BUFFER *)data;
211 - buffer_sprintf(wb, "CLABEL \"%s\" \"%s\" %d\n", name, value, ls & ~(RRDLABEL_FLAG_INTERNAL));
211 + buffer_sprintf(wb, PLUGINSD_KEYWORD_CLABEL " \"%s\" \"%s\" %d\n", name, value, ls & ~(RRDLABEL_FLAG_INTERNAL));
212 return 1;
213 }
214
215 static void rrdpush_send_clabels(BUFFER *wb, RRDSET *st) {
216 if (st->rrdlabels) {
217 if(rrdlabels_walkthrough_read(st->rrdlabels, send_clabels_callback, wb) > 0)
218 - buffer_sprintf(wb, "CLABEL_COMMIT\n");
218 + buffer_sprintf(wb, PLUGINSD_KEYWORD_CLABEL_COMMIT "\n");
219 }
220 }
221
@@ -1410,6 +1410,7 @@ static struct {
1410 {STREAM_CAP_ZSTD, "ZSTD" },
1411 {STREAM_CAP_GZIP, "GZIP" },
1412 {STREAM_CAP_BROTLI, "BROTLI" },
1413 + {STREAM_CAP_PROGRESS, "PROGRESS" },
1414 {0 , NULL },
1415 };
1416
@@ -1485,7 +1486,7 @@ STREAM_CAPABILITIES stream_our_capabilities(RRDHOST *host, bool sender) {
1486 STREAM_CAP_REPLICATION |
1487 STREAM_CAP_BINARY |
1488 STREAM_CAP_INTERPOLATED |
1488 - STREAM_CAP_SLOTS |
1489 + STREAM_CAP_SLOTS | STREAM_CAP_PROGRESS |
1490 STREAM_CAP_COMPRESSIONS_AVAILABLE |
1491 #ifdef NETDATA_TEST_DYNCFG
1492 STREAM_CAP_DYNCFG |
streaming/rrdpush.h
+1
@@ -52,6 +52,7 @@ typedef enum {
52 STREAM_CAP_ZSTD = (1 << 19), // ZSTD compression supported
53 STREAM_CAP_GZIP = (1 << 20), // GZIP compression supported
54 STREAM_CAP_BROTLI = (1 << 21), // BROTLI compression supported
55 + STREAM_CAP_PROGRESS = (1 << 22), // Functions PROGRESS support
56
57 STREAM_CAP_INVALID = (1 << 30), // used as an invalid value for capabilities when this is set
58 // this must be signed int, so don't use the last bit
streaming/sender.c
+29 -4
@@ -1118,9 +1118,8 @@ struct inflight_stream_function {
1118 usec_t received_ut;
1119 };
1120
1121 -void stream_execute_function_callback(BUFFER *func_wb, int code, void *data) {
1121 +static void stream_execute_function_callback(BUFFER *func_wb, int code, void *data) {
1122 struct inflight_stream_function *tmp = data;
1123 -
1123 struct sender_state *s = tmp->sender;
1124
1125 if(rrdhost_can_send_definitions_to_parent(s->host)) {
@@ -1150,6 +1149,20 @@ void stream_execute_function_callback(BUFFER *func_wb, int code, void *data) {
1149 freez(tmp);
1150 }
1151
1152 +static void stream_execute_function_progress_callback(void *data, size_t done, size_t all) {
1153 + struct inflight_stream_function *tmp = data;
1154 + struct sender_state *s = tmp->sender;
1155 +
1156 + if(rrdhost_can_send_definitions_to_parent(s->host)) {
1157 + BUFFER *wb = sender_start(s);
1158 +
1159 + buffer_sprintf(wb, PLUGINSD_KEYWORD_FUNCTION_PROGRESS " '%s' %zu %zu\n",
1160 + string2str(tmp->transaction), done, all);
1161 +
1162 + sender_commit(s, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
1163 + }
1164 +}
1165 +
1166 // This is just a placeholder until the gap filling state machine is inserted
1167 void execute_commands(struct sender_state *s) {
1168 worker_is_busy(WORKER_SENDER_JOB_EXECUTE);
@@ -1204,8 +1217,12 @@ void execute_commands(struct sender_state *s) {
1217 BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX + 1, &netdata_buffers_statistics.buffers_functions);
1218
1219 char *payload = s->receiving_function_payload ? (char *)buffer_tostring(s->function_payload.payload) : NULL;
1207 - int code = rrd_function_run(s->host, wb, timeout, function, false, transaction,
1208 - stream_execute_function_callback, tmp, NULL, NULL, payload);
1220 + int code = rrd_function_run(s->host, wb,
1221 + timeout, HTTP_ACCESS_ADMINS, function, false, transaction,
1222 + stream_execute_function_callback, tmp,
1223 + stream_has_capability(s, STREAM_CAP_PROGRESS) ? stream_execute_function_progress_callback : NULL,
1224 + stream_has_capability(s, STREAM_CAP_PROGRESS) ? tmp : NULL,
1225 + NULL, NULL, payload);
1226
1227 if(code != HTTP_RESP_OK) {
1228 if (!buffer_strlen(wb))
@@ -1267,6 +1284,14 @@ void execute_commands(struct sender_state *s) {
1284 if(transaction && *transaction)
1285 rrd_function_cancel(transaction);
1286 }
1287 + else if(command && strcmp(command, PLUGINSD_KEYWORD_FUNCTION_PROGRESS) == 0) {
1288 + worker_is_busy(WORKER_SENDER_JOB_FUNCTION_REQUEST);
1289 + nd_log(NDLS_ACCESS, NDLP_DEBUG, NULL);
1290 +
1291 + char *transaction = get_word(s->line.words, s->line.num_words, 1);
1292 + if(transaction && *transaction)
1293 + rrd_function_progress(transaction);
1294 + }
1295 else if (command && strcmp(command, PLUGINSD_KEYWORD_REPLAY_CHART) == 0) {
1296 worker_is_busy(WORKER_SENDER_JOB_REPLAY_REQUEST);
1297 nd_log(NDLS_ACCESS, NDLP_DEBUG, NULL);
web/api/queries/query.c
+4
@@ -3468,6 +3468,8 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
3468
3469 internal_fatal(released_ops, "QUERY: released_ops should be NULL when the query starts");
3470
3471 + query_progress_set_finish_line(qt->request.transaction, qt->query.used);
3472 +
3473 QUERY_ENGINE_OPS **ops = NULL;
3474 if(qt->query.used)
3475 ops = onewayalloc_callocz(owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *));
@@ -3639,6 +3641,8 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
3641
3642 break;
3643 }
3644 + else
3645 + query_progress_done_step(qt->request.transaction, 1);
3646 }
3647
3648 // free all resources used by the grouping method
web/api/queries/weights.c
+2
@@ -1717,6 +1717,8 @@ static ssize_t weights_for_rrdmetric(void *data, RRDHOST *host, RRDCONTEXT_ACQUI
1717 return -1;
1718 }
1719
1720 + query_progress_done_step(qwr->transaction, 1);
1721 +
1722 return 1;
1723 }
1724
web/api/queries/weights.h
+2
@@ -57,6 +57,8 @@ typedef struct query_weights_request {
57
58 weights_interrupt_callback_t interrupt_callback;
59 void *interrupt_callback_data;
60 +
61 + uuid_t *transaction;
62 } QUERY_WEIGHTS_REQUEST;
63
64 int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr);
web/api/web_api.c
+27 -12
@@ -5,39 +5,52 @@
5 bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at the point the agent logs in - this should also be saved to /var/lib/netdata
6 DICTIONARY *netdata_authorized_bearers = NULL;
7
8 -static short int web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL endpoint_acl) {
9 - if(endpoint_acl == WEB_CLIENT_ACL_NONE || (endpoint_acl & WEB_CLIENT_ACL_NOCHECK))
8 +static short int web_client_check_acl_and_bearer(struct web_client *w, HTTP_ACL endpoint_acl) {
9 + if(endpoint_acl == HTTP_ACL_NONE || (endpoint_acl & HTTP_ACL_NOCHECK)) {
10 // the endpoint is totally public
11 + w->access = HTTP_ACCESS_ADMINS;
12 return HTTP_RESP_OK;
13 + }
14
15 bool acl_allows = w->acl & endpoint_acl;
16 if(!acl_allows)
17 // the channel we received the request from (w->acl) is not compatible with the endpoint
18 return HTTP_RESP_FORBIDDEN;
19
18 - if(!netdata_is_protected_by_bearer && !(endpoint_acl & WEB_CLIENT_ACL_BEARER_REQUIRED))
20 + if(!netdata_is_protected_by_bearer && !(endpoint_acl & (HTTP_ACL_BEARER_REQUIRED | HTTP_ACL_BEARER_OPTIONAL))) {
21 // bearer protection is not enabled and is not required by the endpoint
22 + w->access = HTTP_ACCESS_ANY;
23 return HTTP_RESP_OK;
24 + }
25
22 - if(!(endpoint_acl & (WEB_CLIENT_ACL_BEARER_REQUIRED|WEB_CLIENT_ACL_BEARER_OPTIONAL)))
26 + if(!(endpoint_acl & (HTTP_ACL_BEARER_REQUIRED | HTTP_ACL_BEARER_OPTIONAL | HTTP_ACL_BEARER_IF_PROTECTED))) {
27 // endpoint does not require a bearer
28 + w->access = HTTP_ACCESS_ANY;
29 return HTTP_RESP_OK;
30 + }
31
26 - if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC)))
32 + if((w->acl & (HTTP_ACL_ACLK | HTTP_ACL_WEBRTC))) {
33 // the request is coming from ACLK or WEBRTC (authorized already),
34 + w->access = HTTP_ACCESS_MEMBERS;
35 return HTTP_RESP_OK;
36 + }
37
30 - // at this point we need a bearer to serve the request
31 - // either because:
32 - //
33 - // 1. WEB_CLIENT_ACL_BEARER_REQUIRED, or
34 - // 2. netdata_is_protected_by_bearer == true
35 - //
38 + // we now need a bearer to serve the request,
39 + // because:
40 + // 1. HTTP_ACL_BEARER_REQUIRED, or
41 + // 2. netdata_is_protected_by_bearer == true
42
43 BEARER_STATUS t = api_check_bearer_token(w);
38 - if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED)
44 + if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED) {
45 // we have a valid bearer on the request
46 + w->access = HTTP_ACCESS_MEMBERS;
47 return HTTP_RESP_OK;
48 + }
49 +
50 + if(endpoint_acl & HTTP_ACL_BEARER_OPTIONAL) {
51 + w->access = HTTP_ACCESS_ANY;
52 + return HTTP_RESP_OK;
53 + }
54
55 netdata_log_info("BEARER: bearer is required for request: code %d", t);
56
@@ -278,6 +291,8 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur
291
292 .interrupt_callback = web_client_interrupt_callback,
293 .interrupt_callback_data = w,
294 +
295 + .transaction = &w->transaction,
296 };
297
298 return web_api_v12_weights(wb, &qwr);
web/api/web_api.h
+1 -1
@@ -29,7 +29,7 @@ BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst,
29 struct web_api_command {
30 const char *command;
31 uint32_t hash;
32 - WEB_CLIENT_ACL acl;
32 + HTTP_ACL acl;
33 int (*callback)(RRDHOST *host, struct web_client *w, char *url);
34 unsigned int allow_subpaths;
35 };
web/api/web_api_v1.c
+37 -31
@@ -822,6 +822,7 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
822 .priority = STORAGE_PRIORITY_NORMAL,
823 .interrupt_callback = web_client_interrupt_callback,
824 .interrupt_callback_data = w,
825 + .transaction = &w->transaction,
826 };
827 qt = query_target_create(&qtr);
828
@@ -1018,12 +1019,12 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *
1019 if(unlikely(action == 'H')) {
1020 // HELLO request, dashboard ACL
1021 analytics_log_dashboard();
1021 - if(unlikely(!web_client_can_access_dashboard(w)))
1022 + if(unlikely(!http_can_access_dashboard(w)))
1023 return web_client_permission_denied(w);
1024 }
1025 else {
1026 // everything else, registry ACL
1026 - if(unlikely(!web_client_can_access_registry(w)))
1027 + if(unlikely(!http_can_access_registry(w)))
1028 return web_client_permission_denied(w);
1029
1030 if(unlikely(do_not_track)) {
@@ -1373,13 +1374,17 @@ static int web_client_api_request_v1_aclk_state(RRDHOST *host, struct web_client
1374 }
1375
1376 int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_client *w, char *url) {
1376 - return web_client_api_request_weights(host, w, url, default_metric_correlations_method,
1377 - WEIGHTS_FORMAT_CHARTS, 1);
1377 + return web_client_api_request_weights(host, w, url, default_metric_correlations_method, WEIGHTS_FORMAT_CHARTS, 1);
1378 }
1379
1380 int web_client_api_request_v1_weights(RRDHOST *host, struct web_client *w, char *url) {
1381 - return web_client_api_request_weights(host, w, url, WEIGHTS_METHOD_ANOMALY_RATE,
1382 - WEIGHTS_FORMAT_CONTEXTS, 1);
1381 + return web_client_api_request_weights(host, w, url, WEIGHTS_METHOD_ANOMALY_RATE, WEIGHTS_FORMAT_CONTEXTS, 1);
1382 +}
1383 +
1384 +static void web_client_progress_functions_update(void *data, size_t done, size_t all) {
1385 + // handle progress updates from the plugin
1386 + struct web_client *w = data;
1387 + query_progress_functions_update(&w->transaction, done, all);
1388 }
1389
1390 int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char *url) {
@@ -1413,8 +1418,9 @@ int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char
1418 char transaction[UUID_COMPACT_STR_LEN];
1419 uuid_unparse_lower_compact(w->transaction, transaction);
1420
1416 - return rrd_function_run(host, wb, timeout, function, true, transaction,
1421 + return rrd_function_run(host, wb, timeout, w->access, function, true, transaction,
1422 NULL, NULL,
1423 + web_client_progress_functions_update, w,
1424 web_client_interrupt_callback, w, NULL);
1425 }
1426
@@ -1547,43 +1553,43 @@ int web_client_api_request_v1_mgmt(RRDHOST *host, struct web_client *w, char *ur
1553 }
1554
1555 static struct web_api_command api_commands_v1[] = {
1550 - { "info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_info, 0 },
1551 - { "data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_data, 0 },
1552 - { "chart", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_chart, 0 },
1553 - { "charts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_charts, 0 },
1554 - { "context", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_context, 0 },
1555 - { "contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_contexts, 0 },
1556 + {"info", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_info, 0 },
1557 + {"data", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_data, 0 },
1558 + {"chart", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_chart, 0 },
1559 + {"charts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_charts, 0 },
1560 + {"context", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_context, 0 },
1561 + {"contexts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_contexts, 0 },
1562
1563 // registry checks the ACL by itself, so we allow everything
1558 - { "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry, 0 },
1564 + {"registry", 0, HTTP_ACL_NOCHECK, web_client_api_request_v1_registry, 0 },
1565
1566 // badges can be fetched with both dashboard and badge permissions
1561 - { "badge.svg", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC | WEB_CLIENT_ACL_BADGE, web_client_api_request_v1_badge, 0 },
1567 + {"badge.svg", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC | HTTP_ACL_BADGE, web_client_api_request_v1_badge, 0 },
1568
1563 - { "alarms", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms, 0 },
1564 - { "alarms_values", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms_values, 0 },
1565 - { "alarm_log", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_log, 0 },
1566 - { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_variables, 0 },
1567 - { "alarm_count", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_count, 0 },
1568 - { "allmetrics", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_allmetrics, 0 },
1569 + {"alarms", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms, 0 },
1570 + {"alarms_values", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms_values, 0 },
1571 + {"alarm_log", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_log, 0 },
1572 + {"alarm_variables", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_variables, 0 },
1573 + {"alarm_count", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_count, 0 },
1574 + {"allmetrics", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_allmetrics, 0 },
1575
1576 #if defined(ENABLE_ML)
1571 - { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_ml_info, 0 },
1572 - // { "ml_models", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models },
1577 + {"ml_info", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_ml_info, 0 },
1578 + // { "ml_models", 0, HTTP_ACL_DASHBOARD, web_client_api_request_v1_ml_models },
1579 #endif
1580
1575 - {"manage", 0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_mgmt, 1 },
1576 - { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_aclk_state, 0 },
1577 - { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_metric_correlations, 0 },
1578 - { "weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_weights, 0 },
1581 + {"manage", 0, HTTP_ACL_MGMT | HTTP_ACL_ACLK, web_client_api_request_v1_mgmt, 1 },
1582 + {"aclk", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_aclk_state, 0 },
1583 + {"metric_correlations", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_metric_correlations, 0 },
1584 + {"weights", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_weights, 0 },
1585
1580 - {"function", 0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function, 0 },
1581 - {"functions", 0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions, 0 },
1586 + {"function", 0, HTTP_ACL_ACLK_WEBRTC_DASHBOARD_WITH_OPTIONAL_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function, 0 },
1587 + {"functions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions, 0 },
1588
1583 - { "dbengine_stats", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats, 0 },
1589 + {"dbengine_stats", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats, 0 },
1590
1591 // terminator
1586 - { NULL, 0, WEB_CLIENT_ACL_NONE, NULL, 0 },
1592 + {NULL, 0, HTTP_ACL_NONE, NULL, 0 },
1593 };
1594
1595 inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url_path_endpoint) {
web/api/web_api_v2.c
+53 -31
@@ -580,6 +580,8 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
580
581 .interrupt_callback = web_client_interrupt_callback,
582 .interrupt_callback_data = w,
583 +
584 + .transaction = &w->transaction,
585 };
586
587 for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++)
@@ -662,6 +664,31 @@ static int web_client_api_request_v2_webrtc(RRDHOST *host __maybe_unused, struct
664 return webrtc_new_connection(w->post_payload, w->response.data);
665 }
666
667 +static int web_client_api_request_v2_progress(RRDHOST *host __maybe_unused, struct web_client *w, char *url) {
668 + char *transaction = NULL;
669 +
670 + while(url) {
671 + char *value = strsep_skip_consecutive_separators(&url, "&");
672 + if(!value || !*value) continue;
673 +
674 + char *name = strsep_skip_consecutive_separators(&value, "=");
675 + if(!name || !*name) continue;
676 + if(!value || !*value) continue;
677 +
678 + // name and value are now the parameters
679 + // they are not null and not empty
680 +
681 + if(!strcmp(name, "transaction")) transaction = value;
682 + }
683 +
684 + uuid_t tr;
685 + uuid_parse_flexi(transaction, tr);
686 +
687 + rrd_function_call_progresser(&tr);
688 +
689 + return web_api_v2_report_progress(&tr, w->response.data);
690 +}
691 +
692 #define CONFIG_API_V2_URL "/api/v2/config"
693 static int web_client_api_request_v2_config(RRDHOST *host __maybe_unused, struct web_client *w, char *query __maybe_unused) {
694
@@ -703,17 +730,11 @@ static int web_client_api_request_v2_config(RRDHOST *host __maybe_unused, struct
730 int http_method;
731 switch (w->mode)
732 {
706 - case WEB_CLIENT_MODE_GET:
707 - http_method = HTTP_METHOD_GET;
708 - break;
709 - case WEB_CLIENT_MODE_POST:
710 - http_method = HTTP_METHOD_POST;
711 - break;
712 - case WEB_CLIENT_MODE_PUT:
713 - http_method = HTTP_METHOD_PUT;
714 - break;
715 - case WEB_CLIENT_MODE_DELETE:
716 - http_method = HTTP_METHOD_DELETE;
733 + case HTTP_REQUEST_MODE_GET:
734 + case HTTP_REQUEST_MODE_POST:
735 + case HTTP_REQUEST_MODE_PUT:
736 + case HTTP_REQUEST_MODE_DELETE:
737 + http_method = w->mode;
738 break;
739 default:
740 buffer_sprintf(w->response.data, "Invalid HTTP method");
@@ -839,35 +860,36 @@ static int web_client_api_request_v2_job_statuses(RRDHOST *host __maybe_unused,
860 }
861
862 static struct web_api_command api_commands_v2[] = {
842 - {"info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info, 0},
863 + {"info", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info, 0},
864
844 - {"data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data, 0},
845 - {"weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights, 0},
865 + {"data", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data, 0},
866 + {"weights", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights, 0},
867
847 - {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts, 0},
848 - {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes, 0},
849 - {"node_instances", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_node_instances, 0},
850 - {"versions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_versions, 0},
851 - {"functions", 0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_functions, 0},
852 - {"q", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q, 0},
853 - {"alerts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alerts, 0},
868 + {"contexts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts, 0},
869 + {"nodes", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes, 0},
870 + {"node_instances", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_node_instances, 0},
871 + {"versions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_versions, 0},
872 + {"functions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_functions, 0},
873 + {"q", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q, 0},
874 + {"alerts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alerts, 0},
875
855 - {"alert_transitions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_transitions, 0},
856 - {"alert_config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_config, 0},
876 + {"alert_transitions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_transitions, 0},
877 + {"alert_config", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_config, 0},
878
858 - {"claim", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v2_claim, 0},
879 + {"claim", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_claim, 0},
880
860 - {"rtc_offer", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_webrtc, 0},
861 - {"bearer_protection", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection, 0},
862 - {"bearer_get_token", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token, 0},
881 + {"rtc_offer", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_webrtc, 0},
882 + {"bearer_protection", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection, 0},
883 + {"bearer_get_token", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token, 0},
884
864 - {"config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_config, 1},
865 - {"job_statuses", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_job_statuses, 0},
885 + {"config", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_config, 1},
886 + {"job_statuses", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_job_statuses, 0},
887
867 - { "ilove.svg", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v2_ilove, 0 },
888 + { "ilove.svg", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_ilove, 0 },
889 + { "progress", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_progress, 0 },
890
891 // terminator
870 - {NULL, 0, WEB_CLIENT_ACL_NONE, NULL, 0},
892 + {NULL, 0, HTTP_ACL_NONE, NULL, 0},
893 };
894
895 inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url_path_endpoint) {
web/rtc/webrtc.c
+3 -3
@@ -293,15 +293,15 @@ static void webrtc_execute_api_request(WEBRTC_DC *chan, const char *request, siz
293 w->interrupt.callback = web_client_stop_callback;
294 w->interrupt.callback_data = chan;
295
296 - w->acl = WEB_CLIENT_ACL_WEBRTC;
296 + w->acl = HTTP_ACL_WEBRTC;
297
298 char *path = (char *)request;
299 if(strncmp(request, "POST ", 5) == 0) {
300 - w->mode = WEB_CLIENT_MODE_POST;
300 + w->mode = HTTP_REQUEST_MODE_POST;
301 path += 10;
302 }
303 else if(strncmp(request, "GET ", 4) == 0) {
304 - w->mode = WEB_CLIENT_MODE_GET;
304 + w->mode = HTTP_REQUEST_MODE_GET;
305 path += 4;
306 }
307
web/server/h2o/http_server.c
+1 -1
@@ -210,7 +210,7 @@ static inline int _netdata_uberhandler(h2o_req_t *req, RRDHOST **host)
210 w.response.header = buffer_create(NBUF_INITIAL_SIZE_RESP, NULL);
211 w.url_query_string_decoded = buffer_create(NBUF_INITIAL_SIZE_RESP, NULL);
212 w.url_as_received = buffer_create(NBUF_INITIAL_SIZE_RESP, NULL);
213 - w.acl = WEB_CLIENT_ACL_DASHBOARD;
213 + w.acl = HTTP_ACL_DASHBOARD;
214
215 char *path_c_str = iovec_to_cstr(&api_command);
216 char *path_unescaped = url_unescape(path_c_str);
web/server/static/static-threaded.c
+3 -3
@@ -142,7 +142,7 @@ static int web_server_file_read_callback(POLLINFO *pi, short int *events) {
142 goto cleanup;
143 }
144
145 - if(unlikely(w->mode != WEB_CLIENT_MODE_FILECOPY || w->ifd == w->ofd)) {
145 + if(unlikely(w->mode != HTTP_REQUEST_MODE_FILECOPY || w->ifd == w->ofd)) {
146 netdata_log_debug(D_WEB_CLIENT, "%llu: PREVENTED ATTEMPT TO READ FILE ON FD %d, ON NON-FILECOPY WEB CLIENT", w->id, pi->fd);
147 retval = -1;
148 goto cleanup;
@@ -296,11 +296,11 @@ static int web_server_rcv_callback(POLLINFO *pi, short int *events) {
296 worker_is_busy(WORKER_JOB_PROCESS);
297 web_client_process_request_from_web_server(w);
298
299 - if (unlikely(w->mode == WEB_CLIENT_MODE_STREAM)) {
299 + if (unlikely(w->mode == HTTP_REQUEST_MODE_STREAM)) {
300 web_client_send(w);
301 }
302
303 - else if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) {
303 + else if(unlikely(w->mode == HTTP_REQUEST_MODE_FILECOPY)) {
304 if(w->pollinfo_filecopy_slot == 0) {
305 netdata_log_debug(D_WEB_CLIENT, "%llu: FILECOPY DETECTED ON FD %d", w->id, pi->fd);
306
web/server/web_client.c
+105 -316
@@ -138,6 +138,9 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) {
138 freez(w->forwarded_host);
139 w->forwarded_host = NULL;
140
141 + freez(w->forwarded_for);
142 + w->forwarded_for = NULL;
143 +
144 freez(w->origin);
145 w->origin = NULL;
146
@@ -163,41 +166,12 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) {
166 web_client_reset_path_flags(w);
167 }
168
166 -const char *get_request_method(struct web_client *w) {
167 - switch(w->mode) {
168 - case WEB_CLIENT_MODE_FILECOPY:
169 - return "FILECOPY";
170 -
171 - case WEB_CLIENT_MODE_OPTIONS:
172 - return "OPTIONS";
173 -
174 - case WEB_CLIENT_MODE_STREAM:
175 - return "STREAM";
176 -
177 - case WEB_CLIENT_MODE_POST:
178 - return "POST";
179 -
180 - case WEB_CLIENT_MODE_PUT:
181 - return "PUT";
182 -
183 - case WEB_CLIENT_MODE_GET:
184 - return "GET";
185 -
186 - case WEB_CLIENT_MODE_DELETE:
187 - return "DELETE";
188 -
189 - default:
190 - return "UNKNOWN";
191 - }
192 -}
193 -
169 void web_client_log_completed_request(struct web_client *w, bool update_web_stats) {
170 struct timeval tv;
171 now_monotonic_high_precision_timeval(&tv);
172
198 - size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len;
199 - size_t sent = size;
200 - if(likely(w->response.zoutput)) sent = (size_t)w->response.zstream.total_out;
173 + size_t size = (w->mode == HTTP_REQUEST_MODE_FILECOPY) ? w->response.rlen : w->response.data->len;
174 + size_t sent = w->response.zoutput ? (size_t)w->response.zstream.total_out : size;
175
176 if(update_web_stats)
177 global_statistics_web_request_completed(dt_usec(&tv, &w->timings.tv_in),
@@ -215,7 +189,7 @@ void web_client_log_completed_request(struct web_client *w, bool update_web_stat
189 ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
190 ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
191 ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
218 - ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, get_request_method(w)),
192 + ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, http_request_method2string(w->mode)),
193 ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
194 ND_LOG_FIELD_U64(NDF_RESPONSE_CODE, w->response.code),
195 ND_LOG_FIELD_U64(NDF_RESPONSE_SENT_BYTES, sent),
@@ -235,13 +209,21 @@ void web_client_log_completed_request(struct web_client *w, bool update_web_stat
209 else if(w->response.code >= 300)
210 prio = NDLP_NOTICE;
211
212 + // cleanup progress
213 + if(web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
214 + web_client_flag_clear(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
215 + query_progress_finished(&w->transaction, 0, w->response.code, total_ut, size, sent);
216 + }
217 +
218 // access log
239 - nd_log(NDLS_ACCESS, prio, NULL);
219 + if(likely(buffer_strlen(w->url_as_received)))
220 + nd_log(NDLS_ACCESS, prio, NULL);
221 }
222
223 void web_client_request_done(struct web_client *w) {
224 ND_LOG_STACK lgs[] = {
225 ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip),
226 + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for),
227 ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
228 ND_LOG_FIELD_END(),
229 };
@@ -251,10 +233,9 @@ void web_client_request_done(struct web_client *w) {
233
234 netdata_log_debug(D_WEB_CLIENT, "%llu: Resetting client.", w->id);
235
254 - if(likely(buffer_strlen(w->url_as_received)))
255 - web_client_log_completed_request(w, true);
236 + web_client_log_completed_request(w, true);
237
257 - if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) {
238 + if(unlikely(w->mode == HTTP_REQUEST_MODE_FILECOPY)) {
239 if(w->ifd != w->ofd) {
240 netdata_log_debug(D_WEB_CLIENT, "%llu: Closing filecopy input file descriptor %d.", w->id, w->ifd);
241
@@ -270,7 +251,7 @@ void web_client_request_done(struct web_client *w) {
251
252 web_client_reset_allocations(w, false);
253
273 - w->mode = WEB_CLIENT_MODE_GET;
254 + w->mode = HTTP_REQUEST_MODE_GET;
255
256 web_client_disable_donottrack(w);
257 web_client_disable_tracking_required(w);
@@ -292,74 +273,6 @@ void web_client_request_done(struct web_client *w) {
273 w->statistics.sent_bytes = 0;
274 }
275
295 -static struct {
296 - const char *extension;
297 - uint32_t hash;
298 - uint8_t contenttype;
299 -} mime_types[] = {
300 - { "html" , 0 , CT_TEXT_HTML}
301 - , {"js" , 0 , CT_APPLICATION_X_JAVASCRIPT}
302 - , {"css" , 0 , CT_TEXT_CSS}
303 - , {"xml" , 0 , CT_TEXT_XML}
304 - , {"xsl" , 0 , CT_TEXT_XSL}
305 - , {"txt" , 0 , CT_TEXT_PLAIN}
306 - , {"svg" , 0 , CT_IMAGE_SVG_XML}
307 - , {"ttf" , 0 , CT_APPLICATION_X_FONT_TRUETYPE}
308 - , {"otf" , 0 , CT_APPLICATION_X_FONT_OPENTYPE}
309 - , {"woff2", 0 , CT_APPLICATION_FONT_WOFF2}
310 - , {"woff" , 0 , CT_APPLICATION_FONT_WOFF}
311 - , {"eot" , 0 , CT_APPLICATION_VND_MS_FONTOBJ}
312 - , {"png" , 0 , CT_IMAGE_PNG}
313 - , {"jpg" , 0 , CT_IMAGE_JPG}
314 - , {"jpeg" , 0 , CT_IMAGE_JPG}
315 - , {"gif" , 0 , CT_IMAGE_GIF}
316 - , {"bmp" , 0 , CT_IMAGE_BMP}
317 - , {"ico" , 0 , CT_IMAGE_XICON}
318 - , {"icns" , 0 , CT_IMAGE_ICNS}
319 - , { NULL, 0, 0}
320 -};
321 -
322 -static inline uint8_t contenttype_for_filename(const char *filename) {
323 - // netdata_log_info("checking filename '%s'", filename);
324 -
325 - static int initialized = 0;
326 - int i;
327 -
328 - if(unlikely(!initialized)) {
329 - for (i = 0; mime_types[i].extension; i++)
330 - mime_types[i].hash = simple_hash(mime_types[i].extension);
331 -
332 - initialized = 1;
333 - }
334 -
335 - const char *s = filename, *last_dot = NULL;
336 -
337 - // find the last dot
338 - while(*s) {
339 - if(unlikely(*s == '.')) last_dot = s;
340 - s++;
341 - }
342 -
343 - if(unlikely(!last_dot || !*last_dot || !last_dot[1])) {
344 - // netdata_log_info("no extension for filename '%s'", filename);
345 - return CT_APPLICATION_OCTET_STREAM;
346 - }
347 - last_dot++;
348 -
349 - // netdata_log_info("extension for filename '%s' is '%s'", filename, last_dot);
350 -
351 - uint32_t hash = simple_hash(last_dot);
352 - for(i = 0; mime_types[i].extension ; i++) {
353 - if(unlikely(hash == mime_types[i].hash && !strcmp(last_dot, mime_types[i].extension))) {
354 - // netdata_log_info("matched extension for filename '%s': '%s'", filename, last_dot);
355 - return mime_types[i].contenttype;
356 - }
357 - }
358 -
359 - // netdata_log_info("not matched extension for filename '%s': '%s'", filename, last_dot);
360 - return CT_APPLICATION_OCTET_STREAM;
361 -}
362 -
276 static int append_slash_to_url_and_redirect(struct web_client *w) {
277 // this function returns a relative redirect
278 // it finds the last path component on the URL and just appends / to it
@@ -508,7 +421,7 @@ static bool find_filename_to_serve(const char *filename, char *dst, size_t dst_l
421 static int mysendfile(struct web_client *w, char *filename) {
422 netdata_log_debug(D_WEB_CLIENT, "%llu: Looking for file '%s/%s'", w->id, netdata_configured_web_dir, filename);
423
511 - if(!web_client_can_access_dashboard(w))
424 + if(!http_can_access_dashboard(w))
425 return web_client_permission_denied(w);
426
427 // skip leading slashes
@@ -576,7 +489,7 @@ static int mysendfile(struct web_client *w, char *filename) {
489 w->response.data->content_type = contenttype_for_filename(web_filename);
490 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%"PRId64" bytes, ifd %d, ofd %d).", w->id, web_filename, (int64_t)statbuf.st_size, w->ifd, w->ofd);
491
579 - w->mode = WEB_CLIENT_MODE_FILECOPY;
492 + w->mode = HTTP_REQUEST_MODE_FILECOPY;
493 web_client_enable_wait_receive(w);
494 web_client_disable_wait_send(w);
495 buffer_flush(w->response.data);
@@ -719,14 +632,14 @@ static inline int check_host_and_call(RRDHOST *host, struct web_client *w, char
632 }
633
634 static inline int UNUSED_FUNCTION(check_host_and_dashboard_acl_and_call)(RRDHOST *host, struct web_client *w, char *url, int (*func)(RRDHOST *, struct web_client *, char *)) {
722 - if(!web_client_can_access_dashboard(w))
635 + if(!http_can_access_dashboard(w))
636 return web_client_permission_denied(w);
637
638 return check_host_and_call(host, w, url, func);
639 }
640
641 static inline int UNUSED_FUNCTION(check_host_and_mgmt_acl_and_call)(RRDHOST *host, struct web_client *w, char *url, int (*func)(RRDHOST *, struct web_client *, char *)) {
729 - if(!web_client_can_access_mgmt(w))
642 + if(!http_can_access_mgmt(w))
643 return web_client_permission_denied(w);
644
645 return check_host_and_call(host, w, url, func);
@@ -736,8 +649,10 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_f
649 ND_LOG_STACK lgs[] = {
650 ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip),
651 ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
652 + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host),
653 + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for),
654 ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
740 - ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, get_request_method(w)),
655 + ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, http_request_method2string(w->mode)),
656 ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
657 ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
658 ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
@@ -745,6 +660,14 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_f
660 };
661 ND_LOG_STACK_PUSH(lgs);
662
663 + if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
664 + web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
665 + query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl,
666 + buffer_tostring(w->url_as_received),
667 + w->post_payload,
668 + w->forwarded_for ? w->forwarded_for : w->client_ip);
669 + }
670 +
671 // get the api version
672 char *tok = strsep_skip_consecutive_separators(&url_path_fragment, "/");
673 if(tok && *tok) {
@@ -853,164 +776,10 @@ const char *web_content_type_to_string(HTTP_CONTENT_TYPE content_type) {
776 }
777 }
778
856 -const char *web_response_code_to_string(int code) {
857 - switch(code) {
858 - case 100:
859 - return "Continue";
860 - case 101:
861 - return "Switching Protocols";
862 - case 102:
863 - return "Processing";
864 - case 103:
865 - return "Early Hints";
866 -
867 - case 200:
868 - return "OK";
869 - case 201:
870 - return "Created";
871 - case 202:
872 - return "Accepted";
873 - case 203:
874 - return "Non-Authoritative Information";
875 - case 204:
876 - return "No Content";
877 - case 205:
878 - return "Reset Content";
879 - case 206:
880 - return "Partial Content";
881 - case 207:
882 - return "Multi-Status";
883 - case 208:
884 - return "Already Reported";
885 - case 226:
886 - return "IM Used";
887 -
888 - case 300:
889 - return "Multiple Choices";
890 - case 301:
891 - return "Moved Permanently";
892 - case 302:
893 - return "Found";
894 - case 303:
895 - return "See Other";
896 - case 304:
897 - return "Not Modified";
898 - case 305:
899 - return "Use Proxy";
900 - case 306:
901 - return "Switch Proxy";
902 - case 307:
903 - return "Temporary Redirect";
904 - case 308:
905 - return "Permanent Redirect";
906 -
907 - case 400:
908 - return "Bad Request";
909 - case 401:
910 - return "Unauthorized";
911 - case 402:
912 - return "Payment Required";
913 - case 403:
914 - return "Forbidden";
915 - case 404:
916 - return "Not Found";
917 - case 405:
918 - return "Method Not Allowed";
919 - case 406:
920 - return "Not Acceptable";
921 - case 407:
922 - return "Proxy Authentication Required";
923 - case 408:
924 - return "Request Timeout";
925 - case 409:
926 - return "Conflict";
927 - case 410:
928 - return "Gone";
929 - case 411:
930 - return "Length Required";
931 - case 412:
932 - return "Precondition Failed";
933 - case 413:
934 - return "Payload Too Large";
935 - case 414:
936 - return "URI Too Long";
937 - case 415:
938 - return "Unsupported Media Type";
939 - case 416:
940 - return "Range Not Satisfiable";
941 - case 417:
942 - return "Expectation Failed";
943 - case 418:
944 - return "I'm a teapot";
945 - case 421:
946 - return "Misdirected Request";
947 - case 422:
948 - return "Unprocessable Entity";
949 - case 423:
950 - return "Locked";
951 - case 424:
952 - return "Failed Dependency";
953 - case 425:
954 - return "Too Early";
955 - case 426:
956 - return "Upgrade Required";
957 - case 428:
958 - return "Precondition Required";
959 - case 429:
960 - return "Too Many Requests";
961 - case 431:
962 - return "Request Header Fields Too Large";
963 - case 451:
964 - return "Unavailable For Legal Reasons";
965 - case 499: // nginx's extension to the standard
966 - return "Client Closed Request";
967 -
968 - case 500:
969 - return "Internal Server Error";
970 - case 501:
971 - return "Not Implemented";
972 - case 502:
973 - return "Bad Gateway";
974 - case 503:
975 - return "Service Unavailable";
976 - case 504:
977 - return "Gateway Timeout";
978 - case 505:
979 - return "HTTP Version Not Supported";
980 - case 506:
981 - return "Variant Also Negotiates";
982 - case 507:
983 - return "Insufficient Storage";
984 - case 508:
985 - return "Loop Detected";
986 - case 510:
987 - return "Not Extended";
988 - case 511:
989 - return "Network Authentication Required";
990 -
991 - default:
992 - if(code >= 100 && code < 200)
993 - return "Informational";
994 -
995 - if(code >= 200 && code < 300)
996 - return "Successful";
997 -
998 - if(code >= 300 && code < 400)
999 - return "Redirection";
1000 -
1001 - if(code >= 400 && code < 500)
1002 - return "Client Error";
1003 -
1004 - if(code >= 500 && code < 600)
1005 - return "Server Error";
1006 -
1007 - return "Undefined Error";
1008 - }
1009 -}
1010 -
779 static inline char *http_header_parse(struct web_client *w, char *s, int parse_useragent) {
780 static uint32_t hash_origin = 0, hash_connection = 0, hash_donottrack = 0, hash_useragent = 0,
1013 - hash_authorization = 0, hash_host = 0, hash_forwarded_host = 0, hash_transaction_id = 0;
781 + hash_authorization = 0, hash_host = 0, hash_forwarded_host = 0, hash_forwarded_for = 0,
782 + hash_transaction_id = 0;
783 static uint32_t hash_accept_encoding = 0;
784
785 if(unlikely(!hash_origin)) {
@@ -1022,6 +791,7 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
791 hash_authorization = simple_uhash("X-Auth-Token");
792 hash_host = simple_uhash("Host");
793 hash_forwarded_host = simple_uhash("X-Forwarded-Host");
794 + hash_forwarded_for = simple_uhash("X-Forwarded-For");
795 hash_transaction_id = simple_uhash("X-Transaction-ID");
796 }
797
@@ -1090,6 +860,11 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
860 strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
861 w->forwarded_host = strdupz(buffer);
862 }
863 + else if(hash == hash_forwarded_for && !strcasecmp(s, "X-Forwarded-For")) {
864 + char buffer[NI_MAXHOST];
865 + strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
866 + w->forwarded_for = strdupz(buffer);
867 + }
868 else if(hash == hash_transaction_id && !strcasecmp(s, "X-Transaction-ID")) {
869 char buffer[UUID_STR_LEN * 2];
870 strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
@@ -1115,29 +890,29 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
890 // is is a valid request?
891 if(!strncmp(s, "GET ", 4)) {
892 s = &s[4];
1118 - w->mode = WEB_CLIENT_MODE_GET;
893 + w->mode = HTTP_REQUEST_MODE_GET;
894 }
895 else if(!strncmp(s, "OPTIONS ", 8)) {
896 s = &s[8];
1122 - w->mode = WEB_CLIENT_MODE_OPTIONS;
897 + w->mode = HTTP_REQUEST_MODE_OPTIONS;
898 }
899 else if(!strncmp(s, "POST ", 5)) {
900 s = &s[5];
1126 - w->mode = WEB_CLIENT_MODE_POST;
901 + w->mode = HTTP_REQUEST_MODE_POST;
902 }
903 else if(!strncmp(s, "PUT ", 4)) {
904 s = &s[4];
1130 - w->mode = WEB_CLIENT_MODE_PUT;
905 + w->mode = HTTP_REQUEST_MODE_PUT;
906 }
907 else if(!strncmp(s, "DELETE ", 7)) {
908 s = &s[7];
1134 - w->mode = WEB_CLIENT_MODE_DELETE;
909 + w->mode = HTTP_REQUEST_MODE_DELETE;
910 }
911 else if(!strncmp(s, "STREAM ", 7)) {
912 s = &s[7];
913
914 #ifdef ENABLE_HTTPS
1140 - if (!SSL_connection(&w->ssl) && web_client_is_using_ssl_force(w)) {
915 + if (!SSL_connection(&w->ssl) && http_is_using_ssl_force(w)) {
916 w->header_parse_tries = 0;
917 w->header_parse_last_size = 0;
918 web_client_disable_wait_receive(w);
@@ -1166,7 +941,7 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
941 }
942 #endif
943
1169 - w->mode = WEB_CLIENT_MODE_STREAM;
944 + w->mode = HTTP_REQUEST_MODE_STREAM;
945 }
946 else {
947 s = NULL;
@@ -1279,7 +1054,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
1054
1055 #ifdef ENABLE_HTTPS
1056 if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) {
1282 - if (!w->ssl.conn && (web_client_is_using_ssl_force(w) || web_client_is_using_ssl_default(w)) && (w->mode != WEB_CLIENT_MODE_STREAM)) {
1057 + if (!w->ssl.conn && (http_is_using_ssl_force(w) || http_is_using_ssl_default(w)) && (w->mode != HTTP_REQUEST_MODE_STREAM)) {
1058 w->header_parse_tries = 0;
1059 w->header_parse_last_size = 0;
1060 web_client_disable_wait_receive(w);
@@ -1295,7 +1070,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
1070 }
1071
1072 // another header line
1298 - s = http_header_parse(w, s, (w->mode == WEB_CLIENT_MODE_STREAM)); // parse user agent
1073 + s = http_header_parse(w, s, (w->mode == HTTP_REQUEST_MODE_STREAM)); // parse user agent
1074 }
1075 }
1076
@@ -1340,7 +1115,7 @@ void web_client_build_http_header(struct web_client *w) {
1115 netdata_log_debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, w->response.code);
1116
1117 const char *content_type_string = web_content_type_to_string(w->response.data->content_type);
1343 - const char *code_msg = web_response_code_to_string(w->response.code);
1118 + const char *code_msg = http_response_code2string(w->response.code);
1119
1120 // prepare the last modified and expiration dates
1121 char rfc7231_date[RFC7231_MAX_LENGTH], rfc7231_expires[RFC7231_MAX_LENGTH];
@@ -1393,7 +1168,7 @@ void web_client_build_http_header(struct web_client *w) {
1168 }
1169 }
1170
1396 - if(w->mode == WEB_CLIENT_MODE_OPTIONS) {
1171 + if(w->mode == HTTP_REQUEST_MODE_OPTIONS) {
1172 buffer_strcat(w->response.header_output,
1173 "Access-Control-Allow-Methods: GET, OPTIONS\r\n"
1174 "Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie, pragma, cache-control, x-auth-token\r\n"
@@ -1580,7 +1355,7 @@ int web_client_api_request_with_node_selection(RRDHOST *host, struct web_client
1355 // entry point for all API requests
1356
1357 ND_LOG_STACK lgs[] = {
1583 - ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, get_request_method(w)),
1358 + ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, http_request_method2string(w->mode)),
1359 ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
1360 ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
1361 ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
@@ -1692,7 +1467,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1467 return web_client_process_url(host, w, decoded_url_path);
1468 }
1469 else if(unlikely(hash == hash_netdata_conf && strcmp(tok, "netdata.conf") == 0)) { // netdata.conf
1695 - if(unlikely(!web_client_can_access_netdataconf(w)))
1470 + if(unlikely(!http_can_access_netdataconf(w)))
1471 return web_client_permission_denied(w);
1472
1473 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: generating netdata.conf ...", w->id);
@@ -1703,7 +1478,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1478 }
1479 #ifdef NETDATA_INTERNAL_CHECKS
1480 else if(unlikely(hash == hash_exit && strcmp(tok, "exit") == 0)) {
1706 - if(unlikely(!web_client_can_access_netdataconf(w)))
1481 + if(unlikely(!http_can_access_netdataconf(w)))
1482 return web_client_permission_denied(w);
1483
1484 w->response.data->content_type = CT_TEXT_PLAIN;
@@ -1719,7 +1494,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1494 return HTTP_RESP_OK;
1495 }
1496 else if(unlikely(hash == hash_debug && strcmp(tok, "debug") == 0)) {
1722 - if(unlikely(!web_client_can_access_netdataconf(w)))
1497 + if(unlikely(!http_can_access_netdataconf(w)))
1498 return web_client_permission_denied(w);
1499
1500 buffer_flush(w->response.data);
@@ -1759,7 +1534,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1534 return HTTP_RESP_BAD_REQUEST;
1535 }
1536 else if(unlikely(hash == hash_mirror && strcmp(tok, "mirror") == 0)) {
1762 - if(unlikely(!web_client_can_access_netdataconf(w)))
1537 + if(unlikely(!http_can_access_netdataconf(w)))
1538 return web_client_permission_denied(w);
1539
1540 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id);
@@ -1799,8 +1574,10 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1574 ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, web_server_log_transport, w),
1575 ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip),
1576 ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
1577 + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host),
1578 + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for),
1579 ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
1803 - ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, get_request_method(w)),
1580 + ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, http_request_method2string(w->mode)),
1581 ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
1582 ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
1583 ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
@@ -1816,9 +1593,17 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1593
1594 switch(http_request_validate(w)) {
1595 case HTTP_VALIDATION_OK:
1596 + if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
1597 + web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
1598 + query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl,
1599 + buffer_tostring(w->url_as_received),
1600 + w->post_payload,
1601 + w->forwarded_for ? w->forwarded_for : w->client_ip);
1602 + }
1603 +
1604 switch(w->mode) {
1820 - case WEB_CLIENT_MODE_STREAM:
1821 - if(unlikely(!web_client_can_access_stream(w))) {
1605 + case HTTP_REQUEST_MODE_STREAM:
1606 + if(unlikely(!http_can_access_stream(w))) {
1607 web_client_permission_denied(w);
1608 return;
1609 }
@@ -1826,13 +1611,13 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1611 w->response.code = rrdpush_receiver_thread_spawn(w, (char *)buffer_tostring(w->url_query_string_decoded), NULL);
1612 return;
1613
1829 - case WEB_CLIENT_MODE_OPTIONS:
1614 + case HTTP_REQUEST_MODE_OPTIONS:
1615 if(unlikely(
1831 - !web_client_can_access_dashboard(w) &&
1832 - !web_client_can_access_registry(w) &&
1833 - !web_client_can_access_badges(w) &&
1834 - !web_client_can_access_mgmt(w) &&
1835 - !web_client_can_access_netdataconf(w)
1616 + !http_can_access_dashboard(w) &&
1617 + !http_can_access_registry(w) &&
1618 + !http_can_access_badges(w) &&
1619 + !http_can_access_mgmt(w) &&
1620 + !http_can_access_netdataconf(w)
1621 )) {
1622 web_client_permission_denied(w);
1623 break;
@@ -1844,17 +1629,17 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1629 w->response.code = HTTP_RESP_OK;
1630 break;
1631
1847 - case WEB_CLIENT_MODE_FILECOPY:
1848 - case WEB_CLIENT_MODE_POST:
1849 - case WEB_CLIENT_MODE_GET:
1850 - case WEB_CLIENT_MODE_PUT:
1851 - case WEB_CLIENT_MODE_DELETE:
1632 + case HTTP_REQUEST_MODE_FILECOPY:
1633 + case HTTP_REQUEST_MODE_POST:
1634 + case HTTP_REQUEST_MODE_GET:
1635 + case HTTP_REQUEST_MODE_PUT:
1636 + case HTTP_REQUEST_MODE_DELETE:
1637 if(unlikely(
1853 - !web_client_can_access_dashboard(w) &&
1854 - !web_client_can_access_registry(w) &&
1855 - !web_client_can_access_badges(w) &&
1856 - !web_client_can_access_mgmt(w) &&
1857 - !web_client_can_access_netdataconf(w)
1638 + !http_can_access_dashboard(w) &&
1639 + !http_can_access_registry(w) &&
1640 + !http_can_access_badges(w) &&
1641 + !http_can_access_mgmt(w) &&
1642 + !http_can_access_netdataconf(w)
1643 )) {
1644 web_client_permission_denied(w);
1645 break;
@@ -1888,6 +1673,10 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1673
1674 w->response.code = (short)web_client_process_url(localhost, w, path);
1675 break;
1676 +
1677 + default:
1678 + web_client_permission_denied(w);
1679 + return;
1680 }
1681 break;
1682
@@ -1907,8 +1696,8 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1696 // wait for more data
1697 // set to normal to prevent web_server_rcv_callback
1698 // from going into stream mode
1910 - if (w->mode == WEB_CLIENT_MODE_STREAM)
1911 - w->mode = WEB_CLIENT_MODE_GET;
1699 + if (w->mode == HTTP_REQUEST_MODE_STREAM)
1700 + w->mode = HTTP_REQUEST_MODE_GET;
1701 return;
1702 }
1703 break;
@@ -1971,22 +1760,22 @@ void web_client_process_request_from_web_server(struct web_client *w) {
1760 else web_client_disable_wait_send(w);
1761
1762 switch(w->mode) {
1974 - case WEB_CLIENT_MODE_STREAM:
1763 + case HTTP_REQUEST_MODE_STREAM:
1764 netdata_log_debug(D_WEB_CLIENT, "%llu: STREAM done.", w->id);
1765 break;
1766
1978 - case WEB_CLIENT_MODE_OPTIONS:
1767 + case HTTP_REQUEST_MODE_OPTIONS:
1768 netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.", w->id, w->response.data->len);
1769 break;
1770
1982 - case WEB_CLIENT_MODE_POST:
1983 - case WEB_CLIENT_MODE_GET:
1984 - case WEB_CLIENT_MODE_PUT:
1985 - case WEB_CLIENT_MODE_DELETE:
1771 + case HTTP_REQUEST_MODE_POST:
1772 + case HTTP_REQUEST_MODE_GET:
1773 + case HTTP_REQUEST_MODE_PUT:
1774 + case HTTP_REQUEST_MODE_DELETE:
1775 netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%zu bytes) to client.", w->id, w->response.data->len);
1776 break;
1777
1989 - case WEB_CLIENT_MODE_FILECOPY:
1778 + case HTTP_REQUEST_MODE_FILECOPY:
1779 if(w->response.rlen) {
1780 netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending data file of %zu bytes to client.", w->id, w->response.rlen);
1781 web_client_enable_wait_receive(w);
@@ -2104,7 +1893,7 @@ ssize_t web_client_send_deflate(struct web_client *w)
1893 if(t < 0) return t;
1894 }
1895
2107 - if(w->mode == WEB_CLIENT_MODE_FILECOPY && web_client_has_wait_receive(w) && w->response.rlen && w->response.rlen > w->response.data->len) {
1896 + if(w->mode == HTTP_REQUEST_MODE_FILECOPY && web_client_has_wait_receive(w) && w->response.rlen && w->response.rlen > w->response.data->len) {
1897 // we have to wait, more data will come
1898 netdata_log_debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id);
1899 web_client_disable_wait_send(w);
@@ -2146,8 +1935,8 @@ ssize_t web_client_send_deflate(struct web_client *w)
1935
1936 // ask for FINISH if we have all the input
1937 int flush = Z_SYNC_FLUSH;
2149 - if((w->mode == WEB_CLIENT_MODE_GET || w->mode == WEB_CLIENT_MODE_POST || w->mode == WEB_CLIENT_MODE_PUT || w->mode == WEB_CLIENT_MODE_DELETE)
2150 - || (w->mode == WEB_CLIENT_MODE_FILECOPY && !web_client_has_wait_receive(w) && w->response.data->len == w->response.rlen)) {
1938 + if((w->mode == HTTP_REQUEST_MODE_GET || w->mode == HTTP_REQUEST_MODE_POST || w->mode == HTTP_REQUEST_MODE_PUT || w->mode == HTTP_REQUEST_MODE_DELETE)
1939 + || (w->mode == HTTP_REQUEST_MODE_FILECOPY && !web_client_has_wait_receive(w) && w->response.data->len == w->response.rlen)) {
1940 flush = Z_FINISH;
1941 netdata_log_debug(D_DEFLATE, "%llu: Requesting Z_FINISH, if possible.", w->id);
1942 }
@@ -2212,7 +2001,7 @@ ssize_t web_client_send(struct web_client *w) {
2001 // A. we have done everything
2002 // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd
2003
2215 - if(w->mode == WEB_CLIENT_MODE_FILECOPY && web_client_has_wait_receive(w) && w->response.rlen && w->response.rlen > w->response.data->len) {
2004 + if(w->mode == HTTP_REQUEST_MODE_FILECOPY && web_client_has_wait_receive(w) && w->response.rlen && w->response.rlen > w->response.data->len) {
2005 // we have to wait, more data will come
2006 netdata_log_debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id);
2007 web_client_disable_wait_send(w);
@@ -2301,7 +2090,7 @@ ssize_t web_client_read_file(struct web_client *w)
2090
2091 ssize_t web_client_receive(struct web_client *w)
2092 {
2304 - if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY))
2093 + if(unlikely(w->mode == HTTP_REQUEST_MODE_FILECOPY))
2094 return web_client_read_file(w);
2095
2096 ssize_t bytes;
@@ -2365,7 +2154,7 @@ void web_client_decode_path_and_query_string(struct web_client *w, const char *p
2154 // do not overwrite this if it is already filled
2155 buffer_strcat(w->url_as_received, path_and_query_string);
2156
2368 - if(w->mode == WEB_CLIENT_MODE_STREAM) {
2157 + if(w->mode == HTTP_REQUEST_MODE_STREAM) {
2158 // in stream mode, there is no path
2159
2160 url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
web/server/web_client.h
+6 -13
@@ -12,16 +12,6 @@ extern int web_enable_gzip, web_gzip_level, web_gzip_strategy;
12 extern int respect_web_browser_do_not_track_policy;
13 extern char *web_x_frame_options;
14
15 -typedef enum web_client_mode {
16 - WEB_CLIENT_MODE_GET = 0,
17 - WEB_CLIENT_MODE_POST = 1,
18 - WEB_CLIENT_MODE_FILECOPY = 2,
19 - WEB_CLIENT_MODE_OPTIONS = 3,
20 - WEB_CLIENT_MODE_STREAM = 4,
21 - WEB_CLIENT_MODE_PUT = 5,
22 - WEB_CLIENT_MODE_DELETE = 6,
23 -} WEB_CLIENT_MODE;
24 -
15 typedef enum {
16 HTTP_VALIDATION_OK,
17 HTTP_VALIDATION_NOT_SUPPORTED,
@@ -52,6 +42,7 @@ typedef enum web_client_flags {
42 WEB_CLIENT_FLAG_PATH_IS_V2 = (1 << 15), // v2 dashboard found on the path
43 WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH = (1 << 16), // the path has a trailing hash
44 WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION = (1 << 17), // the path ends with a filename extension
45 + WEB_CLIENT_FLAG_PROGRESS_TRACKING = (1 << 18), // when set we track the progress of this transaction
46 } WEB_CLIENT_FLAGS;
47
48 #define WEB_CLIENT_FLAG_PATH_WITH_VERSION (WEB_CLIENT_FLAG_PATH_IS_V0|WEB_CLIENT_FLAG_PATH_IS_V1|WEB_CLIENT_FLAG_PATH_IS_V2)
@@ -139,8 +130,9 @@ struct web_client {
130 uuid_t transaction;
131
132 WEB_CLIENT_FLAGS flags; // status flags for the client
142 - WEB_CLIENT_MODE mode; // the operational mode of the client
143 - WEB_CLIENT_ACL acl; // the access list of the client
133 + HTTP_REQUEST_MODE mode; // the operational mode of the client
134 + HTTP_ACL acl; // the access list of the client
135 + HTTP_ACCESS access; // the access level of the client
136 int port_acl; // the operations permitted on the port the client connected to
137 size_t header_parse_tries;
138 size_t header_parse_last_size;
@@ -160,7 +152,8 @@ struct web_client {
152 // THESE NEED TO BE FREED
153 char *auth_bearer_token; // the Bearer auth token (if sent)
154 char *server_host; // the Host: header
163 - char *forwarded_host; // the X-Forwarded-For: header
155 + char *forwarded_host; // the X-Forwarded-Host: header
156 + char *forwarded_for; // the X-Forwarded-For: header
157 char *origin; // the Origin: header
158 char *user_agent; // the User-Agent: header
159
web/server/web_client_cache.c
+1 -1
@@ -114,7 +114,7 @@ struct web_client *web_client_get_from_cache(void) {
114
115 // initialize it
116 w->use_count++;
117 - w->mode = WEB_CLIENT_MODE_GET;
117 + w->mode = HTTP_REQUEST_MODE_GET;
118 memset(w->transaction, 0, sizeof(w->transaction));
119
120 return w;
web/server/web_server.c
+16 -14
@@ -41,13 +41,13 @@ void debug_sockets() {
41 int i;
42
43 for(i = 0 ; i < (int)api_sockets.opened ; i++) {
44 - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_NOCHECK)?"NONE ":"");
45 - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_DASHBOARD)?"dashboard ":"");
46 - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_REGISTRY)?"registry ":"");
47 - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_BADGE)?"badges ":"");
48 - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_MGMT)?"management ":"");
49 - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_STREAMING)?"streaming ":"");
50 - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_NETDATACONF)?"netdata.conf ":"");
44 + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_NOCHECK) ? "NONE " : "");
45 + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_DASHBOARD) ? "dashboard " : "");
46 + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_REGISTRY) ? "registry " : "");
47 + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_BADGE) ? "badges " : "");
48 + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_MGMT) ? "management " : "");
49 + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_STREAMING) ? "streaming " : "");
50 + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_NETDATACONF) ? "netdata.conf " : "");
51 netdata_log_debug(D_WEB_CLIENT, "Socket fd %d name '%s' acl_flags: %s",
52 i,
53 api_sockets.fds_names[i],
@@ -91,37 +91,37 @@ SIMPLE_PATTERN *web_allow_netdataconf_from = NULL;
91 int web_allow_netdataconf_dns;
92
93 void web_client_update_acl_matches(struct web_client *w) {
94 - w->acl = WEB_CLIENT_ACL_NONE;
94 + w->acl = HTTP_ACL_NONE;
95
96 if (!web_allow_dashboard_from ||
97 connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
98 web_allow_dashboard_from, "dashboard", web_allow_dashboard_dns))
99 - w->acl |= WEB_CLIENT_ACL_DASHBOARD;
99 + w->acl |= HTTP_ACL_DASHBOARD;
100
101 if (!web_allow_registry_from ||
102 connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
103 web_allow_registry_from, "registry", web_allow_registry_dns))
104 - w->acl |= WEB_CLIENT_ACL_REGISTRY;
104 + w->acl |= HTTP_ACL_REGISTRY;
105
106 if (!web_allow_badges_from ||
107 connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
108 web_allow_badges_from, "badges", web_allow_badges_dns))
109 - w->acl |= WEB_CLIENT_ACL_BADGE;
109 + w->acl |= HTTP_ACL_BADGE;
110
111 if (!web_allow_mgmt_from ||
112 connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
113 web_allow_mgmt_from, "management", web_allow_mgmt_dns))
114 - w->acl |= WEB_CLIENT_ACL_MGMT;
114 + w->acl |= HTTP_ACL_MGMT;
115
116 if (!web_allow_streaming_from ||
117 connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
118 web_allow_streaming_from, "streaming", web_allow_streaming_dns))
119 - w->acl |= WEB_CLIENT_ACL_STREAMING;
119 + w->acl |= HTTP_ACL_STREAMING;
120
121 if (!web_allow_netdataconf_from ||
122 connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host),
123 web_allow_netdataconf_from, "netdata.conf", web_allow_netdataconf_dns))
124 - w->acl |= WEB_CLIENT_ACL_NETDATACONF;
124 + w->acl |= HTTP_ACL_NETDATACONF;
125
126 w->acl &= w->port_acl;
127 }
@@ -139,6 +139,8 @@ void web_server_log_connection(struct web_client *w, const char *msg) {
139 #endif
140 ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip),
141 ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
142 + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host),
143 + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for),
144 ND_LOG_FIELD_END(),
145 };
146 ND_LOG_STACK_PUSH(lgs);