Dyncfg improvements (#16250)
Costa Tsaousis committed
Oct 20, 2023 at 15:07 UTC
897009e223e56e6515e935dec1df80475668f067
3 files changed
+141
-111
collectors/plugins.d/pluginsd_parser.c
+17
-6
@@ -2438,24 +2438,35 @@ static inline PARSER_RC pluginsd_register_module(char **words __maybe_unused, si
2438
}
2439
2440
static inline PARSER_RC pluginsd_register_job_common(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused, const char *plugin_name) {
2441
- if (atol(words[3]) < 0)
2442
- return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "invalid flags");
2443
- dyncfg_job_flg_t flags = atol(words[3]);
2441
+ const char *module_name = words[0];
2442
+ const char *job_name = words[1];
2443
+ const char *job_type_str = words[2];
2444
+ const char *flags_str = words[3];
2445
+
2446
+ long f = str2l(flags_str);
2447
+
2448
+ if (f < 0)
2449
+ return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "invalid flags received");
2450
+
2451
+ dyncfg_job_flg_t flags = f;
2452
+
2453
if (SERVING_PLUGINSD(parser))
2454
flags |= JOB_FLG_PLUGIN_PUSHED;
2455
else
2456
flags |= JOB_FLG_STREAMING_PUSHED;
2457
2449
- enum job_type job_type = str2job_type(words[2]);
2458
+ enum job_type job_type = dyncfg_str2job_type(job_type_str);
2459
if (job_type == JOB_TYPE_UNKNOWN)
2460
return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "unknown job type");
2461
+
2462
if (SERVING_PLUGINSD(parser) && job_type == JOB_TYPE_USER)
2463
return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "plugins cannot push jobs of type \"user\" (this is allowed only in streaming)");
2464
2455
- if (register_job(parser->user.host->configurable_plugins, plugin_name, words[0], words[1], job_type, flags, 0)) // ignore existing is off as this is explicitly called register job
2465
+ if (register_job(parser->user.host->configurable_plugins, plugin_name, module_name, job_name, job_type, flags, 0)) // ignore existing is off as this is explicitly called register job
2466
return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "error registering job");
2467
2458
- rrdpush_send_dyncfg_reg_job(parser->user.host, plugin_name, words[0], words[1], job_type, flags);
2468
+ rrdpush_send_dyncfg_reg_job(parser->user.host, plugin_name, module_name, job_name, job_type, flags);
2469
+
2470
return PARSER_RC_OK;
2471
}
2472
libnetdata/dyn_conf/README.md
+123
-104
@@ -1,6 +1,6 @@
1
# Netdata Dynamic Configuration
2
3
-Purpose of Netdata Dynamic Configuration is to allow configuration of select Netdata plugins and options through the
3
+Purpose of Netdata Dynamic Configuration is to allow configuration of select Netdata plugins and options through the
4
Netdata API and by extension by UI.
5
6
## HTTP API documentation
@@ -9,159 +9,178 @@ Netdata API and by extension by UI.
9
10
For summary of all jobs and their statuses (for all children that stream to parent) use the following URL:
11
12
-| Method | Endpoint | Description |
13
-|:-------:|-------------------------------|------------------------------------------------------------|
14
-| **GET** | `api/v2/job_statuses` | list of Jobs |
15
-| **GET** | `api/v2/job_statuses?grouped` | list of Jobs (hierarchical, grouped by host/plugin/module) |
12
+| Method | Endpoint | Description |
13
+|:-------:|-------------------------------|------------------------------------------------------------|
14
+| **GET** | `api/v2/job_statuses` | list of Jobs |
15
+| **GET** | `api/v2/job_statuses?grouped` | list of Jobs (hierarchical, grouped by host/plugin/module) |
16
17
### Dyncfg API
18
19
### Top level
20
21
-| Method | Endpoint | Description |
22
-|:-------:|------------------|-----------------------------------------|
23
-| **GET** | `/api/v2/config` | registered Plugins (sent DYNCFG_ENABLE) |
21
+| Method | Endpoint | Description |
22
+|:-------:|------------------|-----------------------------------------|
23
+| **GET** | `/api/v2/config` | registered Plugins (sent DYNCFG_ENABLE) |
24
25
### Plugin level
26
27
-| Method | Endpoint | Description |
28
-|:-------:|-----------------------------------|------------------------------|
29
-| **GET** | `/api/v2/config/[plugin]` | Plugin config |
30
-| **PUT** | `/api/v2/config/[plugin]` | update Plugin config |
31
-| **GET** | `/api/v2/config/[plugin]/modules` | Modules registered by Plugin |
32
-| **GET** | `/api/v2/config/[plugin]/schema` | Plugin config schema |
27
+| Method | Endpoint | Description |
28
+|:-------:|-----------------------------------|------------------------------|
29
+| **GET** | `/api/v2/config/[plugin]` | Plugin config |
30
+| **PUT** | `/api/v2/config/[plugin]` | update Plugin config |
31
+| **GET** | `/api/v2/config/[plugin]/modules` | Modules registered by Plugin |
32
+| **GET** | `/api/v2/config/[plugin]/schema` | Plugin config schema |
33
34
### Module level
35
36
-| Method | Endpoint | Description |
37
-|:-------:|-----------------------------------------------|---------------------------|
38
-| **GET** | `/api/v2/config/<plugin>/[module]` | Module config |
39
-| **PUT** | `/api/v2/config/[plugin]/[module]` | update Module config |
40
-| **GET** | `/api/v2/config/[plugin]/[module]/jobs` | Jobs registered by Module |
41
-| **GET** | `/api/v2/config/[plugin]/[module]/job_schema` | Job config schema |
42
-| **GET** | `/api/v2/config/[plugin]/[module]/schema` | Module config schema |
36
+| Method | Endpoint | Description |
37
+|:-------:|-----------------------------------------------|---------------------------|
38
+| **GET** | `/api/v2/config/<plugin>/[module]` | Module config |
39
+| **PUT** | `/api/v2/config/[plugin]/[module]` | update Module config |
40
+| **GET** | `/api/v2/config/[plugin]/[module]/jobs` | Jobs registered by Module |
41
+| **GET** | `/api/v2/config/[plugin]/[module]/job_schema` | Job config schema |
42
+| **GET** | `/api/v2/config/[plugin]/[module]/schema` | Module config schema |
43
44
### Job level - only for modules where `module_type == job_array`
45
46
-| Method | Endpoint | Description |
47
-|:----------:|------------------------------------------|--------------------------------|
48
-| **GET** | `/api/v2/config/[plugin]/[module]/[job]` | Job config |
49
-| **PUT** | `/api/v2/config/[plugin]/[module]/[job]` | update Job config |
50
-| **POST** | `/api/v2/config/[plugin]/[module]/[job]` | create Job |
51
-| **DELETE** | `/api/v2/config/[plugin]/[module]/[job]` | delete Job (created by Dyncfg) |
46
+| Method | Endpoint | Description |
47
+|:----------:|------------------------------------------|--------------------------------|
48
+| **GET** | `/api/v2/config/[plugin]/[module]/[job]` | Job config |
49
+| **PUT** | `/api/v2/config/[plugin]/[module]/[job]` | update Job config |
50
+| **POST** | `/api/v2/config/[plugin]/[module]/[job]` | create Job |
51
+| **DELETE** | `/api/v2/config/[plugin]/[module]/[job]` | delete Job (created by Dyncfg) |
52
53
-## AGENT<->PLUGIN interface documentation
53
+## Internal Plugins API
54
55
-### 1. DYNCFG_ENABLE
55
+TBD
56
+
57
+## External Plugins API
58
+
59
+### Commands plugins can use
60
+
61
+#### DYNCFG_ENABLE
62
63
Plugin signifies to agent its ability to use new dynamic config and the name it wishes to use by sending
64
59
-```
60
-plugin->agent:
61
-=============
62
-DYNCFG_ENABLE [plugin_url_name]
63
-```
65
+```
66
+DYNCFG_ENABLE [{PLUGIN_NAME}]
67
+```
68
65
-This can be sent only once per lifetime of the plugin (at startup or later) sending it multiple times is considered a
69
+This can be sent only once per lifetime of the plugin (at startup or later) sending it multiple times is considered a
70
protocol violation and plugin might get terminated.
67
-After this command is sent the plugin has to be ready to accept all the new commands/keywords related to dynamic
71
+
72
+After this command is sent the plugin has to be ready to accept all the new commands/keywords related to dynamic
73
configuration (this command lets agent know this plugin is dyncfg capable and wishes to use dyncfg functionality).
74
70
-After this command agent can call
75
+#### DYNCFG_RESET
76
72
-```
73
-agent->plugin:
74
-=============
75
-FUNCTION_PAYLOAD [UUID] 1 "set_plugin_config"
76
-the new configuration
77
-blah blah blah
78
-FUNCTION_PAYLOAD_END
79
-
80
-plugin->agent:
81
-=============
82
-FUNCTION_RESULT_BEGIN [UUID] [(1/0)(accept/reject)] [text/plain] 5
83
-FUNCTION_RESULT_END
84
-```
77
+Sending this, will reset the internal state of the agent, considering this a `DYNCFG_ENABLE`.
78
86
-to set the new config which can be accepted/rejected by plugin by sending answer for this FUNCTION as it would with any
87
-other regular function.
79
+```
80
+DYNCFG_RESET
81
+```
82
89
-The new `FUNCTION_PAYLOAD` command differs from regular `FUNCTION` command exclusively in its ability to send bigger
90
-payloads (configuration file contents) to the plugin (not just parameters list).
83
92
-Agent can also call (after `DYNCFG_ENABLE`)
84
+#### DYNCFG_REGISTER_MODULE
85
86
```
95
-Agent->plugin:
96
-=============
97
-FUNCTION [UID] 1 "get_plugin_config"
98
-
99
-Plugin->agent:
100
-=============
101
-FUNCTION_RESULT_BEGIN [UID] 1 text/plain 5
102
-{
103
- "the currently used config from plugin" : "nice"
104
-}
105
-FUNCTION_RESULT_END
87
+DYNCFG_REGISTER_MODULE {MODULE_NAME} {MODULE_TYPE}
88
```
89
108
-and
90
+Module has to choose one of following types at registration:
91
+
92
+- `single` - module itself has configuration but does not accept any jobs *(this is useful mainly for internal netdata
93
+ configurable things like webserver etc.)*
94
+
95
+- `job_array` - module itself **can** *(not must)* have configuration and it has an array of jobs which can be added,
96
+ modified and deleted. **this is what plugin developer needs in most cases**
97
+
98
+After a module has been registered agent can call `set_module_config`, `get_module_config` and `get_module_config_schema`.
99
+
100
+When `MODULE_TYPE` is `job_array` the agent may also send `set_job_config`, `get_job_config` and `get_job_config_schema`.
101
+
102
+#### DYNCFG_REGISTER_JOB
103
+
104
+The plugin can use `DYNCFG_REGISTER_JOB` to register its own configuration jobs. It should not register jobs configured
105
+via DYNCFG (doing so, the agent will shutdown the plugin).
106
+
107
108
```
111
-Agent->plugin:
112
-=============
113
-FUNCTION [UID] 1 "get_plugin_config_schema"
114
-
115
-Plugin->agent:
116
-=============
117
-FUNCTION_RESULT_BEGIN [UID] 1 text/plain 5
118
-{
119
- "the schema of plugin configuration" : "splendid"
120
-}
121
-FUNCTION_RESULT_END
109
+DYNCFG_REGISTER_JOB {MODULE_NAME} {JOB_NAME} {JOB_TYPE} {FLAGS}
110
```
111
124
-Plugin can also register zero, one or more configurable modules using:
112
+Where:
113
+
114
+- `MODULE_NAME` is the name of the module.
115
+- `JOB_NAME` is the name of the job.
116
+- `JOB_TYPE` is either `stock` or `autodiscovered`.
117
+- `FLAGS`, just send zero.
118
+
119
+#### REPORT_JOB_STATUS
120
121
```
127
-plugin->agent:
128
-=============
129
-DYNCFG_REGISTER_MODULE [module_url_name] (job_array|single)
122
+REPORT_JOB_STATUS {MODULE_NAME} {JOB_NAME} {STATUS} {STATE} "{REASON}"
123
```
124
132
-modules can be added any time during plugins lifetime (you are not required to add them all at startup).
125
+Where:
126
134
-### 2. DYNCFG_REGISTER_MODULE
127
+- `MODULE_NAME` is the name of the module.
128
+- `JOB_NAME` is the name of the job.
129
+- `STATUS` is one of `stopped`, `running`, or `error`.
130
+- `STATE`, just send zero.
131
+- `REASON` is a message describing the status.
132
136
-Module has to choose one of following types at registration:
133
138
-- `single` - module itself has configuration but does not accept any jobs *(this is useful mainly for internal netdata
139
- configurable things like webserver etc.)*
140
-- `job_array` - module itself **can** *(not must)* have configuration and it has an array of jobs which can be added,
141
- modified and deleted. **this is what plugin developer needs in most cases**
134
+### Commands plugins must serve
135
143
-After module has been registered agent can call
136
+Once a plugin calls `DYNCFG_ENABLE`, the must be able to handle these calls.
137
145
-- `set_module_config [module]` FUNCTION_PAYLOAD
146
-- `get_module_config [module]` FUNCTION
147
-- `get_module_config_schema [module]` FUNCTION
138
+function|parameters|prerequisites|request payload|response payload|
139
+:---:|:---:|:---:|:---:|:---:|
140
+`set_plugin_config`|none|`DYNCFG_ENABLE`|plugin configuration|none|
141
+`get_plugin_config`|none|`DYNCFG_ENABLE`|none|plugin configuration|
142
+`get_plugin_config_schema`|none|`DYNCFG_ENABLE`|none|plugin configuration schema|
143
+`set_module_config`|`module_name`|`DYNCFG_REGISTER_MODULE`|module configuration|none|
144
+`get_module_config`|`module_name`|`DYNCFG_REGISTER_MODULE`|none|module configuration|
145
+`get_module_config_schema`|`module_name`|`DYNCFG_REGISTER_MODULE`|none|module configuration schema|
146
+`set_job_config`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|job configuration|none|
147
+`get_job_config`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|none|job configuration|
148
+`get_job_config_schema`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|none|job configuration schema|
149
149
-with same syntax as `set_plugin_config` and `get_plugin_config`. In case of `set` command the plugin has ability to
150
-reject the new configuration pushed to it.
150
+All of them work like this:
151
152
-In a case the module was registered as `job_array` type following commands can be used to manage jobs:
152
+If the request payload is `none`, then the request looks like this:
153
154
-### 3. Job interface for job_array modules
154
+```bash
155
+FUNCTION {TRANSACTION_UUID} {TIMEOUT_SECONDS} "{function} {parameters}"
156
+```
157
+
158
+When there is payload, the request looks like this:
159
+
160
+```bash
161
+FUNCTION_PAYLOAD {TRANSACTION_UUID} {TIMEOUT_SECONDS} "{function} {parameters}"
162
+<payload>
163
+FUNCTION_PAYLOAD_END
164
+```
165
+
166
+In all cases, the response is like this:
167
+
168
+```bash
169
+FUNCTION_RESULT_BEGIN {TRANSACTION_UUID} {HTTP_RESPONSE_CODE} "{CONTENT_TYPE}" {EXPIRATION_TIMESTAMP}
170
+<payload>
171
+FUNCTION_RESULT_END
172
+```
173
+Where:
174
+- `TRANSACTION_UUID` is the same UUID received with the request.
175
+- `HTTP_RESPONSE_CODE` is either `0` (rejected) or `1` (accepted).
176
+- `CONTENT_TYPE` should reflect the `payload` returned.
177
+- `EXPIRATION_TIMESTAMP` can be zero.
178
156
-- `get_job_config_schema [module]` - FUNCTION
157
-- `get_job_config [module] [job]` - FUNCTION
158
-- `set_job_config [module] [job]` - FUNCTION_PAYLOAD
159
-- `delete_job_name [module] [job]` - FUNCTION
179
161
-### 4. Streaming
180
+## DYNCFG with streaming
181
163
-When above commands are transferred trough streaming additionally `plugin_name` is prefixed as first parameter. This is
182
+When above commands are transferred trough streaming additionally `plugin_name` is prefixed as first parameter. This is
183
done to allow routing to appropriate plugin @child.
184
166
-As a plugin developer you don't need to concern yourself with this detail as that parameter is stripped when sent to the
185
+As a plugin developer you don't need to concern yourself with this detail as that parameter is stripped when sent to the
186
plugin *(and added when sent trough streaming)* automagically.
libnetdata/dyn_conf/dyn_conf.h
+1
-1
@@ -112,7 +112,7 @@ static inline const char* job_type2str(enum job_type type)
112
}
113
}
114
115
-static inline enum job_type str2job_type(const char *type_name)
115
+static inline enum job_type dyncfg_str2job_type(const char *type_name)
116
{
117
if (strcmp(type_name, "stock") == 0)
118
return JOB_TYPE_STOCK;