master
c 495 lines 18.4 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "health_internals.h"
4
5 #define HEALTH_CMDAPI_CMD_SILENCEALL "SILENCE ALL"
6 #define HEALTH_CMDAPI_CMD_DISABLEALL "DISABLE ALL"
7 #define HEALTH_CMDAPI_CMD_SILENCE "SILENCE"
8 #define HEALTH_CMDAPI_CMD_DISABLE "DISABLE"
9 #define HEALTH_CMDAPI_CMD_RESET "RESET"
10 #define HEALTH_CMDAPI_CMD_LIST "LIST"
11
12 #define HEALTH_CMDAPI_MSG_AUTHERROR "Auth Error\n"
13 #define HEALTH_CMDAPI_MSG_SILENCEALL "All alarm notifications are silenced\n"
14 #define HEALTH_CMDAPI_MSG_DISABLEALL "All health checks are disabled\n"
15 #define HEALTH_CMDAPI_MSG_RESET "All health checks and notifications are enabled\n"
16 #define HEALTH_CMDAPI_MSG_DISABLE "Health checks disabled for alarms matching the selectors\n"
17 #define HEALTH_CMDAPI_MSG_SILENCE "Alarm notifications silenced for alarms matching the selectors\n"
18 #define HEALTH_CMDAPI_MSG_ADDED "Alarm selector added\n"
19 #define HEALTH_CMDAPI_MSG_STYPEWARNING "WARNING: Added alarm selector to silence/disable alarms without a SILENCE or DISABLE command.\n"
20 #define HEALTH_CMDAPI_MSG_NOSELECTORWARNING "WARNING: SILENCE or DISABLE command is ineffective without defining any alarm selectors.\n"
21
22 SILENCERS *silencers;
23
24 /**
25 * Create Silencer
26 *
27 * Allocate a new silencer to Netdata.
28 *
29 * @return It returns the address off the silencer on success and NULL otherwise
30 */
31 SILENCER *create_silencer(void) {
32 SILENCER *t = callocz(1, sizeof(SILENCER));
33 netdata_log_debug(D_HEALTH, "HEALTH command API: Created empty silencer");
34
35 return t;
36 }
37
38 /**
39 * Health Silencers add
40 *
41 * Add more one silencer to the list of silencers.
42 *
43 * @param silencer
44 */
45 void health_silencers_add(SILENCER *silencer) {
46 // Add the created instance to the linked list in silencers
47 silencer->next = silencers->silencers;
48 silencers->silencers = silencer;
49 netdata_log_debug(
50 D_HEALTH,
51 "HEALTH command API: Added silencer %s:%s:%s:%s",
52 silencer->alarms,
53 silencer->charts,
54 silencer->contexts,
55 silencer->hosts);
56 }
57
58 /**
59 * Silencers Add Parameter
60 *
61 * Create a new silencer and adjust the variables
62 *
63 * @param silencer a pointer to the silencer that will be adjusted
64 * @param key the key value sent by client
65 * @param value the value sent to the key
66 *
67 * @return It returns the silencer configured on success and NULL otherwise
68 */
69 SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) {
70 static uint32_t
71 hash_alarm = 0,
72 hash_template = 0,
73 hash_chart = 0,
74 hash_context = 0,
75 hash_host = 0;
76
77 if (unlikely(!hash_alarm)) {
78 hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
79 hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
80 hash_chart = simple_uhash(HEALTH_CHART_KEY);
81 hash_context = simple_uhash(HEALTH_CONTEXT_KEY);
82 hash_host = simple_uhash(HEALTH_HOST_KEY);
83 }
84
85 uint32_t hash = simple_uhash(key);
86 if (unlikely(silencer == NULL)) {
87 if (
88 (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) ||
89 (hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) ||
90 (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) ||
91 (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) ||
92 (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY))
93 ) {
94 silencer = create_silencer();
95 }
96 }
97
98 if (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
99 silencer->alarms = strdupz(value);
100 silencer->alarms_pattern = simple_pattern_create(silencer->alarms, NULL, SIMPLE_PATTERN_EXACT, true);
101 } else if (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) {
102 silencer->charts = strdupz(value);
103 silencer->charts_pattern = simple_pattern_create(silencer->charts, NULL, SIMPLE_PATTERN_EXACT, true);
104 } else if (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) {
105 silencer->contexts = strdupz(value);
106 silencer->contexts_pattern = simple_pattern_create(silencer->contexts, NULL, SIMPLE_PATTERN_EXACT, true);
107 } else if (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) {
108 silencer->hosts = strdupz(value);
109 silencer->hosts_pattern = simple_pattern_create(silencer->hosts, NULL, SIMPLE_PATTERN_EXACT, true);
110 }
111
112 return silencer;
113 }
114
115 /**
116 * JSON Read Callback
117 *
118 * Callback called by netdata to create the silencer.
119 *
120 * @param e the main json structure
121 *
122 * @return It always return 0.
123 */
124 int health_silencers_json_read_callback(JSON_ENTRY *e)
125 {
126 switch(e->type) {
127 case JSON_OBJECT:
128 #ifndef ENABLE_JSONC
129 e->callback_function = health_silencers_json_read_callback;
130 if(strcmp(e->name,"")) {
131 // init silencer
132 netdata_log_debug(D_HEALTH, "JSON: Got object with a name, initializing new silencer for %s",e->name);
133 #endif
134 e->callback_data = create_silencer();
135 if(e->callback_data) {
136 health_silencers_add(e->callback_data);
137 }
138 #ifndef ENABLE_JSONC
139 }
140 #endif
141 break;
142
143 case JSON_ARRAY:
144 e->callback_function = health_silencers_json_read_callback;
145 break;
146
147 case JSON_STRING:
148 if(!strcmp(e->name,"type")) {
149 netdata_log_debug(D_HEALTH, "JSON: Processing type=%s",e->data.string);
150 if (!strcmp(e->data.string,"SILENCE")) silencers->stype = STYPE_SILENCE_NOTIFICATIONS;
151 else if (!strcmp(e->data.string,"DISABLE")) silencers->stype = STYPE_DISABLE_ALARMS;
152 } else {
153 netdata_log_debug(D_HEALTH, "JSON: Adding %s=%s", e->name, e->data.string);
154 if (e->callback_data)
155 (void)health_silencers_addparam(e->callback_data, e->name, e->data.string);
156 }
157 break;
158
159 case JSON_BOOLEAN:
160 netdata_log_debug(D_HEALTH, "JSON: Processing all_alarms");
161 silencers->all_alarms=e->data.boolean?1:0;
162 break;
163
164 case JSON_NUMBER:
165 case JSON_NULL:
166 break;
167 }
168
169 return 0;
170 }
171
172 /**
173 * Initialize Global Silencers
174 *
175 * Initialize the silencer for the whole netdata system.
176 *
177 * @return It returns 0 on success and -1 otherwise
178 */
179 int health_initialize_global_silencers() {
180 silencers = mallocz(sizeof(SILENCERS));
181 silencers->all_alarms = 0;
182 silencers->stype = STYPE_NONE;
183 silencers->silencers = NULL;
184
185 return 0;
186 }
187
188 // ----------------------------------------------------------------------------
189
190 /**
191 * Free Silencers
192 *
193 * Clean the silencer structure
194 *
195 * @param t is the structure that will be cleaned.
196 */
197 void free_silencers(SILENCER *t) {
198 if (!t) return;
199
200 while(t) {
201 SILENCER *next = t->next;
202
203 simple_pattern_free(t->alarms_pattern);
204 simple_pattern_free(t->charts_pattern);
205 simple_pattern_free(t->contexts_pattern);
206 simple_pattern_free(t->hosts_pattern);
207 freez(t->alarms);
208 freez(t->charts);
209 freez(t->contexts);
210 freez(t->hosts);
211 freez(t);
212
213 t = next;
214 }
215 }
216
217 /**
218 * Silencers to JSON Entry
219 *
220 * Fill the buffer with the other values given.
221 *
222 * @param wb a pointer to the output buffer
223 * @param var the json variable
224 * @param val the json value
225 * @param hasprev has it a previous value?
226 *
227 * @return
228 */
229 int health_silencers2json_entry(BUFFER *wb, char* var, char* val, int hasprev) {
230 if (val) {
231 buffer_sprintf(wb, "%s\n\t\t\t\"%s\": \"%s\"", (hasprev)?",":"", var, val);
232 return 1;
233 } else {
234 return hasprev;
235 }
236 }
237
238 /**
239 * Silencer to JSON
240 *
241 * Write the silencer values using JSON format inside a buffer.
242 *
243 * @param wb is the buffer to write the silencers.
244 */
245 void health_silencers2json(BUFFER *wb) {
246 buffer_sprintf(wb, "{\n\t\"all\": %s,"
247 "\n\t\"type\": \"%s\","
248 "\n\t\"silencers\": [",
249 (silencers->all_alarms)?"true":"false",
250 (silencers->stype == STYPE_NONE)?"None":((silencers->stype == STYPE_DISABLE_ALARMS)?"DISABLE":"SILENCE"));
251
252 SILENCER *silencer;
253 int i = 0, j = 0;
254 for(silencer = silencers->silencers; silencer ; silencer = silencer->next) {
255 if(likely(i)) buffer_strcat(wb, ",");
256 buffer_strcat(wb, "\n\t\t{");
257 j=health_silencers2json_entry(wb, HEALTH_ALARM_KEY, silencer->alarms, j);
258 j=health_silencers2json_entry(wb, HEALTH_CHART_KEY, silencer->charts, j);
259 j=health_silencers2json_entry(wb, HEALTH_CONTEXT_KEY, silencer->contexts, j);
260 j=health_silencers2json_entry(wb, HEALTH_HOST_KEY, silencer->hosts, j);
261 j=0;
262 buffer_strcat(wb, "\n\t\t}");
263 i++;
264 }
265 if(likely(i)) buffer_strcat(wb, "\n\t");
266 buffer_strcat(wb, "]\n}\n");
267 }
268
269
270 /**
271 * Silencer to FILE
272 *
273 * Write the silencer buffer to a file.
274 * @param wb
275 */
276 void health_silencers2file(BUFFER *wb) {
277 if (wb->len == 0) return;
278
279 FILE *fd = fopen(health_silencers_filename(), "wb");
280 if(fd) {
281 size_t written = (size_t)fprintf(fd, "%s", wb->buffer) ;
282 if (written == wb->len ) {
283 netdata_log_info("Silencer changes written to %s", health_silencers_filename());
284 }
285 fclose(fd);
286 return;
287 }
288 netdata_log_error("Silencer changes could not be written to %s. Error %s", health_silencers_filename(), strerror(errno));
289 }
290
291 /**
292 * Request V1 MGMT Health
293 *
294 * Function called by api to management the health.
295 *
296 * @param host main structure with client information!
297 * @param w is the structure with all information of the client request.
298 * @param url is the url that netdata is working
299 *
300 * @return It returns 200 on success and another code otherwise.
301 */
302 int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, char *url) {
303 int ret;
304 (void) host;
305
306 BUFFER *wb = w->response.data;
307 buffer_flush(wb);
308 wb->content_type = CT_TEXT_PLAIN;
309
310 buffer_flush(w->response.data);
311
312 //Local instance of the silencer
313 SILENCER *silencer = NULL;
314 int config_changed = 1;
315
316 if (!w->auth_bearer_token) {
317 buffer_strcat(wb, HEALTH_CMDAPI_MSG_AUTHERROR);
318 ret = HTTP_RESP_FORBIDDEN;
319 } else {
320 netdata_log_debug(D_HEALTH, "HEALTH command API: Comparing secret '%s' to '%s'", w->auth_bearer_token, api_secret);
321 if (strcmp(w->auth_bearer_token, api_secret) != 0) {
322 buffer_strcat(wb, HEALTH_CMDAPI_MSG_AUTHERROR);
323 ret = HTTP_RESP_FORBIDDEN;
324 } else {
325 while (url) {
326 char *value = strsep_skip_consecutive_separators(&url, "&");
327 if (!value || !*value) continue;
328
329 char *key = strsep_skip_consecutive_separators(&value, "=");
330 if (!key || !*key) continue;
331 if (!value || !*value) continue;
332
333 netdata_log_debug(D_WEB_CLIENT, "%llu: API v1 health query param '%s' with value '%s'", w->id, key, value);
334
335 // name and value are now the parameters
336 if (!strcmp(key, "cmd")) {
337 if (!strcmp(value, HEALTH_CMDAPI_CMD_SILENCEALL)) {
338 silencers->all_alarms = 1;
339 silencers->stype = STYPE_SILENCE_NOTIFICATIONS;
340 buffer_strcat(wb, HEALTH_CMDAPI_MSG_SILENCEALL);
341 } else if (!strcmp(value, HEALTH_CMDAPI_CMD_DISABLEALL)) {
342 silencers->all_alarms = 1;
343 silencers->stype = STYPE_DISABLE_ALARMS;
344 buffer_strcat(wb, HEALTH_CMDAPI_MSG_DISABLEALL);
345 } else if (!strcmp(value, HEALTH_CMDAPI_CMD_SILENCE)) {
346 silencers->stype = STYPE_SILENCE_NOTIFICATIONS;
347 buffer_strcat(wb, HEALTH_CMDAPI_MSG_SILENCE);
348 } else if (!strcmp(value, HEALTH_CMDAPI_CMD_DISABLE)) {
349 silencers->stype = STYPE_DISABLE_ALARMS;
350 buffer_strcat(wb, HEALTH_CMDAPI_MSG_DISABLE);
351 } else if (!strcmp(value, HEALTH_CMDAPI_CMD_RESET)) {
352 silencers->all_alarms = 0;
353 silencers->stype = STYPE_NONE;
354 free_silencers(silencers->silencers);
355 silencers->silencers = NULL;
356 buffer_strcat(wb, HEALTH_CMDAPI_MSG_RESET);
357 } else if (!strcmp(value, HEALTH_CMDAPI_CMD_LIST)) {
358 w->response.data->content_type = CT_APPLICATION_JSON;
359 health_silencers2json(wb);
360 config_changed=0;
361 }
362 } else {
363 silencer = health_silencers_addparam(silencer, key, value);
364 }
365 }
366
367 if (likely(silencer)) {
368 health_silencers_add(silencer);
369 buffer_strcat(wb, HEALTH_CMDAPI_MSG_ADDED);
370 if (silencers->stype == STYPE_NONE) {
371 buffer_strcat(wb, HEALTH_CMDAPI_MSG_STYPEWARNING);
372 }
373 }
374 if (unlikely(silencers->stype != STYPE_NONE && !silencers->all_alarms && !silencers->silencers)) {
375 buffer_strcat(wb, HEALTH_CMDAPI_MSG_NOSELECTORWARNING);
376 }
377 ret = HTTP_RESP_OK;
378 }
379 }
380 w->response.data = wb;
381 buffer_no_cacheable(w->response.data);
382 if (ret == HTTP_RESP_OK && config_changed) {
383 BUFFER *jsonb = buffer_create(200, &netdata_buffers_statistics.buffers_health);
384 health_silencers2json(jsonb);
385 health_silencers2file(jsonb);
386 buffer_free(jsonb);
387 }
388
389 return ret;
390 }
391
392 // ----------------------------------------------------------------------------
393
394 const char *health_silencers_filename(void) {
395 return string2str(health_globals.config.silencers_filename);
396 }
397
398 void health_set_silencers_filename(void) {
399 char filename[FILENAME_MAX + 1];
400 snprintfz(filename, FILENAME_MAX, "%s/health.silencers.json", netdata_configured_varlib_dir);
401
402 health_globals.config.silencers_filename =
403 string_strdupz(inicfg_get_filename(&netdata_config, CONFIG_SECTION_HEALTH, "silencers file", filename));
404 }
405
406 void health_silencers_init(void) {
407 FILE *fd = fopen(health_silencers_filename(), "r");
408 if (fd) {
409 fseek(fd, 0 , SEEK_END);
410 off_t length = (off_t) ftell(fd);
411 fseek(fd, 0 , SEEK_SET);
412
413 if (length > 0 && length < HEALTH_SILENCERS_MAX_FILE_LEN) {
414 char *str = mallocz((length+1)* sizeof(char));
415 if(str) {
416 size_t copied;
417 copied = fread(str, sizeof(char), length, fd);
418 if (copied == (length* sizeof(char))) {
419 str[length] = 0x00;
420 json_parse(str, NULL, health_silencers_json_read_callback);
421 netdata_log_info("Parsed health silencers file %s", health_silencers_filename());
422 } else {
423 netdata_log_error("Cannot read the data from health silencers file %s", health_silencers_filename());
424 }
425 freez(str);
426 }
427 } else {
428 netdata_log_error("Health silencers file %s has the size %" PRId64 " that is out of range[ 1 , %d ]. Aborting read.",
429 health_silencers_filename(),
430 (int64_t)length,
431 HEALTH_SILENCERS_MAX_FILE_LEN);
432 }
433 fclose(fd);
434 } else {
435 netdata_log_info("Cannot open the file %s, so Netdata will work with the default health configuration.",
436 health_silencers_filename());
437 }
438 }
439
440 SILENCE_TYPE health_silencers_check_silenced(RRDCALC *rc, const char *host) {
441 SILENCER *s;
442
443 for (s = silencers->silencers; s!=NULL; s=s->next){
444 if (
445 (!s->alarms_pattern || (rc->config.name && s->alarms_pattern && simple_pattern_matches_string(s->alarms_pattern, rc->config.name))) &&
446 (!s->contexts_pattern || (rc->rrdset && rc->rrdset->context && s->contexts_pattern && simple_pattern_matches_string(s->contexts_pattern, rc->rrdset->context))) &&
447 (!s->hosts_pattern || (host && s->hosts_pattern && simple_pattern_matches(s->hosts_pattern, host))) &&
448 (!s->charts_pattern || (rc->chart && s->charts_pattern && simple_pattern_matches_string(s->charts_pattern, rc->chart)))
449 ) {
450 netdata_log_debug(D_HEALTH, "Alarm matches command API silence entry %s:%s:%s:%s", s->alarms,s->charts, s->contexts, s->hosts);
451 if (unlikely(silencers->stype == STYPE_NONE)) {
452 netdata_log_debug(D_HEALTH, "Alarm %s matched a silence entry, but no SILENCE or DISABLE command was issued via the command API. The match has no effect.", rrdcalc_name(rc));
453 } else {
454 netdata_log_debug(D_HEALTH, "Alarm %s via the command API - name:%s context:%s chart:%s host:%s"
455 , (silencers->stype == STYPE_DISABLE_ALARMS)?"Disabled":"Silenced"
456 , rrdcalc_name(rc)
457 , (rc->rrdset)?rrdset_context(rc->rrdset):""
458 , rrdcalc_chart_name(rc)
459 , host
460 );
461 }
462 return silencers->stype;
463 }
464 }
465 return STYPE_NONE;
466 }
467
468 int health_silencers_update_disabled_silenced(RRDHOST *host, RRDCALC *rc) {
469 uint32_t rrdcalc_flags_old = rc->run_flags;
470 // Clear the flags
471 rc->run_flags &= ~(RRDCALC_FLAG_DISABLED | RRDCALC_FLAG_SILENCED);
472 if (unlikely(silencers->all_alarms)) {
473 if (silencers->stype == STYPE_DISABLE_ALARMS) rc->run_flags |= RRDCALC_FLAG_DISABLED;
474 else if (silencers->stype == STYPE_SILENCE_NOTIFICATIONS) rc->run_flags |= RRDCALC_FLAG_SILENCED;
475 } else {
476 SILENCE_TYPE st = health_silencers_check_silenced(rc, rrdhost_hostname(host));
477 if (st == STYPE_DISABLE_ALARMS) rc->run_flags |= RRDCALC_FLAG_DISABLED;
478 else if (st == STYPE_SILENCE_NOTIFICATIONS) rc->run_flags |= RRDCALC_FLAG_SILENCED;
479 }
480
481 if (rrdcalc_flags_old != rc->run_flags) {
482 netdata_log_info(
483 "Alarm silencing changed for host '%s' alarm '%s': Disabled %s->%s Silenced %s->%s",
484 rrdhost_hostname(host),
485 rrdcalc_name(rc),
486 (rrdcalc_flags_old & RRDCALC_FLAG_DISABLED) ? "true" : "false",
487 (rc->run_flags & RRDCALC_FLAG_DISABLED) ? "true" : "false",
488 (rrdcalc_flags_old & RRDCALC_FLAG_SILENCED) ? "true" : "false",
489 (rc->run_flags & RRDCALC_FLAG_SILENCED) ? "true" : "false");
490 }
491 if (rc->run_flags & RRDCALC_FLAG_DISABLED)
492 return 1;
493 else
494 return 0;
495 }