Keep health log history in seconds (#15314)
* rebase * changes queries to delete based on when * readme changes * no need to do migration * wip, protect un-updated events from cleanup * remove index on when_key * fix query for claimed cleanup * if set less than minimum, set minimum * fix query * correct config assign
Emmanuel Vasilakis committed
Jul 12, 2023 at 11:24 UTC
38b38993a6547aa33a0591a7ce3e7461c197e893
11 files changed
+84
-35
daemon/config/README.md
+1
-1
@@ -175,7 +175,7 @@ monitoring](https://github.com/netdata/netdata/blob/master/health/README.md).
175
| script to execute on alarm | `/usr/libexec/netdata/plugins.d/alarm-notify.sh` | The script that sends alarm notifications. Note that in versions before 1.16, the plugins.d directory may be installed in a different location in certain OSs (e.g. under `/usr/lib/netdata`). |
176
| run at least every seconds | `10` | Controls how often all alarm conditions should be evaluated. |
177
| postpone alarms during hibernation for seconds | `60` | Prevents false alarms. May need to be increased if you get alarms during hibernation. |
178
-| rotate log every lines | 2000 | Controls the number of alarm log entries stored in `<lib directory>/health-log.db`, where `<lib directory>` is the one configured in the [\[global\] section](#global-section-options) |
178
+| health log history | `432000` | Specifies the history of alarm events (in seconds) kept in the agent's sqlite database. |
179
| enabled alarms | * | Defines which alarms to load from both user and stock directories. This is a [simple pattern](https://github.com/netdata/netdata/blob/master/libnetdata/simple_pattern/README.md) list of alarm or template names. Can be used to disable specific alarms. For example, `enabled alarms = !oom_kill *` will load all alarms except `oom_kill`. |
180
181
### [web] section options
database/rrd.h
+1
@@ -1105,6 +1105,7 @@ typedef struct alarm_log {
1105
uint32_t next_alarm_id;
1106
unsigned int count;
1107
unsigned int max;
1108
+ uint32_t health_log_history; // the health log history in seconds to be kept in db
1109
ALARM_ENTRY *alarms;
1110
RW_SPINLOCK spinlock;
1111
} ALARM_LOG;
database/sqlite/sqlite_functions.c
-2
@@ -49,7 +49,6 @@ const char *database_config[] = {
49
"config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, "
50
"chart_context text, last_transition_id blob, UNIQUE (host_id, alarm_id)) ;",
51
52
- //TODO indexes
52
"CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id);",
53
54
"CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, "
@@ -62,7 +61,6 @@ const char *database_config[] = {
61
"CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id);",
62
"CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id);",
63
"CREATE INDEX IF NOT EXISTS health_log_d_ind_4 ON health_log_detail (health_log_id);",
65
- //TODO more indexes
64
65
NULL
66
};
database/sqlite/sqlite_health.c
+41
-25
@@ -393,8 +393,8 @@ void sql_health_alarm_log_count(RRDHOST *host) {
393
/* Health related SQL queries
394
Cleans up the health_log_detail table on a non-claimed host
395
*/
396
-#define SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED(limit) "DELETE FROM health_log_detail where health_log_id in (select health_log_id from health_log where host_id = @host_id) ORDER BY unique_id ASC LIMIT %lu;", limit
397
-void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host, size_t rotate_every) {
396
+#define SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED "DELETE FROM health_log_detail WHERE health_log_id IN (SELECT health_log_id FROM health_log WHERE host_id = ?1) AND when_key + ?2 < unixepoch() AND updated_by_id <> 0 AND transition_id NOT IN (SELECT last_transition_id FROM health_log hl WHERE hl.host_id = ?3);"
397
+void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host) {
398
sqlite3_stmt *res = NULL;
399
int rc;
400
char command[MAX_HEALTH_SQL_SIZE + 1];
@@ -408,9 +408,7 @@ void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host, size_t rotate_every
408
char uuid_str[UUID_STR_LEN];
409
uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
410
411
- snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED((unsigned long int) (host->health.health_log_entries_written - rotate_every)));
412
-
413
- rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
411
+ rc = sqlite3_prepare_v2(db_meta, SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED, -1, &res, 0);
412
if (unlikely(rc != SQLITE_OK)) {
413
error_report("Failed to prepare statement to cleanup health log detail table (un-claimed)");
414
return;
@@ -423,15 +421,29 @@ void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host, size_t rotate_every
421
return;
422
}
423
424
+ rc = sqlite3_bind_int64(res, 2, (sqlite3_int64)host->health_log.health_log_history);
425
+ if (unlikely(rc != SQLITE_OK)) {
426
+ error_report("Failed to bind health log history for SQL_CLEANUP_HEALTH_LOG_NOT_CLAIMED.");
427
+ sqlite3_finalize(res);
428
+ return;
429
+ }
430
+
431
+ rc = sqlite3_bind_blob(res, 3, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
432
+ if (unlikely(rc != SQLITE_OK)) {
433
+ error_report("Failed to bind host_id for SQL_CLEANUP_HEALTH_LOG_NOT_CLAIMED.");
434
+ sqlite3_finalize(res);
435
+ return;
436
+ }
437
+
438
rc = sqlite3_step_monitored(res);
439
if (unlikely(rc != SQLITE_DONE))
428
- error_report("Failed to cleanup health log table, rc = %d", rc);
440
+ error_report("Failed to cleanup health log detail table, rc = %d", rc);
441
442
rc = sqlite3_finalize(res);
443
if (unlikely(rc != SQLITE_OK))
444
error_report("Failed to finalize the prepared statement to cleanup health log detail table (un-claimed)");
445
434
- host->health.health_log_entries_written = rotate_every;
446
+ sql_health_alarm_log_count(host);
447
448
snprintfz(command, MAX_HEALTH_SQL_SIZE, "aclk_alert_%s", uuid_str);
449
if (unlikely(table_exists_in_database(command))) {
@@ -442,8 +454,8 @@ void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host, size_t rotate_every
454
/* Health related SQL queries
455
Cleans up the health_log_detail table on a claimed host
456
*/
445
-#define SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(guid, limit) "DELETE from health_log_detail WHERE unique_id NOT IN (SELECT filtered_alert_unique_id FROM aclk_alert_%s) AND unique_id IN (SELECT hld.unique_id FROM health_log hl, health_log_detail hld WHERE hl.host_id = ?1 AND hl.health_log_id = hld.health_log_id) and health_log_id in (SELECT health_log_id FROM health_log WHERE host_id = ?2) ORDER BY unique_id asc LIMIT %lu;", guid, limit
446
-void sql_health_alarm_log_cleanup_claimed(RRDHOST *host, size_t rotate_every) {
457
+#define SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(guid) "DELETE from health_log_detail WHERE unique_id NOT IN (SELECT filtered_alert_unique_id FROM aclk_alert_%s) AND unique_id IN (SELECT hld.unique_id FROM health_log hl, health_log_detail hld WHERE hl.host_id = ?1 AND hl.health_log_id = hld.health_log_id) AND health_log_id IN (SELECT health_log_id FROM health_log WHERE host_id = ?2) AND when_key + ?3 < unixepoch() AND updated_by_id <> 0 AND transition_id NOT IN (SELECT last_transition_id FROM health_log hl WHERE hl.host_id = ?4);", guid
458
+void sql_health_alarm_log_cleanup_claimed(RRDHOST *host) {
459
sqlite3_stmt *res = NULL;
460
int rc;
461
char command[MAX_HEALTH_SQL_SIZE + 1];
@@ -459,11 +471,11 @@ void sql_health_alarm_log_cleanup_claimed(RRDHOST *host, size_t rotate_every) {
471
snprintfz(command, MAX_HEALTH_SQL_SIZE, "aclk_alert_%s", uuid_str);
472
473
if (!table_exists_in_database(command)) {
462
- sql_health_alarm_log_cleanup_not_claimed(host, rotate_every);
474
+ sql_health_alarm_log_cleanup_not_claimed(host);
475
return;
476
}
477
466
- snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(uuid_str, (unsigned long int) (host->health.health_log_entries_written - rotate_every)));
478
+ snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(uuid_str));
479
480
rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
481
if (unlikely(rc != SQLITE_OK)) {
@@ -485,9 +497,23 @@ void sql_health_alarm_log_cleanup_claimed(RRDHOST *host, size_t rotate_every) {
497
return;
498
}
499
500
+ rc = sqlite3_bind_int64(res, 3, (sqlite3_int64)host->health_log.health_log_history);
501
+ if (unlikely(rc != SQLITE_OK)) {
502
+ error_report("Failed to bind health log history for SQL_CLEANUP_HEALTH_LOG_CLAIMED.");
503
+ sqlite3_finalize(res);
504
+ return;
505
+ }
506
+
507
+ rc = sqlite3_bind_blob(res, 4, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
508
+ if (unlikely(rc != SQLITE_OK)) {
509
+ error_report("Failed to bind second host_id for SQL_CLEANUP_HEALTH_LOG_CLAIMED.");
510
+ sqlite3_finalize(res);
511
+ return;
512
+ }
513
+
514
rc = sqlite3_step_monitored(res);
515
if (unlikely(rc != SQLITE_DONE))
490
- error_report("Failed to cleanup health log table, rc = %d", rc);
516
+ error_report("Failed to cleanup health log detail table, rc = %d", rc);
517
518
rc = sqlite3_finalize(res);
519
if (unlikely(rc != SQLITE_OK))
@@ -496,27 +522,17 @@ void sql_health_alarm_log_cleanup_claimed(RRDHOST *host, size_t rotate_every) {
522
sql_health_alarm_log_count(host);
523
524
sql_aclk_alert_clean_dead_entries(host);
525
+
526
}
527
528
/* Health related SQL queries
529
Cleans up the health_log table.
530
*/
531
void sql_health_alarm_log_cleanup(RRDHOST *host) {
505
- static size_t rotate_every = 0;
506
-
507
- if(unlikely(rotate_every == 0)) {
508
- rotate_every = (size_t)config_get_number(CONFIG_SECTION_HEALTH, "rotate log every lines", 2000);
509
- if(rotate_every < 100) rotate_every = 100;
510
- }
511
-
512
- if(likely(host->health.health_log_entries_written < rotate_every)) {
513
- return;
514
- }
515
-
532
if (!claimed()) {
517
- sql_health_alarm_log_cleanup_not_claimed(host, rotate_every);
533
+ sql_health_alarm_log_cleanup_not_claimed(host);
534
} else
519
- sql_health_alarm_log_cleanup_claimed(host, rotate_every);
535
+ sql_health_alarm_log_cleanup_claimed(host);
536
}
537
538
#define SQL_INJECT_REMOVED "insert into health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, transition_id, global_id) select health_log_id, ?1, ?2, ?3, 0, ?4, unixepoch(), 0, 0, flags, exec_run_timestamp, unixepoch(), info, exec_code, -2, new_status, delay, NULL, new_value, 0, ?5, now_usec(0) from health_log_detail where unique_id = ?6 and transition_id = ?7;"
health/health.c
+17
-5
@@ -22,7 +22,6 @@ char *silencers_filename;
22
SIMPLE_PATTERN *conf_enabled_alarms = NULL;
23
DICTIONARY *health_rrdvars;
24
25
-
25
void health_entry_flags_to_json_array(BUFFER *wb, const char *key, HEALTH_ENTRY_FLAGS flags) {
26
buffer_json_member_add_array(wb, key);
27
@@ -803,15 +802,28 @@ static void initialize_health(RRDHOST *host)
802
803
long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max);
804
if(n < 10) {
806
- netdata_log_error("Host '%s': health configuration has invalid max log entries %ld. Using default %u",
807
- rrdhost_hostname(host),
808
- n,
809
- host->health_log.max);
805
+ log_health("Host '%s': health configuration has invalid max log entries %ld. Using default %u", rrdhost_hostname(host), n, host->health_log.max);
806
config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max);
807
}
808
else
809
host->health_log.max = (unsigned int)n;
810
811
+ uint32_t m = config_get_number(CONFIG_SECTION_HEALTH, "health log history", HEALTH_LOG_DEFAULT_HISTORY);
812
+ if (m < HEALTH_LOG_MINIMUM_HISTORY) {
813
+ log_health("Host '%s': health configuration has invalid health log history %u. Using minimum %d", rrdhost_hostname(host), m, HEALTH_LOG_MINIMUM_HISTORY);
814
+ config_set_number(CONFIG_SECTION_HEALTH, "health log history", HEALTH_LOG_MINIMUM_HISTORY);
815
+ m = HEALTH_LOG_MINIMUM_HISTORY;
816
+ }
817
+
818
+ //default health log history is 5 days and not less than a day
819
+ if (host->health_log.health_log_history) {
820
+ if (host->health_log.health_log_history < HEALTH_LOG_MINIMUM_HISTORY)
821
+ host->health_log.health_log_history = HEALTH_LOG_MINIMUM_HISTORY;
822
+ } else
823
+ host->health_log.health_log_history = m;
824
+
825
+ log_health("[%s]: Health log history is set to %u seconds (%u days)", rrdhost_hostname(host), host->health_log.health_log_history, host->health_log.health_log_history / 86400);
826
+
827
conf_enabled_alarms = simple_pattern_create(config_get(CONFIG_SECTION_HEALTH, "enabled alarms", "*"), NULL,
828
SIMPLE_PATTERN_EXACT, true);
829
health/health.h
+8
@@ -31,6 +31,14 @@ void health_entry_flags_to_json_array(BUFFER *wb, const char *key, HEALTH_ENTRY_
31
#define HEALTH_LISTEN_BACKLOG 4096
32
#endif
33
34
+#ifndef HEALTH_LOG_DEFAULT_HISTORY
35
+#define HEALTH_LOG_DEFAULT_HISTORY 432000
36
+#endif
37
+
38
+#ifndef HEALTH_LOG_MINIMUM_HISTORY
39
+#define HEALTH_LOG_MINIMUM_HISTORY 86400
40
+#endif
41
+
42
#define HEALTH_SILENCERS_MAX_FILE_LEN 10000
43
44
extern char *silencers_filename;
streaming/README.md
+1
@@ -55,6 +55,7 @@ node**. This file is automatically generated by Netdata the first time it is sta
55
| [`default memory mode`](#default-memory-mode) | `ram` | The [database](https://github.com/netdata/netdata/blob/master/database/README.md) to use for all nodes using this `API_KEY`. Valid settings are `dbengine`, `map`, `save`, `ram`, or `none`. [Read more →](#default-memory-mode) |
56
| `health enabled by default` | `auto` | Whether alarms and notifications should be enabled for nodes using this `API_KEY`. `auto` enables alarms when the child is connected. `yes` enables alarms always, and `no` disables alarms. |
57
| `default postpone alarms on connect seconds` | `60` | Postpone alarms and notifications for a period of time after the child connects. |
58
+| `default health log history` | `432000` | History of health log events (in seconds) kept in the database. |
59
| `default proxy enabled` | ` ` | Route metrics through a proxy. |
60
| `default proxy destination` | ` ` | Space-separated list of `IP:PORT` for proxies. |
61
| `default proxy api key` | ` ` | The `API_KEY` of the proxy. |
streaming/receiver.c
+6
@@ -411,6 +411,8 @@ static bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
411
}
412
}
413
414
+ host->health_log.health_log_history = rpt->config.alarms_history;
415
+
416
// this is a test
417
// if(rpt->hops <= host->sender->hops)
418
// rrdpush_sender_thread_stop(host, "HOPS MISMATCH", false);
@@ -552,6 +554,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
554
555
rpt->config.health_enabled = (int)default_health_enabled;
556
rpt->config.alarms_delay = 60;
557
+ rpt->config.alarms_history = HEALTH_LOG_DEFAULT_HISTORY;
558
559
rpt->config.rrdpush_enabled = (int)default_rrdpush_enabled;
560
rpt->config.rrdpush_destination = default_rrdpush_destination;
@@ -588,6 +591,9 @@ static void rrdpush_receive(struct receiver_state *rpt)
591
rpt->config.alarms_delay = appconfig_get_number(&stream_config, rpt->key, "default postpone alarms on connect seconds", rpt->config.alarms_delay);
592
rpt->config.alarms_delay = appconfig_get_number(&stream_config, rpt->machine_guid, "postpone alarms on connect seconds", rpt->config.alarms_delay);
593
594
+ rpt->config.alarms_history = appconfig_get_number(&stream_config, rpt->key, "default health log history", rpt->config.alarms_history);
595
+ rpt->config.alarms_history = appconfig_get_number(&stream_config, rpt->machine_guid, "health log history", rpt->config.alarms_history);
596
+
597
rpt->config.rrdpush_enabled = appconfig_get_boolean(&stream_config, rpt->key, "default proxy enabled", rpt->config.rrdpush_enabled);
598
rpt->config.rrdpush_enabled = appconfig_get_boolean(&stream_config, rpt->machine_guid, "proxy enabled", rpt->config.rrdpush_enabled);
599
streaming/rrdpush.h
+1
@@ -400,6 +400,7 @@ struct receiver_state {
400
int update_every;
401
int health_enabled; // CONFIG_BOOLEAN_YES, CONFIG_BOOLEAN_NO, CONFIG_BOOLEAN_AUTO
402
time_t alarms_delay;
403
+ uint32_t alarms_history;
404
int rrdpush_enabled;
405
char *rrdpush_api_key; // DONT FREE - it is allocated in appconfig
406
char *rrdpush_send_charts_matching; // DONT FREE - it is allocated in appconfig
streaming/stream.conf
+6
@@ -155,6 +155,9 @@
155
# postpone alarms for a short period after the sender is connected
156
default postpone alarms on connect seconds = 60
157
158
+ # seconds of health log events to keep
159
+ #default health log history = 432000
160
+
161
# need to route metrics differently? set these.
162
# the defaults are the ones at the [stream] section (above)
163
#default proxy enabled = yes | no
@@ -223,6 +226,9 @@
226
# postpone alarms when the sender connects
227
postpone alarms on connect seconds = 60
228
229
+ # seconds of health log events to keep
230
+ #health log history = 432000
231
+
232
# need to route metrics differently?
233
# the defaults are the ones at the [API KEY] section
234
#proxy enabled = yes | no
web/api/health/README.md
+2
-2
@@ -28,12 +28,12 @@ This API call will return the alarms currently in WARNING or CRITICAL state.
28
29
### Event Log
30
31
-The size of the alarm log is configured in `netdata.conf`. There are 2 settings: the rotation of the alarm log file and the in memory size of the alarm log.
31
+The size of the alarm log is configured in `netdata.conf`. There are 2 settings: the event history kept in the DB (in seconds), and the in memory size of the alarm log.
32
33
```
34
[health]
35
in memory max health log entries = 1000
36
- rotate log every lines = 2000
36
+ health log history = 432000
37
```
38
39
The API call retrieves all entries of the alarm log: