Reduce alert events sent to the cloud. (#12544)
* filter * update filter * queue removed directly * more * logging * cleanup * cleanup 2 * cleanup 3 * finalize instead of reset
Emmanuel Vasilakis committed
May 2, 2022 at 18:36 UTC
d6b1756ea7c0dc28b338aff67800e932d8fe97ec
7 files changed
+147
-15
database/sqlite/sqlite_aclk.c
+1
-1
@@ -628,7 +628,7 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
628
db_execute(buffer_tostring(sql));
629
buffer_flush(sql);
630
631
- buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str, uuid_str, uuid_str);
631
+ buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str);
632
db_execute(buffer_tostring(sql));
633
buffer_flush(sql);
634
database/sqlite/sqlite_aclk.h
+1
-3
@@ -103,9 +103,7 @@ static inline char *get_str_from_uuid(uuid_t *uuid)
103
104
#define TABLE_ACLK_ALERT "CREATE TABLE IF NOT EXISTS aclk_alert_%s (sequence_id INTEGER PRIMARY KEY, " \
105
"alert_unique_id, date_created, date_submitted, date_cloud_ack, " \
106
- "unique(alert_unique_id)); " \
107
- "insert into aclk_alert_%s (alert_unique_id, date_created) " \
108
- "select unique_id alert_unique_id, strftime('%%s') date_created from health_log_%s where new_status <> 0 and new_status <> -2 order by unique_id asc on conflict (alert_unique_id) do nothing;"
106
+ "unique(alert_unique_id));"
107
108
#define INDEX_ACLK_CHART "CREATE INDEX IF NOT EXISTS aclk_chart_index_%s ON aclk_chart_%s (unique_id);"
109
database/sqlite/sqlite_aclk_alert.c
+142
-8
@@ -8,9 +8,120 @@
8
#include "../../aclk/aclk.h"
9
#endif
10
11
+time_t removed_when(uint32_t alarm_id, uint32_t before_unique_id, uint32_t after_unique_id, char *uuid_str) {
12
+ sqlite3_stmt *res = NULL;
13
+ int rc = 0;
14
+ time_t when = 0;
15
+ char sql[ACLK_SYNC_QUERY_SIZE];
16
+
17
+ snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "select when_key from health_log_%s where alarm_id = %u " \
18
+ "and unique_id > %u and unique_id < %u " \
19
+ "and new_status = -2;", uuid_str, alarm_id, after_unique_id, before_unique_id);
20
+
21
+ rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
22
+ if (rc != SQLITE_OK) {
23
+ error_report("Failed to prepare statement when trying to find removed gap.");
24
+ return 0;
25
+ }
26
+
27
+ rc = sqlite3_step(res);
28
+ if (likely(rc == SQLITE_ROW)) {
29
+ when = (time_t) sqlite3_column_int64(res, 0);
30
+ }
31
+
32
+ rc = sqlite3_finalize(res);
33
+ if (unlikely(rc != SQLITE_OK))
34
+ error_report("Failed to finalize statement when trying to find removed gap, rc = %d", rc);
35
+
36
+ return when;
37
+}
38
+
39
+#define MAX_REMOVED_PERIOD 900
40
+//decide if some events should be sent or not
41
+int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)
42
+{
43
+ sqlite3_stmt *res = NULL;
44
+ char uuid_str[GUID_LEN + 1];
45
+ uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
46
+ int send = 1, rc = 0;
47
+
48
+ if (ae->new_status == RRDCALC_STATUS_REMOVED || ae->new_status == RRDCALC_STATUS_UNINITIALIZED) {
49
+ return 0;
50
+ }
51
+
52
+ if (unlikely(uuid_is_null(ae->config_hash_id)))
53
+ return 0;
54
+
55
+ char sql[ACLK_SYNC_QUERY_SIZE];
56
+ uuid_t config_hash_id;
57
+ RRDCALC_STATUS status;
58
+ uint32_t unique_id;
59
+
60
+ //get the previous sent event of this alarm_id
61
+ snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "select hl.new_status, hl.config_hash_id, hl.unique_id from health_log_%s hl, aclk_alert_%s aa \
62
+ where hl.unique_id = aa.alert_unique_id \
63
+ and hl.alarm_id = %u and hl.unique_id <> %u \
64
+ order by alarm_event_id desc LIMIT 1;", uuid_str, uuid_str, ae->alarm_id, ae->unique_id);
65
+
66
+ rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
67
+ if (rc != SQLITE_OK) {
68
+ error_report("Failed to prepare statement when trying to filter alert events.");
69
+ send = 1;
70
+ return send;
71
+ }
72
+
73
+ rc = sqlite3_step(res);
74
+ if (likely(rc == SQLITE_ROW)) {
75
+ status = (RRDCALC_STATUS) sqlite3_column_int(res, 0);
76
+ if (sqlite3_column_type(res, 1) != SQLITE_NULL)
77
+ uuid_copy(config_hash_id, *((uuid_t *) sqlite3_column_blob(res, 1)));
78
+ unique_id = (uint32_t) sqlite3_column_int64(res, 2);
79
+
80
+ } else {
81
+ send = 1;
82
+ goto done;
83
+ }
84
+
85
+ if (ae->new_status != (RRDCALC_STATUS)status) {
86
+ send = 1;
87
+ goto done;
88
+ }
89
+
90
+ if (uuid_compare(ae->config_hash_id, config_hash_id)) {
91
+ send = 1;
92
+ goto done;
93
+ }
94
+
95
+ //same status, same config
96
+ if (ae->new_status == RRDCALC_STATUS_CLEAR) {
97
+ send = 0;
98
+ goto done;
99
+ }
100
+
101
+ //detect a long off period of the agent, TODO make global
102
+ if (ae->new_status == RRDCALC_STATUS_WARNING || ae->new_status == RRDCALC_STATUS_CRITICAL) {
103
+ time_t when = removed_when(ae->alarm_id, ae->unique_id, unique_id, uuid_str);
104
+
105
+ if (when && (when + (time_t)MAX_REMOVED_PERIOD) < ae->when) {
106
+ send = 1;
107
+ goto done;
108
+ } else {
109
+ send = 0;
110
+ goto done;
111
+ }
112
+ }
113
+
114
+done:
115
+ rc = sqlite3_finalize(res);
116
+ if (unlikely(rc != SQLITE_OK))
117
+ error_report("Failed to finalize statement when trying to filter alert events, rc = %d", rc);
118
+
119
+ return send;
120
+}
121
+
122
// will replace call to aclk_update_alarm in health/health_log.c
123
// and handle both cases
13
-int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae)
124
+int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)
125
{
126
//check aclk architecture and handle old json alarm update to cloud
127
//include also the valid statuses for this case
@@ -30,17 +141,18 @@ int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae)
141
if (!claimed())
142
return 0;
143
33
- if (ae->flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED)
34
- return 0;
35
-
36
- if (ae->new_status == RRDCALC_STATUS_REMOVED || ae->new_status == RRDCALC_STATUS_UNINITIALIZED)
37
- return 0;
38
-
144
if (unlikely(!host->dbsync_worker))
145
return 1;
146
42
- if (unlikely(uuid_is_null(ae->config_hash_id)))
147
+ if (ae->flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED) {
148
return 0;
149
+ }
150
+
151
+ if (!skip_filter) {
152
+ if (!should_send_to_cloud(host, ae)) {
153
+ return 0;
154
+ }
155
+ }
156
157
int rc = 0;
158
@@ -296,6 +408,22 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
408
return;
409
}
410
411
+void sql_queue_existing_alerts_to_aclk(RRDHOST *host)
412
+{
413
+ char uuid_str[GUID_LEN + 1];
414
+ uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
415
+ BUFFER *sql = buffer_create(1024);
416
+
417
+ buffer_sprintf(sql,"insert into aclk_alert_%s (alert_unique_id, date_created) " \
418
+ "select unique_id alert_unique_id, strftime('%%s') date_created from health_log_%s " \
419
+ "where new_status <> 0 and new_status <> -2 and config_hash_id is not null and updated_by_id = 0 " \
420
+ "order by unique_id asc on conflict (alert_unique_id) do nothing;", uuid_str, uuid_str);
421
+
422
+ db_execute(buffer_tostring(sql));
423
+
424
+ buffer_free(sql);
425
+}
426
+
427
void aclk_send_alarm_health_log(char *node_id)
428
{
429
if (unlikely(!node_id))
@@ -593,6 +721,9 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start
721
log_access("ACLK STA [%s (N/A)]: Ignoring request to stream alert state changes, health is disabled.", node_id);
722
return;
723
}
724
+
725
+ if (unlikely(batch_id == 1) && unlikely(start_seq_id == 1))
726
+ sql_queue_existing_alerts_to_aclk(host);
727
} else
728
wc = (struct aclk_database_worker_config *)find_inactive_wc_by_node_id(node_id);
729
@@ -644,6 +775,9 @@ void sql_queue_removed_alerts_to_aclk(RRDHOST *host)
775
if (unlikely(!host->dbsync_worker))
776
return;
777
778
+ if (!claimed())
779
+ return;
780
+
781
struct aclk_database_cmd cmd;
782
memset(&cmd, 0, sizeof(cmd));
783
cmd.opcode = ACLK_DATABASE_QUEUE_REMOVED_ALERTS;
database/sqlite/sqlite_aclk_alert.h
+1
@@ -26,5 +26,6 @@ void sql_process_queue_removed_alerts_to_aclk(struct aclk_database_worker_config
26
void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
27
void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, uint64_t snapshot_id, uint64_t sequence_id);
28
int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert_status);
29
+extern int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter);
30
31
#endif //NETDATA_SQLITE_ACLK_ALERT_H
database/sqlite/sqlite_aclk_chart.h
-1
@@ -39,7 +39,6 @@ struct aclk_chart_sync_stats {
39
extern int queue_chart_to_aclk(RRDSET *st);
40
extern int queue_dimension_to_aclk(RRDDIM *rd);
41
extern void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id);
42
-extern int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae);
42
int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
43
int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
44
int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
health/health.c
+1
-1
@@ -804,7 +804,7 @@ void *health_main(void *ptr) {
804
rc->value = NAN;
805
#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
806
if (netdata_cloud_setting && likely(!aclk_alert_reloaded))
807
- sql_queue_removed_alerts_to_aclk(host);
807
+ sql_queue_alarm_to_aclk(host, ae, 1);
808
#endif
809
}
810
}
health/health_log.c
+1
-1
@@ -162,7 +162,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
162
163
#ifdef ENABLE_ACLK
164
if (netdata_cloud_setting) {
165
- sql_queue_alarm_to_aclk(host, ae);
165
+ sql_queue_alarm_to_aclk(host, ae, 0);
166
}
167
#endif
168
}