@cryptotaxi247 / netdata-1 / commits / c51dd576b

Reduce unnecessary alert events to the cloud (#13897)

* reduce alert events to the cloud * proper column, set filtered when queing existing * increase max removed period to a day * add constraint, fix queries

Emmanuel Vasilakis committed Nov 4, 2022 at 19:50 UTC c51dd576b07b5cbf6d4d883bca8c85d603fed399

4 files changed +62 -18

database/sqlite/sqlite_aclk.h

+1 -1

@@ -67,7 +67,7 @@ static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out)

67 }

69 #define TABLE_ACLK_ALERT "CREATE TABLE IF NOT EXISTS aclk_alert_%s (sequence_id INTEGER PRIMARY KEY, " \

70 - "alert_unique_id, date_created, date_submitted, date_cloud_ack, " \

70 + "alert_unique_id, date_created, date_submitted, date_cloud_ack, filtered_alert_unique_id NOT NULL, " \

71 "unique(alert_unique_id));"

73 #define INDEX_ACLK_ALERT "CREATE INDEX IF NOT EXISTS aclk_alert_index_%s ON aclk_alert_%s (alert_unique_id);"

database/sqlite/sqlite_aclk_alert.c

+25 -16

@@ -36,7 +36,14 @@ time_t removed_when(uint32_t alarm_id, uint32_t before_unique_id, uint32_t after

36 return when;

37 }

39 -#define MAX_REMOVED_PERIOD 900

39 +void update_filtered(ALARM_ENTRY *ae, uint32_t unique_id, char *uuid_str) {

40 + char sql[ACLK_SYNC_QUERY_SIZE];

41 + snprintfz(sql, ACLK_SYNC_QUERY_SIZE-1, "UPDATE aclk_alert_%s SET filtered_alert_unique_id = %u where filtered_alert_unique_id = %u", uuid_str, ae->unique_id, unique_id);

42 + sqlite3_exec_monitored(db_meta, sql, 0, 0, NULL);

43 + ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;

44 +}

45 +

46 +#define MAX_REMOVED_PERIOD 86400

47 //decide if some events should be sent or not

48 int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)

49 {

@@ -56,12 +63,13 @@ int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)

63 uuid_t config_hash_id;

64 RRDCALC_STATUS status;

65 uint32_t unique_id;

59 -

66 +

67 //get the previous sent event of this alarm_id

68 + //base the search on the last filtered event

69 snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "select hl.new_status, hl.config_hash_id, hl.unique_id from health_log_%s hl, aclk_alert_%s aa \

62 - where hl.unique_id = aa.alert_unique_id \

63 - and hl.alarm_id = %u and hl.unique_id <> %u \

64 - order by alarm_event_id desc LIMIT 1;", uuid_str, uuid_str, ae->alarm_id, ae->unique_id);

70 + where hl.unique_id = aa.filtered_alert_unique_id \

71 + and hl.alarm_id = %u \

72 + order by alarm_event_id desc LIMIT 1;", uuid_str, uuid_str, ae->alarm_id);

74 rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);

75 if (rc != SQLITE_OK) {

@@ -93,8 +101,9 @@ int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)

101 }

102

103 //same status, same config

96 - if (ae->new_status == RRDCALC_STATUS_CLEAR) {

104 + if (ae->new_status == RRDCALC_STATUS_CLEAR || ae->new_status == RRDCALC_STATUS_UNDEFINED) {

105 send = 0;

106 + update_filtered(ae, unique_id, uuid_str);

107 goto done;

108 }

109

@@ -107,6 +116,7 @@ int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)

116 goto done;

117 } else {

118 send = 0;

119 + update_filtered(ae, unique_id, uuid_str);

120 goto done;

121 }

122 }

@@ -130,6 +140,8 @@ int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)

140 return 0;

141 }

142

143 + CHECK_SQLITE_CONNECTION(db_meta);

144 +

145 if (!skip_filter) {

146 if (!should_send_to_cloud(host, ae)) {

147 return 0;

@@ -137,9 +149,6 @@ int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)

149 }

150

151 int rc = 0;

140 -

141 - CHECK_SQLITE_CONNECTION(db_meta);

142 -

152 sqlite3_stmt *res_alert = NULL;

153 char uuid_str[GUID_LEN + 1];

154 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

@@ -148,8 +157,8 @@ int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)

157

158 buffer_sprintf(

159 sql,

151 - "INSERT INTO aclk_alert_%s (alert_unique_id, date_created) "

152 - "VALUES (@alert_unique_id, unixepoch()) on conflict (alert_unique_id) do nothing; ",

160 + "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) "

161 + "VALUES (@alert_unique_id, unixepoch(), @alert_unique_id) on conflict (alert_unique_id) do nothing; ",

162 uuid_str);

163

164 rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res_alert, 0);

@@ -401,8 +410,8 @@ void sql_queue_existing_alerts_to_aclk(RRDHOST *host)

410 BUFFER *sql = buffer_create(1024);

411

412 buffer_sprintf(sql,"delete from aclk_alert_%s; " \

404 - "insert into aclk_alert_%s (alert_unique_id, date_created) " \

405 - "select unique_id alert_unique_id, unixepoch() from health_log_%s " \

413 + "insert into aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \

414 + "select unique_id alert_unique_id, unixepoch(), unique_id alert_unique_id from health_log_%s " \

415 "where new_status <> 0 and new_status <> -2 and config_hash_id is not null and updated_by_id = 0 " \

416 "order by unique_id asc on conflict (alert_unique_id) do nothing;", uuid_str, uuid_str, uuid_str);

417

@@ -728,8 +737,8 @@ void sql_process_queue_removed_alerts_to_aclk(struct aclk_database_worker_config

737

738 BUFFER *sql = buffer_create(1024);

739

731 - buffer_sprintf(sql,"insert into aclk_alert_%s (alert_unique_id, date_created) " \

732 - "select unique_id alert_unique_id, unixepoch() from health_log_%s " \

740 + buffer_sprintf(sql,"insert into aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \

741 + "select unique_id alert_unique_id, unixepoch(), unique_id alert_unique_id from health_log_%s " \

742 "where new_status = -2 and updated_by_id = 0 and unique_id not in " \

743 "(select alert_unique_id from aclk_alert_%s) order by unique_id asc " \

744 "on conflict (alert_unique_id) do nothing;", wc->uuid_str, wc->uuid_str, wc->uuid_str);

@@ -1015,7 +1024,7 @@ void sql_aclk_alert_clean_dead_entries(RRDHOST *host)

1024

1025 BUFFER *sql = buffer_create(1024);

1026

1018 - buffer_sprintf(sql,"delete from aclk_alert_%s where alert_unique_id not in "

1027 + buffer_sprintf(sql,"delete from aclk_alert_%s where filtered_alert_unique_id not in "

1028 " (select unique_id from health_log_%s); ", uuid_str, uuid_str);

1029

1030 char *err_msg = NULL;

database/sqlite/sqlite_db_migration.c

+35

@@ -148,6 +148,40 @@ static int do_migration_v5_v6(sqlite3 *database, const char *name)

148 return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v5_v6[0]);

149 }

150

151 +static int do_migration_v6_v7(sqlite3 *database, const char *name)

152 +{

153 + UNUSED(name);

154 + info("Running \"%s\" database migration", name);

155 +

156 + char sql[256];

157 +

158 + int rc;

159 + sqlite3_stmt *res = NULL;

160 + snprintfz(sql, 255, "SELECT name FROM sqlite_schema WHERE type ='table' AND name LIKE 'aclk_alert_%%';");

161 + rc = sqlite3_prepare_v2(database, sql, -1, &res, 0);

162 + if (rc != SQLITE_OK) {

163 + error_report("Failed to prepare statement to alter aclk_alert tables");

164 + return 1;

165 + }

166 +

167 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

168 + char *table = strdupz((char *) sqlite3_column_text(res, 0));

169 + if (!column_exists_in_table(table, "filtered_alert_unique_id")) {

170 + snprintfz(sql, 255, "ALTER TABLE %s ADD filtered_alert_unique_id", table);

171 + sqlite3_exec_monitored(database, sql, 0, 0, NULL);

172 + snprintfz(sql, 255, "UPDATE %s SET filtered_alert_unique_id = alert_unique_id", table);

173 + sqlite3_exec_monitored(database, sql, 0, 0, NULL);

174 + }

175 + freez(table);

176 + }

177 +

178 + rc = sqlite3_finalize(res);

179 + if (unlikely(rc != SQLITE_OK))

180 + error_report("Failed to finalize statement when altering aclk_alert tables, rc = %d", rc);

181 +

182 + return 0;

183 +}

184 +

185

186 static int do_migration_noop(sqlite3 *database, const char *name)

187 {

@@ -198,6 +232,7 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = {

232 {.name = "v3 to v4", .func = do_migration_v3_v4},

233 {.name = "v4 to v5", .func = do_migration_v4_v5},

234 {.name = "v5 to v6", .func = do_migration_v5_v6},

235 + {.name = "v6 to v7", .func = do_migration_v6_v7},

236 // the terminator of this array

237 {.name = NULL, .func = NULL}

238 };

database/sqlite/sqlite_functions.c

+1 -1

@@ -3,7 +3,7 @@

3 #include "sqlite_functions.h"

4 #include "sqlite_db_migration.h"

6 -#define DB_METADATA_VERSION 6

6 +#define DB_METADATA_VERSION 7

8 const char *database_config[] = {

9 "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "