@cryptotaxi247 / netdata-1 / commits / c0c1e0e85

Better cleanup of health log table (#15045)

Emmanuel Vasilakis committed May 23, 2023 at 15:56 UTC c0c1e0e85a627d0509a37ea4e7ef00c2cf4aa29f

9 files changed +348 -248

database/sqlite/sqlite_aclk_alert.c

+7 -11

@@ -75,7 +75,7 @@ static inline bool is_event_from_alert_variable_config(uint32_t unique_id, char

75 return ret;

76 }

78 -#define MAX_REMOVED_PERIOD 86400

78 +#define MAX_REMOVED_PERIOD 604800 //a week

79 //decide if some events should be sent or not

81 #define SQL_SELECT_ALERT_BY_ID "SELECT hl.new_status, hl.config_hash_id, hl.unique_id FROM health_log_%s hl, aclk_alert_%s aa " \

@@ -321,7 +321,7 @@ void aclk_push_alert_event(struct aclk_sync_host_config *wc)

321 }

322 }

323

324 - char uuid_str[GUID_LEN + 1];

324 + char uuid_str[UUID_STR_LEN];

325 uint64_t first_sequence_id = 0;

326 uint64_t last_sequence_id = 0;

327 static __thread uint64_t log_first_sequence_id = 0;

@@ -463,7 +463,7 @@ void aclk_push_alert_events_for_all_hosts(void)

463

464 void sql_queue_existing_alerts_to_aclk(RRDHOST *host)

465 {

466 - char uuid_str[GUID_LEN + 1];

466 + char uuid_str[UUID_STR_LEN];

467 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

468 BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);

469

@@ -747,7 +747,7 @@ void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id __maybe_unus

747 void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_ENTRY *ae, RRDHOST *host)

748 {

749 char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");

750 - char config_hash_id[GUID_LEN + 1];

750 + char config_hash_id[UUID_STR_LEN];

751 uuid_unparse_lower(ae->config_hash_id, config_hash_id);

752

753 alarm_log->chart = strdupz(ae_chart_name(ae));

@@ -939,18 +939,14 @@ void aclk_push_alert_snapshot_event(char *node_id __maybe_unused)

939 #endif

940 }

941

942 -#define SQL_DELETE_ALERT_ENTRIES "DELETE FROM aclk_alert_%s WHERE filtered_alert_unique_id NOT IN (SELECT unique_id FROM health_log_%s);"

943 -

942 +#define SQL_DELETE_ALERT_ENTRIES "DELETE FROM aclk_alert_%s WHERE filtered_alert_unique_id + %d < UNIXEPOCH();"

943 void sql_aclk_alert_clean_dead_entries(RRDHOST *host)

944 {

946 - if (!claimed())

947 - return;

948 -

945 char uuid_str[UUID_STR_LEN];

946 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

947

952 - char sql[512];

953 - snprintfz(sql,511,SQL_DELETE_ALERT_ENTRIES, uuid_str, uuid_str);

948 + char sql[ACLK_SYNC_QUERY_SIZE];

949 + snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, SQL_DELETE_ALERT_ENTRIES, uuid_str, MAX_REMOVED_PERIOD);

950

951 char *err_msg = NULL;

952 int rc = sqlite3_exec_monitored(db_meta, sql, NULL, NULL, &err_msg);

database/sqlite/sqlite_db_migration.c

+1 -1

@@ -12,7 +12,7 @@ static int return_int_cb(void *data, int argc, char **argv, char **column)

12 }

15 -static int table_exists_in_database(const char *table)

15 +int table_exists_in_database(const char *table)

16 {

17 char *err_msg = NULL;

18 char sql[128];

database/sqlite/sqlite_db_migration.h

@@ -8,5 +8,6 @@

9 int perform_database_migration(sqlite3 *database, int target_version);

10 int perform_context_database_migration(sqlite3 *database, int target_version);

11 +int table_exists_in_database(const char *table);

13 #endif //NETDATA_SQLITE_DB_MIGRATION_H

database/sqlite/sqlite_health.c

+288 -28

@@ -2,6 +2,7 @@

3 #include "sqlite_health.h"

4 #include "sqlite_functions.h"

5 +#include "sqlite_db_migration.h"

7 #define MAX_HEALTH_SQL_SIZE 2048

8 #define sqlite3_bind_string_or_null(res,key,param) ((key) ? sqlite3_bind_text(res, param, string2str(key), -1, SQLITE_STATIC) : sqlite3_bind_null(res, param))

@@ -20,7 +21,7 @@ int sql_create_health_log_table(RRDHOST *host) {

21 return 1;

22 }

23 - char uuid_str[GUID_LEN + 1];

24 + char uuid_str[UUID_STR_LEN];

25 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

27 snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CREATE_HEALTH_LOG_TABLE(uuid_str));

@@ -53,7 +54,7 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {

54 return;

55 }

56 - char uuid_str[GUID_LEN + 1];

57 + char uuid_str[UUID_STR_LEN];

58 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

60 snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_UPDATE_HEALTH_LOG(uuid_str));

@@ -128,7 +129,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {

129 return;

130 }

131

131 - char uuid_str[GUID_LEN + 1];

132 + char uuid_str[UUID_STR_LEN];

133 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

134

135 snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_INSERT_HEALTH_LOG(uuid_str));

@@ -358,34 +359,61 @@ void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae)

359 }

360

361 /* Health related SQL queries

361 - Cleans up the health_log table.

362 + Get a count of rows from health log table

363 */

363 -#define SQL_CLEANUP_HEALTH_LOG(guid,guid2,limit) "DELETE from health_log_%s where unique_id in (SELECT unique_id from health_log_%s order by unique_id asc LIMIT %lu);", guid, guid2, limit

364 -void sql_health_alarm_log_cleanup(RRDHOST *host) {

364 +#define SQL_COUNT_HEALTH_LOG(guid) "SELECT count(1) FROM health_log_%s;", guid

365 +void sql_health_alarm_log_count(RRDHOST *host) {

366 sqlite3_stmt *res = NULL;

366 - static size_t rotate_every = 0;

367 int rc;

368 char command[MAX_HEALTH_SQL_SIZE + 1];

369

370 - if(unlikely(rotate_every == 0)) {

371 - rotate_every = (size_t)config_get_number(CONFIG_SECTION_HEALTH, "rotate log every lines", 2000);

372 - if(rotate_every < 100) rotate_every = 100;

370 + if (unlikely(!db_meta)) {

371 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

372 + error_report("Database has not been initialized");

373 + return;

374 }

375

375 - if(likely(host->health.health_log_entries_written < rotate_every)) {

376 + char uuid_str[UUID_STR_LEN];

377 + uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

378 +

379 + snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_COUNT_HEALTH_LOG(uuid_str));

380 +

381 + rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);

382 + if (unlikely(rc != SQLITE_OK)) {

383 + error_report("Failed to prepare statement to count health log entries from db");

384 return;

385 }

386

387 + rc = sqlite3_step_monitored(res);

388 + if (likely(rc == SQLITE_ROW))

389 + host->health.health_log_entries_written = (size_t) sqlite3_column_int64(res, 0);

390 +

391 + rc = sqlite3_finalize(res);

392 + if (unlikely(rc != SQLITE_OK))

393 + error_report("Failed to finalize the prepared statement to count health log entries from db");

394 +

395 + info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", rrdhost_hostname(host), uuid_str, (unsigned long int) host->health.health_log_entries_written);

396 +}

397 +

398 +/* Health related SQL queries

399 + Cleans up the health_log table on a non-claimed host

400 +*/

401 +#define SQL_CLEANUP_HEALTH_LOG_NOT_CLAIMED(guid,limit) "DELETE FROM health_log_%s ORDER BY unique_id ASC LIMIT %lu;", guid, limit

402 +void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host, size_t rotate_every) {

403 + sqlite3_stmt *res = NULL;

404 + int rc;

405 + char command[MAX_HEALTH_SQL_SIZE + 1];

406 +

407 if (unlikely(!db_meta)) {

408 if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)

409 error_report("Database has not been initialized");

410 return;

411 }

412

385 - char uuid_str[GUID_LEN + 1];

413 + char uuid_str[UUID_STR_LEN];

414 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

415

388 - snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG(uuid_str, uuid_str, (unsigned long int) (host->health.health_log_entries_written - rotate_every)));

416 + snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG_NOT_CLAIMED(uuid_str, (unsigned long int) (host->health.health_log_entries_written - rotate_every)));

417

418 rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);

419 if (unlikely(rc != SQLITE_OK)) {

@@ -403,14 +431,17 @@ void sql_health_alarm_log_cleanup(RRDHOST *host) {

431

432 host->health.health_log_entries_written = rotate_every;

433

406 - sql_aclk_alert_clean_dead_entries(host);

434 + snprintfz(command, MAX_HEALTH_SQL_SIZE, "aclk_alert_%s", uuid_str);

435 + if (unlikely(table_exists_in_database(command))) {

436 + sql_aclk_alert_clean_dead_entries(host);

437 + }

438 }

439

440 /* Health related SQL queries

410 - Get a count of rows from health log table

441 + Cleans up the health_log table on a claimed host

442 */

412 -#define SQL_COUNT_HEALTH_LOG(guid) "SELECT count(1) FROM health_log_%s;", guid

413 -void sql_health_alarm_log_count(RRDHOST *host) {

443 +#define SQL_CLEANUP_HEALTH_LOG_CLAIMED(guid, guid2, guid3, limit) "DELETE from health_log_%s WHERE unique_id NOT IN (SELECT filtered_alert_unique_id FROM aclk_alert_%s) AND unique_id IN (SELECT unique_id FROM health_log_%s ORDER BY unique_id asc LIMIT %lu);", guid, guid2, guid3, limit

444 +void sql_health_alarm_log_cleanup_claimed(RRDHOST *host, size_t rotate_every) {

445 sqlite3_stmt *res = NULL;

446 int rc;

447 char command[MAX_HEALTH_SQL_SIZE + 1];

@@ -421,26 +452,55 @@ void sql_health_alarm_log_count(RRDHOST *host) {

452 return;

453 }

454

424 - char uuid_str[GUID_LEN + 1];

455 + char uuid_str[UUID_STR_LEN];

456 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

457 + snprintfz(command, MAX_HEALTH_SQL_SIZE, "aclk_alert_%s", uuid_str);

458

427 - snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_COUNT_HEALTH_LOG(uuid_str));

459 + if (!table_exists_in_database(command)) {

460 + sql_health_alarm_log_cleanup_not_claimed(host, rotate_every);

461 + return;

462 + }

463 +

464 + snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG_CLAIMED(uuid_str, uuid_str, uuid_str, (unsigned long int) (host->health.health_log_entries_written - rotate_every)));

465

466 rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);

467 if (unlikely(rc != SQLITE_OK)) {

431 - error_report("Failed to prepare statement to count health log entries from db");

468 + error_report("Failed to prepare statement to cleanup health log table");

469 return;

470 }

471

472 rc = sqlite3_step_monitored(res);

436 - if (likely(rc == SQLITE_ROW))

437 - host->health.health_log_entries_written = (size_t) sqlite3_column_int64(res, 0);

473 + if (unlikely(rc != SQLITE_DONE))

474 + error_report("Failed to cleanup health log table, rc = %d", rc);

475

476 rc = sqlite3_finalize(res);

477 if (unlikely(rc != SQLITE_OK))

441 - error_report("Failed to finalize the prepared statement to count health log entries from db");

478 + error_report("Failed to finalize the prepared statement to cleanup health log table");

479

443 - info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", rrdhost_hostname(host), uuid_str, (unsigned long int) host->health.health_log_entries_written);

480 + sql_health_alarm_log_count(host);

481 +

482 + sql_aclk_alert_clean_dead_entries(host);

483 +}

484 +

485 +/* Health related SQL queries

486 + Cleans up the health_log table.

487 +*/

488 +void sql_health_alarm_log_cleanup(RRDHOST *host) {

489 + static size_t rotate_every = 0;

490 +

491 + if(unlikely(rotate_every == 0)) {

492 + rotate_every = (size_t)config_get_number(CONFIG_SECTION_HEALTH, "rotate log every lines", 2000);

493 + if(rotate_every < 100) rotate_every = 100;

494 + }

495 +

496 + if(likely(host->health.health_log_entries_written < rotate_every)) {

497 + return;

498 + }

499 +

500 + if (!claimed()) {

501 + sql_health_alarm_log_cleanup_not_claimed(host, rotate_every);

502 + } else

503 + sql_health_alarm_log_cleanup_claimed(host, rotate_every);

504 }

505

506 #define SQL_INJECT_REMOVED(guid, guid2) "insert into health_log_%s (hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, " \

@@ -608,7 +668,7 @@ void sql_check_removed_alerts_state(char *uuid_str)

668 /* Health related SQL queries

669 Load from the health log table

670 */

611 -#define SQL_LOAD_HEALTH_LOG(guid,limit) "SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type, chart_context FROM (SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type, chart_context FROM health_log_%s order by unique_id desc limit %u) order by unique_id asc;", guid, limit

671 +#define SQL_LOAD_HEALTH_LOG(guid) "SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type, chart_context FROM health_log_%s group by alarm_id having max(alarm_event_id);", guid

672 void sql_health_alarm_log_load(RRDHOST *host) {

673 sqlite3_stmt *res = NULL;

674 int ret;

@@ -623,12 +683,12 @@ void sql_health_alarm_log_load(RRDHOST *host) {

683 return;

684 }

685

626 - char uuid_str[GUID_LEN + 1];

686 + char uuid_str[UUID_STR_LEN];

687 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

688

689 sql_check_removed_alerts_state(uuid_str);

690

631 - snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_LOAD_HEALTH_LOG(uuid_str, host->health_log.max));

691 + snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_LOAD_HEALTH_LOG(uuid_str));

692

693 ret = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);

694 if (unlikely(ret != SQLITE_OK)) {

@@ -1076,7 +1136,7 @@ int alert_hash_and_store_config(

1136 EVP_MD_CTX_destroy(evpctx);

1137 fatal_assert(hash_len > sizeof(uuid_t));

1138

1079 - char uuid_str[GUID_LEN + 1];

1139 + char uuid_str[UUID_STR_LEN];

1140 uuid_unparse_lower(*((uuid_t *)&hash_value), uuid_str);

1141 uuid_copy(hash_id, *((uuid_t *)&hash_value));

1142

@@ -1091,3 +1151,203 @@ int alert_hash_and_store_config(

1151

1152 return 1;

1153 }

1154 +

1155 +#define SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT "SELECT new_status FROM health_log_%s WHERE alarm_id = %u AND unique_id != %u AND flags & %d ORDER BY unique_id DESC LIMIT 1"

1156 +int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_STATUS *last_executed_status)

1157 +{

1158 + int rc = 0, ret = -1;

1159 + char command[MAX_HEALTH_SQL_SIZE + 1];

1160 +

1161 + char uuid_str[UUID_STR_LEN];

1162 + uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

1163 +

1164 + sqlite3_stmt *res = NULL;

1165 +

1166 + snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT, uuid_str, ae->alarm_id, ae->unique_id, HEALTH_ENTRY_FLAG_EXEC_RUN);

1167 +

1168 + rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);

1169 + if (rc != SQLITE_OK) {

1170 + error_report("Failed to prepare statement when trying to get last executed status");

1171 + return ret;

1172 + }

1173 +

1174 + ret = 0;

1175 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

1176 + *last_executed_status = (RRDCALC_STATUS) sqlite3_column_int(res, 0);

1177 + ret = 1;

1178 + }

1179 +

1180 + rc = sqlite3_finalize(res);

1181 + if (unlikely(rc != SQLITE_OK))

1182 + error_report("Failed to finalize the statement.");

1183 +

1184 + return ret;

1185 +}

1186 +

1187 +#define SQL_SELECT_HEALTH_LOG(guid) "SELECT hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type, chart_context FROM health_log_%s WHERE 1=1 ", guid

1188 +void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart) {

1189 +

1190 + buffer_strcat(wb, "[");

1191 +

1192 + unsigned int max = host->health_log.max;

1193 + unsigned int count = 0;

1194 +

1195 + sqlite3_stmt *res = NULL;

1196 + int rc;

1197 +

1198 + BUFFER *command = buffer_create(MAX_HEALTH_SQL_SIZE, NULL);

1199 + char uuid_str[UUID_STR_LEN];

1200 + uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

1201 +

1202 + buffer_sprintf(command, SQL_SELECT_HEALTH_LOG(uuid_str));

1203 +

1204 + if (chart) {

1205 + char chart_sql[MAX_HEALTH_SQL_SIZE + 1];

1206 + snprintfz(chart_sql, MAX_HEALTH_SQL_SIZE, "AND chart = '%s' ", chart);

1207 + buffer_strcat(command, chart_sql);

1208 + }

1209 +

1210 + if (after) {

1211 + char after_sql[MAX_HEALTH_SQL_SIZE + 1];

1212 + snprintfz(after_sql, MAX_HEALTH_SQL_SIZE, "AND unique_id > %u ", after);

1213 + buffer_strcat(command, after_sql);

1214 + }

1215 +

1216 + {

1217 + char limit_sql[MAX_HEALTH_SQL_SIZE + 1];

1218 + snprintfz(limit_sql, MAX_HEALTH_SQL_SIZE, "ORDER BY unique_id DESC LIMIT %u ", max);

1219 + buffer_strcat(command, limit_sql);

1220 + }

1221 +

1222 + rc = sqlite3_prepare_v2(db_meta, buffer_tostring(command), -1, &res, 0);

1223 + if (unlikely(rc != SQLITE_OK)) {

1224 + error_report("Failed to prepare statement SQL_SELECT_HEALTH_LOG");

1225 + return;

1226 + }

1227 +

1228 + while (sqlite3_step(res) == SQLITE_ROW) {

1229 +

1230 + char old_value_string[100 + 1];

1231 + char new_value_string[100 + 1];

1232 +

1233 + char config_hash_id[UUID_STR_LEN];

1234 + uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, 4)), config_hash_id);

1235 +

1236 + char *edit_command = health_edit_command_from_source((char *)sqlite3_column_text(res, 18));

1237 +

1238 + if (count)

1239 + buffer_sprintf(wb, ",");

1240 +

1241 + count++;

1242 +

1243 + buffer_sprintf(

1244 + wb,

1245 + "\n\t{\n"

1246 + "\t\t\"hostname\": \"%s\",\n"

1247 + "\t\t\"utc_offset\": %d,\n"

1248 + "\t\t\"timezone\": \"%s\",\n"

1249 + "\t\t\"unique_id\": %u,\n"

1250 + "\t\t\"alarm_id\": %u,\n"

1251 + "\t\t\"alarm_event_id\": %u,\n"

1252 + "\t\t\"config_hash_id\": \"%s\",\n"

1253 + "\t\t\"name\": \"%s\",\n"

1254 + "\t\t\"chart\": \"%s\",\n"

1255 + "\t\t\"context\": \"%s\",\n"

1256 + "\t\t\"family\": \"%s\",\n"

1257 + "\t\t\"class\": \"%s\",\n"

1258 + "\t\t\"component\": \"%s\",\n"

1259 + "\t\t\"type\": \"%s\",\n"

1260 + "\t\t\"processed\": %s,\n"

1261 + "\t\t\"updated\": %s,\n"

1262 + "\t\t\"exec_run\": %lu,\n"

1263 + "\t\t\"exec_failed\": %s,\n"

1264 + "\t\t\"exec\": \"%s\",\n"

1265 + "\t\t\"recipient\": \"%s\",\n"

1266 + "\t\t\"exec_code\": %d,\n"

1267 + "\t\t\"source\": \"%s\",\n"

1268 + "\t\t\"command\": \"%s\",\n"

1269 + "\t\t\"units\": \"%s\",\n"

1270 + "\t\t\"when\": %lu,\n"

1271 + "\t\t\"duration\": %lu,\n"

1272 + "\t\t\"non_clear_duration\": %lu,\n"

1273 + "\t\t\"status\": \"%s\",\n"

1274 + "\t\t\"old_status\": \"%s\",\n"

1275 + "\t\t\"delay\": %d,\n"

1276 + "\t\t\"delay_up_to_timestamp\": %lu,\n"

1277 + "\t\t\"updated_by_id\": %u,\n"

1278 + "\t\t\"updates_id\": %u,\n"

1279 + "\t\t\"value_string\": \"%s\",\n"

1280 + "\t\t\"old_value_string\": \"%s\",\n"

1281 + "\t\t\"last_repeat\": \"%lu\",\n"

1282 + "\t\t\"silenced\": \"%s\",\n",

1283 + sqlite3_column_text(res, 0),

1284 + host->utc_offset,

1285 + rrdhost_abbrev_timezone(host),

1286 + (unsigned int) sqlite3_column_int64(res, 1),

1287 + (unsigned int) sqlite3_column_int64(res, 2),

1288 + (unsigned int) sqlite3_column_int64(res, 3),

1289 + config_hash_id,

1290 + sqlite3_column_text(res, 13),

1291 + sqlite3_column_text(res, 14),

1292 + sqlite3_column_text(res, 31),

1293 + sqlite3_column_text(res, 15),

1294 + sqlite3_column_text(res, 28) ? (const char *) sqlite3_column_text(res, 28) : (char *) "Unknown",

1295 + sqlite3_column_text(res, 29) ? (const char *) sqlite3_column_text(res, 29) : (char *) "Unknown",

1296 + sqlite3_column_text(res, 30) ? (const char *) sqlite3_column_text(res, 30) : (char *) "Unknown",

1297 + (sqlite3_column_int64(res, 10) & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false",

1298 + (sqlite3_column_int64(res, 10) & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false",

1299 + (long unsigned int)sqlite3_column_int64(res, 11),

1300 + (sqlite3_column_int64(res, 10) & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false",

1301 + sqlite3_column_text(res, 16) ? (const char *) sqlite3_column_text(res, 16) : string2str(host->health.health_default_exec),

1302 + sqlite3_column_text(res, 17) ? (const char *) sqlite3_column_text(res, 17) : string2str(host->health.health_default_recipient),

1303 + sqlite3_column_int(res, 21),

1304 + sqlite3_column_text(res, 18),

1305 + edit_command,

1306 + sqlite3_column_text(res, 19),

1307 + (long unsigned int)sqlite3_column_int64(res, 7),

1308 + (long unsigned int)sqlite3_column_int64(res, 8),

1309 + (long unsigned int)sqlite3_column_int64(res, 9),

1310 + rrdcalc_status2string(sqlite3_column_int(res, 22)),

1311 + rrdcalc_status2string(sqlite3_column_int(res, 23)),

1312 + sqlite3_column_int(res, 24),

1313 + (long unsigned int)sqlite3_column_int64(res, 12),

1314 + (unsigned int)sqlite3_column_int64(res, 5),

1315 + (unsigned int)sqlite3_column_int64(res, 6),

1316 + sqlite3_column_type(res, 25) == SQLITE_NULL ? "-" : format_value_and_unit(new_value_string, 100, sqlite3_column_double(res, 25), (char *) sqlite3_column_text(res, 19), -1),

1317 + sqlite3_column_type(res, 26) == SQLITE_NULL ? "-" : format_value_and_unit(old_value_string, 100, sqlite3_column_double(res, 26), (char *) sqlite3_column_text(res, 19), -1),

1318 + (long unsigned int)sqlite3_column_int64(res, 27),

1319 + (sqlite3_column_int64(res, 10) & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false");

1320 +

1321 + health_string2json(wb, "\t\t", "info", (char *) sqlite3_column_text(res, 20), ",\n");

1322 +

1323 + if(unlikely(sqlite3_column_int64(res, 10) & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {

1324 + buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n");

1325 + }

1326 +

1327 + buffer_strcat(wb, "\t\t\"value\":");

1328 + if (sqlite3_column_type(res, 25) == SQLITE_NULL)

1329 + buffer_strcat(wb, "null");

1330 + else

1331 + buffer_print_netdata_double(wb, sqlite3_column_double(res, 25));

1332 + buffer_strcat(wb, ",\n");

1333 +

1334 + buffer_strcat(wb, "\t\t\"old_value\":");

1335 + if (sqlite3_column_type(res, 26) == SQLITE_NULL)

1336 + buffer_strcat(wb, "null");

1337 + else

1338 + buffer_print_netdata_double(wb, sqlite3_column_double(res, 26));

1339 + buffer_strcat(wb, "\n");

1340 +

1341 + buffer_strcat(wb, "\t}");

1342 +

1343 + freez(edit_command);

1344 + }

1345 +

1346 + buffer_strcat(wb, "\n]");

1347 +

1348 + rc = sqlite3_finalize(res);

1349 + if (unlikely(rc != SQLITE_OK))

1350 + error_report("Failed to finalize statement for SQL_SELECT_HEALTH_LOG");

1351 +

1352 + buffer_free(command);

1353 +}

database/sqlite/sqlite_health.h

@@ -14,4 +14,6 @@ void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);

14 void sql_health_alarm_log_cleanup(RRDHOST *host);

15 int alert_hash_and_store_config(uuid_t hash_id, struct alert_config *cfg, int store_hash);

16 void sql_aclk_alert_clean_dead_entries(RRDHOST *host);

17 +int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_STATUS *last_executed_status);

18 +void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart);

19 #endif //NETDATA_SQLITE_HEALTH_H

health/health.c

+47 -34

@@ -412,17 +412,13 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {

412 // find the previous notification for the same alarm

413 // which we have run the exec script

414 // exception: alarms with HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION set

415 + RRDCALC_STATUS last_executed_status = -3;

416 if(likely(!(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) {

416 - uint32_t id = ae->alarm_id;

417 - ALARM_ENTRY *t;

418 - for(t = ae->next; t ; t = t->next) {

419 - if(t->alarm_id == id && t->flags & HEALTH_ENTRY_FLAG_EXEC_RUN)

420 - break;

421 - }

417 + int ret = sql_health_get_last_executed_event(host, ae, &last_executed_status);

418

423 - if(likely(t)) {

419 + if (likely(ret == 1)) {

420 // we have executed this alarm notification in the past

425 - if(t && t->new_status == ae->new_status) {

421 + if(last_executed_status == ae->new_status) {

422 // don't send the notification for the same status again

423 debug(D_HEALTH, "Health not sending again notification for alarm '%s.%s' status %s", ae_chart_name(ae), ae_name(ae)

424 , rrdcalc_status2string(ae->new_status));

@@ -561,6 +557,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {

557 ae->flags |= HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS;

558 ae->exec_spawn_serial = spawn_enq_cmd(command_to_run);

559 enqueue_alarm_notify_in_progress(ae);

560 + health_alarm_log_save(host, ae);

561 } else {

562 error("Failed to format command arguments");

563 }

@@ -628,35 +625,32 @@ static inline void health_alarm_log_process(RRDHOST *host) {

625 // remember this for the next iteration

626 host->health_last_processed_id = first_waiting;

627

631 - bool cleanup_excess_log_entries = host->health_log.count > host->health_log.max;

632 -

633 - if (!cleanup_excess_log_entries)

634 - return;

635 -

636 - // cleanup excess entries in the log

628 + //delete those that are updated, no in progress execution, and is not repeating

629 netdata_rwlock_wrlock(&host->health_log.alarm_log_rwlock);

630

639 - ALARM_ENTRY *last = NULL;

640 - unsigned int count = host->health_log.max * 2 / 3;

641 - for(ae = host->health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;

642 -

643 - if(ae && last && last->next == ae)

644 - last->next = NULL;

645 - else

646 - ae = NULL;

647 -

648 - while(ae) {

649 - debug(D_HEALTH, "Health removing alarm log entry with id: %u", ae->unique_id);

650 -

651 - ALARM_ENTRY *t = ae->next;

652 -

653 - if(likely(!(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING))) {

654 - health_alarm_wait_for_execution(ae);

631 + ALARM_ENTRY *prev = host->health_log.alarms;

632 + for(ae = host->health_log.alarms; ae ; ae = ae->next) {

633 +

634 + if((likely(!(ae->flags & HEALTH_ENTRY_FLAG_IS_REPEATING)) &&

635 + (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) &&

636 + (ae->flags & HEALTH_ENTRY_FLAG_SAVED) &&

637 + !(ae->flags & HEALTH_ENTRY_FLAG_EXEC_IN_PROGRESS))

638 + ||

639 + ((ae->new_status == RRDCALC_STATUS_REMOVED) &&

640 + (ae->flags & HEALTH_ENTRY_FLAG_SAVED) &&

641 + (ae->when + 3600 < now_realtime_sec())))

642 + {

643 +

644 + if (ae == host->health_log.alarms) {

645 + host->health_log.alarms = ae->next;

646 + prev = ae->next;

647 + } else {

648 + prev->next = ae->next;

649 + }

650 health_alarm_log_free_one_nochecks_nounlink(ae);

656 - host->health_log.count--;

657 - }

658 -

659 - ae = t;

651 + ae = prev;

652 + } else

653 + prev = ae;

654 }

655

656 netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);

@@ -904,8 +898,24 @@ static int update_disabled_silenced(RRDHOST *host, RRDCALC *rc) {

898 return 0;

899 }

900

901 +static void sql_health_postpone_queue_removed(RRDHOST *host __maybe_unused) {

902 +#ifdef ENABLE_ACLK

903 + if (netdata_cloud_setting) {

904 + struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;

905 + if (unlikely(!wc)) {

906 + return;

907 + }

908 +

909 + if (wc->alert_queue_removed >= 1) {

910 + wc->alert_queue_removed+=6;

911 + }

912 + }

913 +#endif

914 +}

915 +

916 static void health_execute_delayed_initializations(RRDHOST *host) {

917 RRDSET *st;

918 + bool must_postpone = false;

919

920 if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION)) return;

921 rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);

@@ -941,8 +951,11 @@ static void health_execute_delayed_initializations(RRDHOST *host) {

951 rrdvar_store_for_chart(host, st);

952 }

953 rrddim_foreach_done(rd);

954 + must_postpone = true;

955 }

956 rrdset_foreach_done(st);

957 + if (must_postpone)

958 + sql_health_postpone_queue_removed(host);

959 }

960

961 /**

health/health.h

+1 -3

@@ -41,7 +41,6 @@ void health_reload(void);

41 void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* context, RRDCALC_STATUS status);

42 void health_alarms2json(RRDHOST *host, BUFFER *wb, int all);

43 void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all);

44 -void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart);

45 void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf);

46 void health_api_v1_chart_custom_variables2json(RRDSET *st, BUFFER *buf);

@@ -87,11 +86,10 @@ void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae);

87 void *health_cmdapi_thread(void *ptr);

90 -void health_label_log_save(RRDHOST *host);

91 -

89 char *health_edit_command_from_source(const char *source);

90 void sql_refresh_hashes(void);

92 void health_add_host_labels(void);

93 +void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix);

95 #endif //NETDATA_HEALTH_H

health/health_json.c

-170

@@ -13,136 +13,6 @@ void health_string2json(BUFFER *wb, const char *prefix, const char *label, const

13 buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);

14 }

16 -void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {

17 - char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");

18 - char config_hash_id[GUID_LEN + 1];

19 - uuid_unparse_lower(ae->config_hash_id, config_hash_id);

20 -

21 - buffer_sprintf(wb,

22 - "\n\t{\n"

23 - "\t\t\"hostname\": \"%s\",\n"

24 - "\t\t\"utc_offset\": %d,\n"

25 - "\t\t\"timezone\": \"%s\",\n"

26 - "\t\t\"unique_id\": %u,\n"

27 - "\t\t\"alarm_id\": %u,\n"

28 - "\t\t\"alarm_event_id\": %u,\n"

29 - "\t\t\"config_hash_id\": \"%s\",\n"

30 - "\t\t\"name\": \"%s\",\n"

31 - "\t\t\"chart\": \"%s\",\n"

32 - "\t\t\"context\": \"%s\",\n"

33 - "\t\t\"family\": \"%s\",\n"

34 - "\t\t\"class\": \"%s\",\n"

35 - "\t\t\"component\": \"%s\",\n"

36 - "\t\t\"type\": \"%s\",\n"

37 - "\t\t\"processed\": %s,\n"

38 - "\t\t\"updated\": %s,\n"

39 - "\t\t\"exec_run\": %lu,\n"

40 - "\t\t\"exec_failed\": %s,\n"

41 - "\t\t\"exec\": \"%s\",\n"

42 - "\t\t\"recipient\": \"%s\",\n"

43 - "\t\t\"exec_code\": %d,\n"

44 - "\t\t\"source\": \"%s\",\n"

45 - "\t\t\"command\": \"%s\",\n"

46 - "\t\t\"units\": \"%s\",\n"

47 - "\t\t\"when\": %lu,\n"

48 - "\t\t\"duration\": %lu,\n"

49 - "\t\t\"non_clear_duration\": %lu,\n"

50 - "\t\t\"status\": \"%s\",\n"

51 - "\t\t\"old_status\": \"%s\",\n"

52 - "\t\t\"delay\": %d,\n"

53 - "\t\t\"delay_up_to_timestamp\": %lu,\n"

54 - "\t\t\"updated_by_id\": %u,\n"

55 - "\t\t\"updates_id\": %u,\n"

56 - "\t\t\"value_string\": \"%s\",\n"

57 - "\t\t\"old_value_string\": \"%s\",\n"

58 - "\t\t\"last_repeat\": \"%lu\",\n"

59 - "\t\t\"silenced\": \"%s\",\n"

60 - , rrdhost_hostname(host)

61 - , host->utc_offset

62 - , rrdhost_abbrev_timezone(host)

63 - , ae->unique_id

64 - , ae->alarm_id

65 - , ae->alarm_event_id

66 - , config_hash_id

67 - , ae_name(ae)

68 - , ae_chart_name(ae)

69 - , ae_chart_context(ae)

70 - , ae_family(ae)

71 - , ae->classification?ae_classification(ae):"Unknown"

72 - , ae->component?ae_component(ae):"Unknown"

73 - , ae->type?ae_type(ae):"Unknown"

74 - , (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false"

75 - , (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false"

76 - , (unsigned long)ae->exec_run_timestamp

77 - , (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false"

78 - , ae->exec?ae_exec(ae):string2str(host->health.health_default_exec)

79 - , ae->recipient?ae_recipient(ae):string2str(host->health.health_default_recipient)

80 - , ae->exec_code

81 - , ae_source(ae)

82 - , edit_command

83 - , ae_units(ae)

84 - , (unsigned long)ae->when

85 - , (unsigned long)ae->duration

86 - , (unsigned long)ae->non_clear_duration

87 - , rrdcalc_status2string(ae->new_status)

88 - , rrdcalc_status2string(ae->old_status)

89 - , ae->delay

90 - , (unsigned long)ae->delay_up_to_timestamp

91 - , ae->updated_by_id

92 - , ae->updates_id

93 - , ae_new_value_string(ae)

94 - , ae_old_value_string(ae)

95 - , (unsigned long)ae->last_repeat

96 - , (ae->flags & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false"

97 - );

98 -

99 - health_string2json(wb, "\t\t", "info", ae->info ? ae_info(ae) : "", ",\n");

100 -

101 - if(unlikely(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {

102 - buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n");

103 - }

104 -

105 - buffer_strcat(wb, "\t\t\"value\":");

106 - buffer_print_netdata_double(wb, ae->new_value);

107 - buffer_strcat(wb, ",\n");

108 -

109 - buffer_strcat(wb, "\t\t\"old_value\":");

110 - buffer_print_netdata_double(wb, ae->old_value);

111 - buffer_strcat(wb, "\n");

112 -

113 - buffer_strcat(wb, "\t}");

114 -

115 - freez(edit_command);

116 -}

117 -

118 -void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart) {

119 -

120 - buffer_strcat(wb, "[");

121 -

122 - unsigned int max = host->health_log.max;

123 - unsigned int count = 0;

124 -

125 - STRING *chart_string = string_strdupz(chart);

126 -

127 - netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);

128 -

129 - ALARM_ENTRY *ae;

130 - for (ae = host->health_log.alarms; ae && count < max; ae = ae->next) {

131 - if ((ae->unique_id > after) && (!chart || chart_string == ae->chart)) {

132 - if (likely(count))

133 - buffer_strcat(wb, ",");

134 - health_alarm_entry2json_nolock(wb, ae, host);

135 - count++;

136 - }

137 - }

138 -

139 - netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);

140 -

141 - string_freez(chart_string);

142 -

143 - buffer_strcat(wb, "\n]\n");

144 -}

145 -

16 static inline void health_rrdcalc_values2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC *rc) {

17 (void)host;

18 buffer_sprintf(wb,

@@ -397,43 +267,3 @@ void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all) {

267 buffer_strcat(wb, "\n\t}\n}\n");

268 }

269

400 -static int have_recent_alarm(RRDHOST *host, uint32_t alarm_id, uint32_t mark)

401 -{

402 - ALARM_ENTRY *ae = host->health_log.alarms;

403 -

404 - while(ae) {

405 - if (ae->alarm_id == alarm_id && ae->unique_id > mark &&

406 - (ae->new_status != RRDCALC_STATUS_WARNING && ae->new_status != RRDCALC_STATUS_CRITICAL))

407 - return 1;

408 - ae = ae->next;

409 - }

410 - return 0;

411 -}

412 -

413 -void health_active_log_alarms_2json(RRDHOST *host, BUFFER *wb) {

414 - netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);

415 -

416 - buffer_sprintf(wb, "[\n");

417 -

418 - unsigned int max = host->health_log.max;

419 - unsigned int count = 0;

420 - ALARM_ENTRY *ae;

421 - for(ae = host->health_log.alarms; ae && count < max ; ae = ae->next) {

422 - if (!ae->updated_by_id &&

423 - ((ae->new_status == RRDCALC_STATUS_WARNING || ae->new_status == RRDCALC_STATUS_CRITICAL) ||

424 - ((ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL) &&

425 - ae->new_status == RRDCALC_STATUS_REMOVED))) {

426 -

427 - if (have_recent_alarm(host, ae->alarm_id, ae->unique_id))

428 - continue;

429 -

430 - if (likely(count))

431 - buffer_strcat(wb, ",");

432 - health_alarm_entry2json_nolock(wb, ae, host);

433 - count++;

434 - }

435 - }

436 - buffer_strcat(wb, "]");

437 -

438 - netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);

439 -}

web/api/web_api_v1.c

+1 -1

@@ -354,7 +354,7 @@ inline int web_client_api_request_v1_alarm_log(RRDHOST *host, struct web_client

354

355 buffer_flush(w->response.data);

356 w->response.data->content_type = CT_APPLICATION_JSON;

357 - health_alarm_log2json(host, w->response.data, after, chart);

357 + sql_health_alarm_log2json(host, w->response.data, after, chart);

358 return HTTP_RESP_OK;

359 }

360