@cryptotaxi247 / netdata-1 / commits / 3dfe351be

Improve alerts (#18080)

* Hold alert version history Add health version 2 Calculate alert version and store in database Add alert capability Add some checks * Add do_eval_expression * Check point add version and point in time * Fix version calculations * Remove unused field Code cleanup * Update aclk schemas * Add alerts in queue only if claimed Always process pending queue * Handle processing of alert queue for archived hosts * Rework snapshot, update aclk-state * Fix correct order for alert / checkpoint count * Fix compile without cloud * Remove unused parameter * Skip ACLK alert processing if still initializing * Remove extra bind (use UNIXEPOCH()) Cleanup table aliases Add DEFINEs for alert delays Add missing newline * Make sure config exists

Stelios Fragkakis committed Jul 17, 2024 at 22:27 UTC 3dfe351bed0cd741b443ead0ca801a8a928079df

19 files changed +984 -1194

src/aclk/aclk-schemas

+1 -1

@@ -1 +1 @@

1 -Subproject commit 83c661c0dcddb9526814ebbd0668fbc3e281f03f

1 +Subproject commit e07e855389c34a28645fa044e3fc6cb7a6f02a79

src/aclk/aclk.c

+25 -38

@@ -52,9 +52,9 @@ time_t aclk_block_until = 0;

52 #ifdef ENABLE_ACLK

53 mqtt_wss_client mqttwss_client;

55 -netdata_mutex_t aclk_shared_state_mutex = NETDATA_MUTEX_INITIALIZER;

56 -#define ACLK_SHARED_STATE_LOCK netdata_mutex_lock(&aclk_shared_state_mutex)

57 -#define ACLK_SHARED_STATE_UNLOCK netdata_mutex_unlock(&aclk_shared_state_mutex)

55 +//netdata_mutex_t aclk_shared_state_mutex = NETDATA_MUTEX_INITIALIZER;

56 +//#define ACLK_SHARED_STATE_LOCK netdata_mutex_lock(&aclk_shared_state_mutex)

57 +//#define ACLK_SHARED_STATE_UNLOCK netdata_mutex_unlock(&aclk_shared_state_mutex)

59 struct aclk_shared_state aclk_shared_state = {

60 .mqtt_shutdown_msg_id = -1,

@@ -1058,30 +1058,24 @@ void aclk_send_bin_msg(char *msg, size_t msg_len, enum aclk_topics subtopic, con

1058

1059 static void fill_alert_status_for_host(BUFFER *wb, RRDHOST *host)

1060 {

1061 - struct proto_alert_status status;

1062 - memset(&status, 0, sizeof(status));

1063 - if (get_proto_alert_status(host, &status)) {

1064 - buffer_strcat(wb, "\nFailed to get alert streaming status for this host");

1061 + struct aclk_sync_cfg_t *wc = host->aclk_config;

1062 + if (!wc)

1063 return;

1066 - }

1064 +

1065 buffer_sprintf(wb,

1066 "\n\t\tUpdates: %d"

1069 - "\n\t\tPending Min Seq ID: %"PRIu64

1070 - "\n\t\tPending Max Seq ID: %"PRIu64

1071 - "\n\t\tLast Submitted Seq ID: %"PRIu64,

1072 - status.alert_updates,

1073 - status.pending_min_sequence_id,

1074 - status.pending_max_sequence_id,

1075 - status.last_submitted_sequence_id

1067 + "\n\t\tCheckpoints: %d"

1068 + "\n\t\tAlert count: %d"

1069 + "\n\t\tAlert snapshot count: %d",

1070 + wc->stream_alerts,

1071 + wc->checkpoint_count,

1072 + wc->alert_count,

1073 + wc->snapshot_count

1074 );

1075 }

1078 -#endif /* ENABLE_ACLK */

1076

1077 char *aclk_state(void)

1078 {

1082 -#ifndef ENABLE_ACLK

1083 - return strdupz("ACLK Available: No");

1084 -#else

1079 BUFFER *wb = buffer_create(1024, &netdata_buffers_statistics.buffers_aclk);

1080 struct tm *tmptr, tmbuf;

1081 char *ret;

@@ -1163,28 +1157,25 @@ char *aclk_state(void)

1157 ret = strdupz(buffer_tostring(wb));

1158 buffer_free(wb);

1159 return ret;

1166 -#endif /* ENABLE_ACLK */

1160 }

1161

1169 -#ifdef ENABLE_ACLK

1162 static void fill_alert_status_for_host_json(json_object *obj, RRDHOST *host)

1163 {

1172 - struct proto_alert_status status;

1173 - memset(&status, 0, sizeof(status));

1174 - if (get_proto_alert_status(host, &status))

1164 + struct aclk_sync_cfg_t *wc = host->aclk_config;

1165 + if (!wc)

1166 return;

1167

1177 - json_object *tmp = json_object_new_int(status.alert_updates);

1168 + json_object *tmp = json_object_new_int(wc->stream_alerts);

1169 json_object_object_add(obj, "updates", tmp);

1170

1180 - tmp = json_object_new_int(status.pending_min_sequence_id);

1181 - json_object_object_add(obj, "pending-min-seq-id", tmp);

1171 + tmp = json_object_new_int(wc->checkpoint_count);

1172 + json_object_object_add(obj, "checkpoint-count", tmp);

1173

1183 - tmp = json_object_new_int(status.pending_max_sequence_id);

1184 - json_object_object_add(obj, "pending-max-seq-id", tmp);

1174 + tmp = json_object_new_int(wc->alert_count);

1175 + json_object_object_add(obj, "alert-count", tmp);

1176

1186 - tmp = json_object_new_int(status.last_submitted_sequence_id);

1187 - json_object_object_add(obj, "last-submitted-seq-id", tmp);

1177 + tmp = json_object_new_int(wc->snapshot_count);

1178 + json_object_object_add(obj, "alert-snapshot-count", tmp);

1179 }

1180

1181 static json_object *timestamp_to_json(const time_t *t)

@@ -1197,13 +1188,9 @@ static json_object *timestamp_to_json(const time_t *t)

1188 }

1189 return NULL;

1190 }

1200 -#endif /* ENABLE_ACLK */

1191

1192 char *aclk_state_json(void)

1193 {

1204 -#ifndef ENABLE_ACLK

1205 - return strdupz("{\"aclk-available\":false}");

1206 -#else

1194 json_object *tmp, *grp, *msg = json_object_new_object();

1195

1196 tmp = json_object_new_boolean(1);

@@ -1313,8 +1300,8 @@ char *aclk_state_json(void)

1300 char *str = strdupz(json_object_to_json_string_ext(msg, JSON_C_TO_STRING_PLAIN));

1301 json_object_put(msg);

1302 return str;

1316 -#endif /* ENABLE_ACLK */

1303 }

1304 +#endif /* ENABLE_ACLK */

1305

1306 void add_aclk_host_labels(void) {

1307 RRDLABELS *labels = localhost->rrdlabels;

@@ -1347,7 +1334,7 @@ void add_aclk_host_labels(void) {

1334

1335 void aclk_queue_node_info(RRDHOST *host, bool immediate)

1336 {

1350 - struct aclk_sync_cfg_t *wc = host->aclk_config;

1351 - if (likely(wc))

1337 + struct aclk_sync_cfg_t *wc = host->aclk_config;

1338 + if (wc)

1339 wc->node_info_send_time = (host == localhost || immediate) ? 1 : now_realtime_sec();

1340 }

src/aclk/aclk_capas.c

+2 -2

@@ -16,7 +16,7 @@ const struct capability *aclk_get_agent_capas()

16 { .name = "ctx", .version = 1, .enabled = 1 },

17 { .name = "funcs", .version = 1, .enabled = 1 },

18 { .name = "http_api_v2", .version = HTTP_API_V2_VERSION, .enabled = 1 },

19 - { .name = "health", .version = 1, .enabled = 0 }, // index 7, below

19 + { .name = "health", .version = 2, .enabled = 0 }, // index 7, below

20 { .name = "req_cancel", .version = 1, .enabled = 1 },

21 { .name = "dyncfg", .version = 2, .enabled = 1 },

22 { .name = NULL, .version = 0, .enabled = 0 }

@@ -46,7 +46,7 @@ struct capability *aclk_get_node_instance_capas(RRDHOST *host)

46 { .name = "ctx", .version = 1, .enabled = 1 },

47 { .name = "funcs", .version = functions ? 1 : 0, .enabled = functions ? 1 : 0 },

48 { .name = "http_api_v2", .version = HTTP_API_V2_VERSION, .enabled = 1 },

49 - { .name = "health", .version = 1, .enabled = host->health.health_enabled },

49 + { .name = "health", .version = 2, .enabled = host->health.health_enabled },

50 { .name = "req_cancel", .version = 1, .enabled = 1 },

51 { .name = "dyncfg", .version = 2, .enabled = dyncfg },

52 { .name = NULL, .version = 0, .enabled = 0 }

src/aclk/aclk_rx_msgs.c

+3 -3

@@ -347,7 +347,7 @@ int start_alarm_streaming(const char *msg, size_t msg_len)

347 netdata_log_error("Error parsing StartAlarmStreaming");

348 return 1;

349 }

350 - aclk_start_alert_streaming(res.node_id, res.resets);

350 + aclk_start_alert_streaming(res.node_id, res.version);

351 freez(res.node_id);

352 return 0;

353 }

@@ -361,7 +361,7 @@ int send_alarm_checkpoint(const char *msg, size_t msg_len)

361 freez(sac.claim_id);

362 return 1;

363 }

364 - aclk_send_alarm_checkpoint(sac.node_id, sac.claim_id);

364 + aclk_alert_version_check(sac.node_id, sac.claim_id, sac.version);

365 freez(sac.node_id);

366 freez(sac.claim_id);

367 return 0;

@@ -375,7 +375,7 @@ int send_alarm_configuration(const char *msg, size_t msg_len)

375 freez(config_hash);

376 return 1;

377 }

378 - aclk_send_alarm_configuration(config_hash);

378 + aclk_send_alert_configuration(config_hash);

379 freez(config_hash);

380 return 0;

381 }

src/aclk/schema-wrappers/alarm_stream.cc

@@ -22,6 +22,7 @@ struct start_alarm_streaming parse_start_alarm_streaming(const char *data, size_

23 ret.node_id = strdupz(msg.node_id().c_str());

24 ret.resets = msg.resets();

25 + ret.version = msg.version();

27 return ret;

28 }

@@ -37,6 +38,7 @@ struct send_alarm_checkpoint parse_send_alarm_checkpoint(const char *data, size_

39 ret.node_id = strdupz(msg.node_id().c_str());

40 ret.claim_id = strdupz(msg.claim_id().c_str());

41 + ret.version = msg.version();

43 return ret;

44 }

@@ -118,6 +120,7 @@ static void fill_alarm_log_entry(struct alarm_log_entry *data, AlarmLogEntry *pr

120 proto->set_transition_id(data->transition_id);

121 proto->set_chart_name(data->chart_name);

122 proto->set_summary(data->summary);

123 + proto->set_alert_version(data->version);

124 }

125

126 char *generate_alarm_log_entry(size_t *len, struct alarm_log_entry *data)

src/aclk/schema-wrappers/alarm_stream.h

+10 -2

@@ -13,6 +13,7 @@ extern "C" {

14 struct start_alarm_streaming {

15 char *node_id;

16 + uint64_t version;

17 bool resets;

18 };

@@ -36,8 +37,6 @@ struct alarm_log_entry {

37 char *name;

38 char *family;

39 - uint64_t batch_id;

40 - uint64_t sequence_id;

40 uint64_t when;

42 char *config_hash;

@@ -76,13 +75,22 @@ struct alarm_log_entry {

75 char *chart_name;

77 uint64_t event_id;

78 + uint64_t version;

79 char *transition_id;

80 char *summary;

81 +

82 + // local book keeping

83 + int64_t health_log_id;

84 + int64_t alarm_id;

85 + int64_t unique_id;

86 + int64_t sequence_id;

87 };

89 struct send_alarm_checkpoint {

90 char *node_id;

91 char *claim_id;

92 + uint64_t version;

93 + uint64_t when_end;

94 };

96 struct alarm_checkpoint {

src/daemon/commands.c

@@ -306,10 +306,17 @@ static cmd_status_t cmd_ping_execute(char *args, char **message)

306 static cmd_status_t cmd_aclk_state(char *args, char **message)

307 {

308 netdata_log_info("COMMAND: Reopening aclk/cloud state.");

309 +#ifdef ENABLE_ACLK

310 if (strstr(args, "json"))

311 *message = aclk_state_json();

312 else

313 *message = aclk_state();

314 +#else

315 + if (strstr(args, "json"))

316 + *message = strdupz("{\"aclk-available\":false}");

317 + else

318 + *message = strdupz("ACLK Available: No");

319 +#endif

320

321 return CMD_STATUS_SUCCESS;

322 }

src/daemon/unit_test.c

+1 -16

@@ -1674,22 +1674,7 @@ int test_sqlite(void) {

1674 return 1;

1675 }

1676

1677 - BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, NULL);

1678 - char *uuid_str = "0000_000";

1679 -

1680 - buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str);

1681 - rc = sqlite3_exec_monitored(db_mt, buffer_tostring(sql), 0, 0, NULL);

1682 - if (rc != SQLITE_OK)

1683 - goto error;

1684 -

1685 - buffer_free(sql);

1677 fprintf(stderr,"SQLite is OK\n");

1687 - rc = sqlite3_close_v2(db_mt);

1678 + (void) sqlite3_close_v2(db_mt);

1679 return 0;

1689 -error:

1690 - rc = sqlite3_close_v2(db_mt);

1691 - fprintf(stderr,"SQLite statement failed: %s\n", buffer_tostring(sql));

1692 - buffer_free(sql);

1693 - fprintf(stderr,"SQLite tests failed\n");

1694 - return 1;

1680 }

src/database/sqlite/sqlite_aclk.c

+20 -208

@@ -9,7 +9,6 @@ struct aclk_sync_config_s {

9 uv_thread_t thread;

10 uv_loop_t loop;

11 uv_timer_t timer_req;

12 - time_t cleanup_after; // Start a cleanup after this timestamp

12 uv_async_t async;

13 bool initialized;

14 SPINLOCK cmd_queue_lock;

@@ -24,7 +23,7 @@ void sanity_check(void) {

23 #ifdef ENABLE_ACLK

24 static struct aclk_database_cmd aclk_database_deq_cmd(void)

25 {

27 - struct aclk_database_cmd ret;

26 + struct aclk_database_cmd ret = { 0 };

28 spinlock_lock(&aclk_sync_config.cmd_queue_lock);

29 if(aclk_sync_config.cmd_base) {

@@ -35,7 +34,6 @@ static struct aclk_database_cmd aclk_database_deq_cmd(void)

34 }

35 else {

36 ret.opcode = ACLK_DATABASE_NOOP;

38 - ret.completion = NULL;

37 }

38 spinlock_unlock(&aclk_sync_config.cmd_queue_lock);

@@ -176,70 +174,24 @@ static int create_host_callback(void *data, int argc, char **argv, char **column

174

175 #ifdef ENABLE_ACLK

176

179 -#define SQL_SELECT_HOST_BY_UUID "SELECT host_id FROM host WHERE host_id = @host_id"

180 -static int is_host_available(nd_uuid_t *host_id)

181 -{

182 - sqlite3_stmt *res = NULL;

183 - int rc = 0;

184 -

185 - if (!REQUIRE_DB(db_meta))

186 - return 1;

187 -

188 - if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_HOST_BY_UUID, &res))

189 - return 1;

190 -

191 - int param = 0;

192 - SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_id, sizeof(*host_id), SQLITE_STATIC));

193 -

194 - param = 0;

195 - rc = sqlite3_step_monitored(res);

196 -

197 -done:

198 - REPORT_BIND_FAIL(res, param);

199 - SQLITE_FINALIZE(res);

200 - return (rc == SQLITE_ROW);

201 -}

177 +#define SQL_SELECT_ACLK_ALERT_TABLES \

178 + "SELECT 'DROP '||type||' IF EXISTS '||name||';' FROM sqlite_schema WHERE name LIKE 'aclk_alert_%' AND type IN ('table', 'trigger', 'index')"

179

203 -// OPCODE: ACLK_DATABASE_DELETE_HOST

204 -static void sql_delete_aclk_table_list(char *host_guid)

180 +static void sql_delete_aclk_table_list(void)

181 {

206 - char uuid_str[UUID_STR_LEN];

207 - char host_str[UUID_STR_LEN];

208 -

209 - int rc;

210 - nd_uuid_t host_uuid;

211 -

212 - if (unlikely(!host_guid))

213 - return;

214 -

215 - rc = uuid_parse(host_guid, host_uuid);

216 - freez(host_guid);

217 - if (rc)

218 - return;

219 -

220 - uuid_unparse_lower(host_uuid, host_str);

221 - uuid_unparse_lower_fix(&host_uuid, uuid_str);

222 -

223 - if (is_host_available(&host_uuid))

224 - return;

225 -

182 sqlite3_stmt *res = NULL;

227 - BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, &netdata_buffers_statistics.buffers_sqlite);

183

229 - buffer_sprintf(sql,"SELECT 'drop '||type||' IF EXISTS '||name||';' FROM sqlite_schema " \

230 - "WHERE name LIKE 'aclk_%%_%s' AND type IN ('table', 'trigger', 'index')", uuid_str);

184 + BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, NULL);

185

232 - if (!PREPARE_STATEMENT(db_meta, buffer_tostring(sql), &res))

186 + if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_ACLK_ALERT_TABLES, &res))

187 goto fail;

188

235 - buffer_flush(sql);

236 -

189 while (sqlite3_step_monitored(res) == SQLITE_ROW)

190 buffer_strcat(sql, (char *) sqlite3_column_text(res, 0));

191

192 SQLITE_FINALIZE(res);

193

242 - rc = db_execute(db_meta, buffer_tostring(sql));

194 + int rc = db_execute(db_meta, buffer_tostring(sql));

195 if (unlikely(rc))

196 netdata_log_error("Failed to drop unused ACLK tables");

197

@@ -285,53 +237,6 @@ skip:

237 freez(machine_guid);

238 }

239

288 -

289 -static int sql_check_aclk_table(void *data __maybe_unused, int argc __maybe_unused, char **argv __maybe_unused, char **column __maybe_unused)

290 -{

291 - struct aclk_database_cmd cmd;

292 - memset(&cmd, 0, sizeof(cmd));

293 - cmd.opcode = ACLK_DATABASE_DELETE_HOST;

294 - cmd.param[0] = strdupz((char *) argv[0]);

295 - aclk_database_enq_cmd(&cmd);

296 - return 0;

297 -}

298 -

299 -#define SQL_SELECT_ACLK_ACTIVE_LIST "SELECT REPLACE(SUBSTR(name,19),'_','-') FROM sqlite_schema " \

300 - "WHERE name LIKE 'aclk_chart_latest_%' AND type IN ('table')"

301 -

302 -static void sql_check_aclk_table_list(void)

303 -{

304 - char *err_msg = NULL;

305 - int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ACTIVE_LIST, sql_check_aclk_table, NULL, &err_msg);

306 - if (rc != SQLITE_OK) {

307 - error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);

308 - sqlite3_free(err_msg);

309 - }

310 -}

311 -

312 -#define SQL_ALERT_CLEANUP "DELETE FROM aclk_alert_%s WHERE date_submitted IS NOT NULL AND CAST(date_cloud_ack AS INT) < unixepoch()-%d"

313 -

314 -static int sql_maint_aclk_sync_database(void *data __maybe_unused, int argc __maybe_unused, char **argv, char **column __maybe_unused)

315 -{

316 - char sql[ACLK_SYNC_QUERY_SIZE];

317 - snprintfz(sql,sizeof(sql) - 1, SQL_ALERT_CLEANUP, (char *) argv[0], ACLK_DELETE_ACK_ALERTS_INTERNAL);

318 - if (unlikely(db_execute(db_meta, sql)))

319 - error_report("Failed to clean stale ACLK alert entries");

320 - return 0;

321 -}

322 -

323 -#define SQL_SELECT_ACLK_ALERT_LIST "SELECT SUBSTR(name,12) FROM sqlite_schema WHERE name LIKE 'aclk_alert_%' AND type IN ('table')"

324 -

325 -static void sql_maint_aclk_sync_database_all(void)

326 -{

327 - char *err_msg = NULL;

328 - int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ALERT_LIST, sql_maint_aclk_sync_database, NULL, &err_msg);

329 - if (rc != SQLITE_OK) {

330 - error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);

331 - sqlite3_free(err_msg);

332 - }

333 -}

334 -

240 static int aclk_config_parameters(void *data __maybe_unused, int argc __maybe_unused, char **argv, char **column __maybe_unused)

241 {

242 char uuid_str[UUID_STR_LEN];

@@ -339,7 +244,7 @@ static int aclk_config_parameters(void *data __maybe_unused, int argc __maybe_un

244

245 RRDHOST *host = rrdhost_find_by_guid(uuid_str);

246 if (host != localhost)

342 - sql_create_aclk_table(host, (nd_uuid_t *) argv[0], (nd_uuid_t *) argv[1]);

247 + create_aclk_config(host, (nd_uuid_t *)argv[0], (nd_uuid_t *)argv[1]);

248 return 0;

249 }

250

@@ -356,16 +261,7 @@ static void timer_cb(uv_timer_t *handle)

261 uv_stop(handle->loop);

262 uv_update_time(handle->loop);

263

359 - struct aclk_sync_config_s *config = handle->data;

360 - struct aclk_database_cmd cmd;

361 - memset(&cmd, 0, sizeof(cmd));

362 -

363 - if (config->cleanup_after < now_realtime_sec()) {

364 - cmd.opcode = ACLK_DATABASE_CLEANUP;

365 - aclk_database_enq_cmd(&cmd);

366 - config->cleanup_after += ACLK_DATABASE_CLEANUP_INTERVAL;

367 - }

368 -

264 + struct aclk_database_cmd cmd = { 0 };

265 if (aclk_connected) {

266 cmd.opcode = ACLK_DATABASE_PUSH_ALERT;

267 aclk_database_enq_cmd(&cmd);

@@ -373,7 +269,7 @@ static void timer_cb(uv_timer_t *handle)

269 }

270 }

271

376 -static void aclk_synchronization(void *arg __maybe_unused)

272 +static void aclk_synchronization(void *arg)

273 {

274 struct aclk_sync_config_s *config = arg;

275 uv_thread_set_name_np("ACLKSYNC");

@@ -381,14 +277,9 @@ static void aclk_synchronization(void *arg __maybe_unused)

277 service_register(SERVICE_THREAD_TYPE_EVENT_LOOP, NULL, NULL, NULL, true);

278

279 worker_register_job_name(ACLK_DATABASE_NOOP, "noop");

384 - worker_register_job_name(ACLK_DATABASE_CLEANUP, "cleanup");

385 - worker_register_job_name(ACLK_DATABASE_DELETE_HOST, "node delete");

280 worker_register_job_name(ACLK_DATABASE_NODE_STATE, "node state");

281 worker_register_job_name(ACLK_DATABASE_PUSH_ALERT, "alert push");

282 worker_register_job_name(ACLK_DATABASE_PUSH_ALERT_CONFIG, "alert conf push");

389 - worker_register_job_name(ACLK_DATABASE_PUSH_ALERT_CHECKPOINT,"alert checkpoint");

390 - worker_register_job_name(ACLK_DATABASE_PUSH_ALERT_SNAPSHOT, "alert snapshot");

391 - worker_register_job_name(ACLK_DATABASE_QUEUE_REMOVED_ALERTS, "alerts check");

283 worker_register_job_name(ACLK_DATABASE_TIMER, "timer");

284

285 uv_loop_t *loop = &config->loop;

@@ -401,9 +292,10 @@ static void aclk_synchronization(void *arg __maybe_unused)

292

293 netdata_log_info("Starting ACLK synchronization thread");

294

404 - config->cleanup_after = now_realtime_sec() + ACLK_DATABASE_CLEANUP_FIRST;

295 config->initialized = true;

296

297 + sql_delete_aclk_table_list();

298 +

299 while (likely(service_running(SERVICE_ACLKSYNC))) {

300 enum aclk_database_opcode opcode;

301 worker_is_idle();

@@ -422,26 +314,17 @@ static void aclk_synchronization(void *arg __maybe_unused)

314 worker_is_busy(opcode);

315

316 switch (opcode) {

317 + default:

318 case ACLK_DATABASE_NOOP:

319 /* the command queue was empty, do nothing */

320 break;

428 -// MAINTENANCE

429 - case ACLK_DATABASE_CLEANUP:

430 - // Scan all aclk_alert_ tables and cleanup as needed

431 - sql_maint_aclk_sync_database_all();

432 - sql_check_aclk_table_list();

433 - break;

434 -

435 - case ACLK_DATABASE_DELETE_HOST:

436 - sql_delete_aclk_table_list(cmd.param[0]);

437 - break;

321 // NODE STATE

322 case ACLK_DATABASE_NODE_STATE:;

323 RRDHOST *host = cmd.param[0];

324 int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;

325 struct aclk_sync_cfg_t *ahc = host->aclk_config;

326 if (unlikely(!ahc))

444 - sql_create_aclk_table(host, &host->host_uuid, host->node_id);

327 + create_aclk_config(host, &host->host_uuid, host->node_id);

328 aclk_host_state_update(host, live, 1);

329 break;

330 case ACLK_DATABASE_NODE_UNREGISTER:

@@ -455,17 +338,7 @@ static void aclk_synchronization(void *arg __maybe_unused)

338 case ACLK_DATABASE_PUSH_ALERT:

339 aclk_push_alert_events_for_all_hosts();

340 break;

458 - case ACLK_DATABASE_PUSH_ALERT_SNAPSHOT:;

459 - aclk_push_alert_snapshot_event(cmd.param[0]);

460 - break;

461 - case ACLK_DATABASE_QUEUE_REMOVED_ALERTS:

462 - sql_process_queue_removed_alerts_to_aclk(cmd.param[0]);

463 - break;

464 - default:

465 - break;

341 }

467 - if (cmd.completion)

468 - completion_mark_complete(cmd.completion);

342 } while (opcode != ACLK_DATABASE_NOOP);

343 }

344

@@ -489,39 +362,11 @@ static void aclk_synchronization_init(void)

362

363 // -------------------------------------------------------------

364

492 -void sql_create_aclk_table(RRDHOST *host __maybe_unused, nd_uuid_t *host_uuid __maybe_unused, nd_uuid_t *node_id __maybe_unused)

365 +void create_aclk_config(RRDHOST *host __maybe_unused, nd_uuid_t *host_uuid __maybe_unused, nd_uuid_t *node_id __maybe_unused)

366 {

367 #ifdef ENABLE_ACLK

495 - char uuid_str[UUID_STR_LEN];

496 - char host_guid[UUID_STR_LEN];

497 - int rc;

498 -

499 - uuid_unparse_lower_fix(host_uuid, uuid_str);

500 - uuid_unparse_lower(*host_uuid, host_guid);

501 -

502 - char sql[ACLK_SYNC_QUERY_SIZE];

503 -

504 - snprintfz(sql, sizeof(sql) - 1, TABLE_ACLK_ALERT, uuid_str);

505 - rc = db_execute(db_meta, sql);

506 - if (unlikely(rc))

507 - error_report("Failed to create ACLK alert table for host %s", host ? rrdhost_hostname(host) : host_guid);

508 - else {

509 - snprintfz(sql, sizeof(sql) - 1, INDEX_ACLK_ALERT1, uuid_str, uuid_str);

510 - rc = db_execute(db_meta, sql);

511 - if (unlikely(rc))

512 - error_report(

513 - "Failed to create ACLK alert table index 1 for host %s", host ? string2str(host->hostname) : host_guid);

514 -

515 - snprintfz(sql, sizeof(sql) - 1, INDEX_ACLK_ALERT2, uuid_str, uuid_str);

516 - rc = db_execute(db_meta, sql);

517 - if (unlikely(rc))

518 - error_report(

519 - "Failed to create ACLK alert table index 2 for host %s", host ? string2str(host->hostname) : host_guid);

520 - }

521 - if (likely(host) && unlikely(host->aclk_config))

522 - return;

368

524 - if (unlikely(!host))

369 + if (!host || host->aclk_config)

370 return;

371

372 struct aclk_sync_cfg_t *wc = callocz(1, sizeof(struct aclk_sync_cfg_t));

@@ -535,8 +380,7 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, nd_uuid_t *host_uuid __

380 }

381

382 wc->host = host;

538 - strcpy(wc->uuid_str, uuid_str);

539 - wc->alert_updates = 0;

383 + wc->stream_alerts = false;

384 time_t now = now_realtime_sec();

385 wc->node_info_send_time = (host == localhost || NULL == localhost) ? now - 25 : now;

386 #endif

@@ -579,7 +423,7 @@ void sql_aclk_sync_init(void)

423 if (!number_of_children)

424 aclk_queue_node_info(localhost, true);

425

582 - rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_INSTANCES,aclk_config_parameters, NULL,&err_msg);

426 + rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_INSTANCES, aclk_config_parameters, NULL, &err_msg);

427

428 if (rc != SQLITE_OK) {

429 error_report("SQLite error when configuring host ACLK synchonization parameters, rc = %d (%s)", rc, err_msg);

@@ -591,15 +435,12 @@ void sql_aclk_sync_init(void)

435 #endif

436 }

437

594 -// Public

595 -

438 static inline void queue_aclk_sync_cmd(enum aclk_database_opcode opcode, const void *param0, const void *param1)

439 {

440 struct aclk_database_cmd cmd;

441 cmd.opcode = opcode;

442 cmd.param[0] = (void *) param0;

443 cmd.param[1] = (void *) param1;

602 - cmd.completion = NULL;

444 aclk_database_enq_cmd(&cmd);

445 }

446

@@ -612,35 +453,12 @@ void aclk_push_alert_config(const char *node_id, const char *config_hash)

453 queue_aclk_sync_cmd(ACLK_DATABASE_PUSH_ALERT_CONFIG, strdupz(node_id), strdupz(config_hash));

454 }

455

615 -void aclk_push_node_alert_snapshot(const char *node_id)

616 -{

617 - if (unlikely(!aclk_sync_config.initialized))

618 - return;

619 -

620 - queue_aclk_sync_cmd(ACLK_DATABASE_PUSH_ALERT_SNAPSHOT, strdupz(node_id), NULL);

621 -}

622 -

623 -

624 -void aclk_push_node_removed_alerts(const char *node_id)

625 -{

626 - if (unlikely(!aclk_sync_config.initialized))

627 - return;

628 -

629 - queue_aclk_sync_cmd(ACLK_DATABASE_QUEUE_REMOVED_ALERTS, strdupz(node_id), NULL);

630 -}

631 -

456 void schedule_node_info_update(RRDHOST *host __maybe_unused)

457 {

458 #ifdef ENABLE_ACLK

459 if (unlikely(!host))

460 return;

637 -

638 - struct aclk_database_cmd cmd;

639 - memset(&cmd, 0, sizeof(cmd));

640 - cmd.opcode = ACLK_DATABASE_NODE_STATE;

641 - cmd.param[0] = host;

642 - cmd.completion = NULL;

643 - aclk_database_enq_cmd(&cmd);

461 + queue_aclk_sync_cmd(ACLK_DATABASE_NODE_STATE, host, NULL);

462 #endif

463 }

464

@@ -649,12 +467,6 @@ void unregister_node(const char *machine_guid)

467 {

468 if (unlikely(!machine_guid))

469 return;

652 -

653 - struct aclk_database_cmd cmd;

654 - memset(&cmd, 0, sizeof(cmd));

655 - cmd.opcode = ACLK_DATABASE_NODE_UNREGISTER;

656 - cmd.param[0] = strdupz(machine_guid);

657 - cmd.completion = NULL;

658 - aclk_database_enq_cmd(&cmd);

470 + queue_aclk_sync_cmd(ACLK_DATABASE_NODE_UNREGISTER, strdupz(machine_guid), NULL);

471 }

472 #endif

src/database/sqlite/sqlite_aclk.h

+7 -37

@@ -3,21 +3,9 @@

3 #ifndef NETDATA_SQLITE_ACLK_H

4 #define NETDATA_SQLITE_ACLK_H

6 -#define ACLK_MAX_ALERT_UPDATES "5"

7 -#define ACLK_DATABASE_CLEANUP_FIRST (1200)

8 -#define ACLK_DATABASE_CLEANUP_INTERVAL (3600)

9 -#define ACLK_DELETE_ACK_ALERTS_INTERNAL (86400)

6 +#define ACLK_MAX_ALERT_UPDATES "50"

7 #define ACLK_SYNC_QUERY_SIZE 512

12 -static inline void uuid_unparse_lower_fix(nd_uuid_t *uuid, char *out)

13 -{

14 - uuid_unparse_lower(*uuid, out);

15 - out[8] = '_';

16 - out[13] = '_';

17 - out[18] = '_';

18 - out[23] = '_';

19 -}

20 -

9 static inline int uuid_parse_fix(char *in, nd_uuid_t uuid)

10 {

11 in[8] = '-';

@@ -32,25 +20,11 @@ static inline int claimed()

20 return localhost->aclk_state.claimed_id != NULL;

21 }

35 -#define TABLE_ACLK_ALERT \

36 - "CREATE TABLE IF NOT EXISTS aclk_alert_%s (sequence_id INTEGER PRIMARY KEY, " \

37 - "alert_unique_id, date_created, date_submitted, date_cloud_ack, filtered_alert_unique_id NOT NULL, " \

38 - "UNIQUE(alert_unique_id))"

39 -

40 -#define INDEX_ACLK_ALERT1 "CREATE INDEX IF NOT EXISTS aclk_alert_index1_%s ON aclk_alert_%s (filtered_alert_unique_id)"

41 -#define INDEX_ACLK_ALERT2 "CREATE INDEX IF NOT EXISTS aclk_alert_index2_%s ON aclk_alert_%s (date_submitted)"

42 -

23 enum aclk_database_opcode {

24 ACLK_DATABASE_NOOP = 0,

45 -

46 - ACLK_DATABASE_CLEANUP,

47 - ACLK_DATABASE_DELETE_HOST,

25 ACLK_DATABASE_NODE_STATE,

26 ACLK_DATABASE_PUSH_ALERT,

27 ACLK_DATABASE_PUSH_ALERT_CONFIG,

51 - ACLK_DATABASE_PUSH_ALERT_SNAPSHOT,

52 - ACLK_DATABASE_PUSH_ALERT_CHECKPOINT,

53 - ACLK_DATABASE_QUEUE_REMOVED_ALERTS,

28 ACLK_DATABASE_NODE_UNREGISTER,

29 ACLK_DATABASE_TIMER,

@@ -62,29 +36,25 @@ enum aclk_database_opcode {

36 struct aclk_database_cmd {

37 enum aclk_database_opcode opcode;

38 void *param[2];

65 - struct completion *completion;

39 struct aclk_database_cmd *prev, *next;

40 };

42 typedef struct aclk_sync_cfg_t {

43 RRDHOST *host;

71 - int alert_updates;

72 - int alert_checkpoint_req;

73 - int alert_queue_removed;

44 + int8_t send_snapshot;

45 + bool stream_alerts;

46 + int alert_count;

47 + int snapshot_count;

48 + int checkpoint_count;

49 time_t node_info_send_time;

50 time_t node_collectors_send;

76 - char uuid_str[UUID_STR_LEN];

51 char node_id[UUID_STR_LEN];

52 char *alerts_snapshot_uuid; // will contain the snapshot_uuid value if snapshot was requested

79 - uint64_t alerts_log_first_sequence_id;

80 - uint64_t alerts_log_last_sequence_id;

53 } aclk_sync_cfg_t;

83 -void sql_create_aclk_table(RRDHOST *host, nd_uuid_t *host_uuid, nd_uuid_t *node_id);

55 +void create_aclk_config(RRDHOST *host, nd_uuid_t *host_uuid, nd_uuid_t *node_id);

56 void sql_aclk_sync_init(void);

57 void aclk_push_alert_config(const char *node_id, const char *config_hash);

86 -void aclk_push_node_alert_snapshot(const char *node_id);

87 -void aclk_push_node_removed_alerts(const char *node_id);

58 void schedule_node_info_update(RRDHOST *host);

59 #ifdef ENABLE_ACLK

60 void unregister_node(const char *machine_guid);

src/database/sqlite/sqlite_aclk_alert.c

+626 -640

@@ -5,7 +5,6 @@

6 #ifdef ENABLE_ACLK

7 #include "../../aclk/aclk_alarm_api.h"

8 -#endif

9 #define SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param) \

10 ({ \

@@ -13,33 +12,6 @@

12 sqlite3_column_bytes((res), (_param)) ? strdupz((char *)sqlite3_column_text((res), (_param))) : NULL; \

13 })

16 -#define SQL_UPDATE_FILTERED_ALERT \

17 - "UPDATE aclk_alert_%s SET filtered_alert_unique_id = @new_alert, date_created = UNIXEPOCH() " \

18 - "WHERE filtered_alert_unique_id = @old_alert"

19 -

20 -static void update_filtered(ALARM_ENTRY *ae, int64_t unique_id, char *uuid_str)

21 -{

22 - sqlite3_stmt *res = NULL;

23 -

24 - char sql[ACLK_SYNC_QUERY_SIZE];

25 - snprintfz(sql, sizeof(sql) - 1, SQL_UPDATE_FILTERED_ALERT, uuid_str);

26 -

27 - if (!PREPARE_STATEMENT(db_meta, sql, &res))

28 - return;

29 -

30 - int param = 0;

31 - SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, ae->unique_id));

32 - SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, unique_id));

33 -

34 - param = 0;

35 - if (likely(sqlite3_step_monitored(res) == SQLITE_DONE))

36 - ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;

37 -

38 -done:

39 - REPORT_BIND_FAIL(res, param);

40 - SQLITE_FINALIZE(res);

41 -}

42 -

15 #define SQL_SELECT_VARIABLE_ALERT_BY_UNIQUE_ID \

16 "SELECT hld.unique_id FROM health_log hl, alert_hash ah, health_log_detail hld " \

17 "WHERE hld.unique_id = @unique_id AND hl.config_hash_id = ah.hash_id AND hld.health_log_id = hl.health_log_id " \

@@ -47,9 +19,9 @@ done:

20 static inline bool is_event_from_alert_variable_config(int64_t unique_id, nd_uuid_t *host_id)

21 {

50 - sqlite3_stmt *res = NULL;

22 + static __thread sqlite3_stmt *res = NULL;

52 - if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_VARIABLE_ALERT_BY_UNIQUE_ID, &res))

24 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_SELECT_VARIABLE_ALERT_BY_UNIQUE_ID, &res))

25 return false;

27 bool ret = false;

@@ -63,115 +35,141 @@ static inline bool is_event_from_alert_variable_config(int64_t unique_id, nd_uui

36 done:

37 REPORT_BIND_FAIL(res, param);

66 - SQLITE_FINALIZE(res);

38 + SQLITE_RESET(res);

39 return ret;

40 }

42 #define MAX_REMOVED_PERIOD 604800 //a week

72 -//decide if some events should be sent or not

73 -#define SQL_SELECT_ALERT_BY_ID \

74 - "SELECT hld.new_status, hl.config_hash_id, hld.unique_id FROM health_log hl, aclk_alert_%s aa, health_log_detail hld " \

75 - "WHERE hl.host_id = @host_id AND hld.unique_id = aa.filtered_alert_unique_id " \

76 - "AND hld.alarm_id = @alarm_id AND hl.health_log_id = hld.health_log_id " \

77 - "ORDER BY hld.rowid DESC LIMIT 1"

44 +#define SQL_UPDATE_ALERT_VERSION_TRANSITION \

45 + "UPDATE alert_version SET unique_id = @unique_id WHERE health_log_id = @health_log_id"

79 -static bool should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)

47 +static void update_alert_version_transition(int64_t health_log_id, int64_t unique_id)

48 {

81 - sqlite3_stmt *res = NULL;

49 + static __thread sqlite3_stmt *res = NULL;

83 - if (ae->new_status == RRDCALC_STATUS_UNINITIALIZED ||

84 - (ae->new_status == RRDCALC_STATUS_REMOVED &&

85 - !(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)))

86 - return 0;

51 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_UPDATE_ALERT_VERSION_TRANSITION, &res))

52 + return;

88 - if (unlikely(uuid_is_null(ae->config_hash_id) || !host->aclk_config))

89 - return 0;

54 + int param = 0;

55 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, unique_id));

56 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, health_log_id));

57 +

58 + param = 0;

59 + int rc = sqlite3_step_monitored(res);

60 + if (rc != SQLITE_DONE)

61 + error_report("Failed to update alert_version to latest transition");

62 +

63 +done:

64 + REPORT_BIND_FAIL(res, param);

65 + SQLITE_RESET(res);

66 +}

91 - char sql[ACLK_SYNC_QUERY_SIZE];

68 +//decide if some events should be sent or not

69 +

70 +#define SQL_SELECT_LAST_ALERT_STATUS "SELECT status FROM alert_version WHERE health_log_id = @health_log_id "

93 - //get the previous sent event of this alarm_id

94 - //base the search on the last filtered event

95 - snprintfz(sql, sizeof(sql) - 1, SQL_SELECT_ALERT_BY_ID, host->aclk_config->uuid_str);

72 +static bool cloud_status_matches(int64_t health_log_id, RRDCALC_STATUS status)

73 +{

74 + static __thread sqlite3_stmt *res = NULL;

97 - if (!PREPARE_STATEMENT(db_meta, sql, &res))

76 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_SELECT_LAST_ALERT_STATUS, &res))

77 return true;

79 bool send = false;

81 int param = 0;

103 - SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

104 - SQLITE_BIND_FAIL(done, sqlite3_bind_int(res, ++param, (int) ae->alarm_id));

82 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(res, ++param, health_log_id));

84 param = 0;

85 int rc = sqlite3_step_monitored(res);

86 if (likely(rc == SQLITE_ROW)) {

109 - nd_uuid_t config_hash_id;

110 - RRDCALC_STATUS status = (RRDCALC_STATUS)sqlite3_column_int(res, 0);

111 -

112 - if (sqlite3_column_type(res, 1) != SQLITE_NULL)

113 - uuid_copy(config_hash_id, *((nd_uuid_t *)sqlite3_column_blob(res, 1)));

114 -

115 - int64_t unique_id = sqlite3_column_int64(res, 2);

116 -

117 - if (ae->new_status != (RRDCALC_STATUS)status || !uuid_eq(ae->config_hash_id, config_hash_id))

118 - send = true;

119 - else

120 - update_filtered(ae, unique_id, host->aclk_config->uuid_str);

121 - } else

122 - send = true;

87 + RRDCALC_STATUS current_status = (RRDCALC_STATUS)sqlite3_column_int(res, 0);

88 + send = (current_status == status);

89 + }

91 done:

92 REPORT_BIND_FAIL(res, param);

126 - SQLITE_FINALIZE(res);

93 + SQLITE_RESET(res);

94 return send;

95 }

97 #define SQL_QUEUE_ALERT_TO_CLOUD \

131 - "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \

132 - "VALUES (@alert_unique_id, UNIXEPOCH(), @alert_unique_id) ON CONFLICT (alert_unique_id) DO NOTHING"

133 -

134 -void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, bool skip_filter)

98 + "INSERT INTO aclk_queue (host_id, health_log_id, unique_id, date_created)" \

99 + " VALUES (@host_id, @health_log_id, @unique_id, UNIXEPOCH())" \

100 + " ON CONFLICT(host_id, health_log_id) DO UPDATE SET unique_id=excluded.unique_id, " \

101 + " date_created=excluded.date_created"

102 +

103 +//

104 +// Attempt to insert an alert to the submit queue to reach the cloud

105 +//

106 +// The alert will NOT be added in the submit queue if

107 +// - Cloud is already aware of the alert status

108 +// - The transition refers to a variable

109 +//

110 +static int insert_alert_to_submit_queue(RRDHOST *host, int64_t health_log_id, uint32_t unique_id, RRDCALC_STATUS status)

111 {

136 - sqlite3_stmt *res = NULL;

137 - char sql[ACLK_SYNC_QUERY_SIZE];

112 + static __thread sqlite3_stmt *res = NULL;

113

139 - if (!service_running(SERVICE_ACLK))

140 - return;

114 + if (cloud_status_matches(health_log_id, status)) {

115 + update_alert_version_transition(health_log_id, unique_id);

116 + return 1;

117 + }

118

142 - if (!claimed() || ae->flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED)

143 - return;

119 + if (is_event_from_alert_variable_config(unique_id, &host->host_uuid))

120 + return 2;

121

145 - if (false == skip_filter && !should_send_to_cloud(host, ae))

146 - return;

122 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_QUEUE_ALERT_TO_CLOUD, &res))

123 + return -1;

124

148 - if (is_event_from_alert_variable_config(ae->unique_id, &host->host_uuid))

149 - return;

125 + int param = 0;

126 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

127 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, health_log_id));

128 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (int64_t) unique_id));

129

151 - snprintfz(sql, sizeof(sql) - 1, SQL_QUEUE_ALERT_TO_CLOUD, host->aclk_config->uuid_str);

130 + param = 0;

131 + int rc = execute_insert(res);

132 + if (unlikely(rc != SQLITE_DONE))

133 + error_report("Failed to insert alert in the submit queue %"PRIu32", rc = %d", unique_id, rc);

134

153 - if (!PREPARE_STATEMENT(db_meta, sql, &res))

154 - return;

135 +done:

136 + REPORT_BIND_FAIL(res, param);

137 + SQLITE_RESET(res);

138 + return 0;

139 +}

140 +

141 +#define SQL_DELETE_QUEUE_ALERT_TO_CLOUD \

142 + "DELETE FROM aclk_queue WHERE host_id = @host_id AND sequence_id BETWEEN @seq1 AND @seq2"

143 +

144 +//

145 +// Delete a range of alerts from the submit queue (after being sent to the the cloud)

146 +//

147 +static int delete_alert_from_submit_queue(RRDHOST *host, int64_t first_seq_id, int64_t last_seq_id)

148 +{

149 + static __thread sqlite3_stmt *res = NULL;

150 +

151 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_DELETE_QUEUE_ALERT_TO_CLOUD, &res))

152 + return -1;

153

154 int param = 0;

157 - SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, ae->unique_id));

155 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

156 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, first_seq_id));

157 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, last_seq_id));

158

159 param = 0;

160 - int rc = execute_insert(res);

161 - if (unlikely(rc == SQLITE_DONE)) {

162 - ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;

163 - rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

164 - } else

165 - error_report("Failed to store alert event %"PRIu32", rc = %d", ae->unique_id, rc);

160 + int rc = sqlite3_step_monitored(res);

161 + if (rc != SQLITE_DONE)

162 + error_report("Failed to delete submitted to ACLK");

163

164 done:

165 REPORT_BIND_FAIL(res, param);

169 - SQLITE_FINALIZE(res);

166 + SQLITE_RESET(res);

167 + return 0;

168 }

169

170 int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)

171 {

174 -#ifdef ENABLE_ACLK

172 +

173 switch(status) {

174 case RRDCALC_STATUS_REMOVED:

175 return ALARM_STATUS_REMOVED;

@@ -191,10 +189,6 @@ int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)

189 default:

190 return ALARM_STATUS_UNKNOWN;

191 }

194 -#else

195 - UNUSED(status);

196 - return 1;

197 -#endif

192 }

193

194 static inline char *sqlite3_uuid_unparse_strdupz(sqlite3_stmt *res, int iCol) {

@@ -219,245 +213,437 @@ static inline char *sqlite3_text_strdupz_empty(sqlite3_stmt *res, int iCol) {

213 return strdupz(ret);

214 }

215

216 +#define SQL_UPDATE_ALERT_VERSION \

217 + "INSERT INTO alert_version (health_log_id, unique_id, status, version, date_submitted)" \

218 + " VALUES (@health_log_id, @unique_id, @status, @version, UNIXEPOCH())" \

219 + " ON CONFLICT(health_log_id) DO UPDATE SET status = excluded.status, version = excluded.version, " \

220 + " unique_id=excluded.unique_id, date_submitted=excluded.date_submitted"

221 +

222 +//

223 +// Store a new alert transition along with the version after sending to the cloud

224 +// - Update an existing alert with the updated version, status, transition and date submitted

225 +//

226 +static void sql_update_alert_version(int64_t health_log_id, int64_t unique_id, RRDCALC_STATUS status, uint64_t version)

227 +{

228 + static __thread sqlite3_stmt *res = NULL;

229 +

230 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_UPDATE_ALERT_VERSION, &res))

231 + return;

232

223 -static void aclk_push_alert_event(struct aclk_sync_cfg_t *wc __maybe_unused)

233 + int param = 0;

234 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, health_log_id));

235 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, unique_id));

236 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(res, ++param, status));

237 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, version));

238 +

239 + param = 0;

240 + int rc = sqlite3_step_monitored(res);

241 + if (rc != SQLITE_DONE)

242 + error_report("Failed to execute sql_update_alert_version");

243 +

244 +done:

245 + REPORT_BIND_FAIL(res, param);

246 + SQLITE_RESET(res);

247 +}

248 +

249 +#define SQL_SELECT_ALERT_TO_DUMMY \

250 + "SELECT aq.sequence_id, hld.unique_id, hld.when_key, hld.new_status, hld.health_log_id" \

251 + " FROM health_log hl, aclk_queue aq, alert_hash ah, health_log_detail hld" \

252 + " WHERE hld.unique_id = aq.unique_id AND hl.config_hash_id = ah.hash_id" \

253 + " AND hl.host_id = @host_id AND aq.host_id = hl.host_id AND hl.health_log_id = hld.health_log_id" \

254 + " ORDER BY aq.sequence_id ASC"

255 +

256 +//

257 +// Check all queued alerts for a host and commit them as if they have been send to the cloud

258 +// this will produce new versions as needed. We need this because we are about to send a

259 +// a snapshot so we can include the latest transition.

260 +//

261 +static void commit_alert_events(RRDHOST *host)

262 {

225 -#ifdef ENABLE_ACLK

226 - int rc;

263 + sqlite3_stmt *res = NULL;

264

228 - if (unlikely(!wc->alert_updates)) {

229 - nd_log(NDLS_ACCESS, NDLP_NOTICE,

230 - "ACLK STA [%s (%s)]: Ignoring alert push event, updates have been turned off for this node.",

231 - wc->node_id,

232 - wc->host ? rrdhost_hostname(wc->host) : "N/A");

265 + if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_ALERT_TO_DUMMY, &res))

266 return;

267 +

268 + int param = 0;

269 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

270 +

271 + int64_t first_sequence_id = 0;

272 + int64_t last_sequence_id = 0;

273 +

274 + param = 0;

275 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

276 +

277 + last_sequence_id = sqlite3_column_int64(res, 0);

278 + if (first_sequence_id == 0)

279 + first_sequence_id = last_sequence_id;

280 +

281 + int64_t unique_id = sqlite3_column_int(res, 1);

282 + int64_t version = sqlite3_column_int64(res, 2);

283 + RRDCALC_STATUS status = (RRDCALC_STATUS)sqlite3_column_int(res, 3);

284 + int64_t health_log_id = sqlite3_column_int64(res, 4);

285 +

286 + sql_update_alert_version(health_log_id, unique_id, status, version);

287 }

288

289 + if (first_sequence_id)

290 + delete_alert_from_submit_queue(host, first_sequence_id, last_sequence_id);

291 +

292 +done:

293 + REPORT_BIND_FAIL(res, param);

294 + SQLITE_FINALIZE(res);

295 +}

296 +

297 +typedef enum {

298 + SEQUENCE_ID,

299 + UNIQUE_ID,

300 + ALARM_ID,

301 + CONFIG_HASH_ID,

302 + UPDATED_BY_ID,

303 + WHEN_KEY,

304 + DURATION,

305 + NON_CLEAR_DURATION,

306 + FLAGS,

307 + EXEC_RUN_TIMESTAMP,

308 + DELAY_UP_TO_TIMESTAMP,

309 + NAME,

310 + CHART,

311 + EXEC,

312 + RECIPIENT,

313 + SOURCE,

314 + UNITS,

315 + INFO,

316 + EXEC_CODE,

317 + NEW_STATUS,

318 + OLD_STATUS,

319 + DELAY,

320 + NEW_VALUE,

321 + OLD_VALUE,

322 + LAST_REPEAT,

323 + CHART_CONTEXT,

324 + TRANSITION_ID,

325 + ALARM_EVENT_ID,

326 + CHART_NAME,

327 + SUMMARY,

328 + HEALTH_LOG_ID,

329 + VERSION

330 +} HealthLogDetails;

331 +

332 +void health_alarm_log_populate(

333 + struct alarm_log_entry *alarm_log,

334 + sqlite3_stmt *res,

335 + RRDHOST *host,

336 + RRDCALC_STATUS *status)

337 +{

338 + char old_value_string[100 + 1];

339 + char new_value_string[100 + 1];

340 +

341 + RRDCALC_STATUS current_status = (RRDCALC_STATUS)sqlite3_column_int(res, NEW_STATUS);

342 + if (status)

343 + *status = current_status;

344 +

345 + char *source = (char *) sqlite3_column_text(res, SOURCE);

346 + alarm_log->command = source ? health_edit_command_from_source(source) : strdupz("UNKNOWN=0=UNKNOWN");

347 +

348 + alarm_log->chart = strdupz((char *) sqlite3_column_text(res, CHART));

349 + alarm_log->name = strdupz((char *) sqlite3_column_text(res, NAME));

350 +

351 + alarm_log->when = sqlite3_column_int64(res, WHEN_KEY);

352 +

353 + alarm_log->config_hash = sqlite3_uuid_unparse_strdupz(res, CONFIG_HASH_ID);

354 +

355 + alarm_log->utc_offset = host->utc_offset;

356 + alarm_log->timezone = strdupz(rrdhost_abbrev_timezone(host));

357 + alarm_log->exec_path = sqlite3_column_bytes(res, EXEC) ?

358 + strdupz((char *)sqlite3_column_text(res, EXEC)) :

359 + strdupz((char *)string2str(host->health.health_default_exec));

360 +

361 + alarm_log->conf_source = source ? strdupz(source) : strdupz("");

362 +

363 + time_t duration = sqlite3_column_int64(res, DURATION);

364 + alarm_log->duration = (duration > 0) ? duration : 0;

365 +

366 + alarm_log->non_clear_duration = sqlite3_column_int64(res, NON_CLEAR_DURATION);

367 +

368 + alarm_log->status = rrdcalc_status_to_proto_enum(current_status);

369 + alarm_log->old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS)sqlite3_column_int64(res, OLD_STATUS));

370 + alarm_log->delay = sqlite3_column_int64(res, DELAY);

371 + alarm_log->delay_up_to_timestamp = sqlite3_column_int64(res, DELAY_UP_TO_TIMESTAMP);

372 + alarm_log->last_repeat = sqlite3_column_int64(res, LAST_REPEAT);

373 +

374 + uint64_t flags = sqlite3_column_int64(res, FLAGS);

375 + char *recipient = (char *) sqlite3_column_text(res, RECIPIENT);

376 + alarm_log->silenced =

377 + ((flags & HEALTH_ENTRY_FLAG_SILENCED) || (recipient && !strncmp(recipient, "silent", 6))) ? 1 : 0;

378 +

379 + double value = sqlite3_column_double(res, NEW_VALUE);

380 + double old_value = sqlite3_column_double(res, OLD_VALUE);

381 +

382 + alarm_log->value_string =

383 + sqlite3_column_type(res, NEW_VALUE) == SQLITE_NULL ?

384 + strdupz((char *)"-") :

385 + strdupz((char *)format_value_and_unit(

386 + new_value_string, 100, value, (char *)sqlite3_column_text(res, UNITS), -1));

387 +

388 + alarm_log->old_value_string =

389 + sqlite3_column_type(res, OLD_VALUE) == SQLITE_NULL ?

390 + strdupz((char *)"-") :

391 + strdupz((char *)format_value_and_unit(

392 + old_value_string, 100, old_value, (char *)sqlite3_column_text(res, UNITS), -1));

393 +

394 + alarm_log->value = (!isnan(value)) ? (NETDATA_DOUBLE)value : 0;

395 + alarm_log->old_value = (!isnan(old_value)) ? (NETDATA_DOUBLE)old_value : 0;

396 +

397 + alarm_log->updated = (flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;

398 + alarm_log->rendered_info = sqlite3_text_strdupz_empty(res, INFO);

399 + alarm_log->chart_context = sqlite3_text_strdupz_empty(res, CHART_CONTEXT);

400 + alarm_log->chart_name = sqlite3_text_strdupz_empty(res, CHART_NAME);

401 +

402 + alarm_log->transition_id = sqlite3_uuid_unparse_strdupz(res, TRANSITION_ID);

403 + alarm_log->event_id = sqlite3_column_int64(res, ALARM_EVENT_ID);

404 + alarm_log->version = sqlite3_column_int64(res, VERSION);

405 +

406 + alarm_log->summary = sqlite3_text_strdupz_empty(res, SUMMARY);

407 +

408 + alarm_log->health_log_id = sqlite3_column_int64(res, HEALTH_LOG_ID);

409 + alarm_log->unique_id = sqlite3_column_int64(res, UNIQUE_ID);

410 + alarm_log->alarm_id = sqlite3_column_int64(res, ALARM_ID);

411 + alarm_log->sequence_id = sqlite3_column_int64(res, SEQUENCE_ID);

412 +}

413 +

414 +#define SQL_SELECT_ALERT_TO_PUSH \

415 + "SELECT aq.sequence_id, hld.unique_id, hld.alarm_id, hl.config_hash_id, hld.updated_by_id, hld.when_key," \

416 + " hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, hld.delay_up_to_timestamp, hl.name," \

417 + " hl.chart, hl.exec, hl.recipient, ah.source, hl.units, hld.info, hld.exec_code, hld.new_status," \

418 + " hld.old_status, hld.delay, hld.new_value, hld.old_value, hld.last_repeat, hl.chart_context, hld.transition_id," \

419 + " hld.alarm_event_id, hl.chart_name, hld.summary, hld.health_log_id, hld.when_key" \

420 + " FROM health_log hl, aclk_queue aq, alert_hash ah, health_log_detail hld" \

421 + " WHERE hld.unique_id = aq.unique_id AND hl.config_hash_id = ah.hash_id" \

422 + " AND hl.host_id = @host_id AND aq.host_id = hl.host_id AND hl.health_log_id = hld.health_log_id" \

423 + " ORDER BY aq.sequence_id ASC LIMIT "ACLK_MAX_ALERT_UPDATES

424 +

425 +static void aclk_push_alert_event(RRDHOST *host __maybe_unused)

426 +{

427 +

428 char *claim_id = get_agent_claimid();

237 - if (unlikely(!claim_id))

429 + if (!claim_id || !host->node_id)

430 return;

431

240 - if (unlikely(!wc->host)) {

432 + sqlite3_stmt *res = NULL;

433 +

434 + if (!PREPARE_STATEMENT(db_meta, SQL_SELECT_ALERT_TO_PUSH, &res)) {

435 freez(claim_id);

436 return;

437 }

438

245 - BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);

246 -

247 - sqlite3_stmt *res = NULL;

439 + int param = 0;

440 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

441

249 - buffer_sprintf(

250 - sql,

251 - "SELECT aa.sequence_id, hld.unique_id, hld.alarm_id, hl.config_hash_id, hld.updated_by_id, hld.when_key, "

252 - " hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, hld.delay_up_to_timestamp, hl.name, "

253 - " hl.chart, hl.exec, hl.recipient, ha.source, hl.units, hld.info, hld.exec_code, hld.new_status, "

254 - " hld.old_status, hld.delay, hld.new_value, hld.old_value, hld.last_repeat, hl.chart_context, hld.transition_id, "

255 - " hld.alarm_event_id, hl.chart_name, hld.summary "

256 - " FROM health_log hl, aclk_alert_%s aa, alert_hash ha, health_log_detail hld "

257 - " WHERE hld.unique_id = aa.alert_unique_id AND hl.config_hash_id = ha.hash_id AND aa.date_submitted IS NULL "

258 - " AND hl.host_id = @host_id AND hl.health_log_id = hld.health_log_id "

259 - " ORDER BY aa.sequence_id ASC LIMIT "ACLK_MAX_ALERT_UPDATES,

260 - wc->uuid_str);

261 -

262 - if (!PREPARE_STATEMENT(db_meta, buffer_tostring(sql), &res)) {

263 -

264 - BUFFER *sql_fix = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);

265 - buffer_sprintf(sql_fix, TABLE_ACLK_ALERT, wc->uuid_str);

266 -

267 - rc = db_execute(db_meta, buffer_tostring(sql_fix));

268 - if (unlikely(rc))

269 - error_report("Failed to create ACLK alert table for host %s", rrdhost_hostname(wc->host));

270 - buffer_free(sql_fix);

271 -

272 - // Try again

273 - if (!PREPARE_STATEMENT(db_meta, buffer_tostring(sql), &res)) {

274 - buffer_free(sql);

275 - freez(claim_id);

276 - return;

277 - }

278 - }

442 + char node_id_str[UUID_STR_LEN];

443 + uuid_unparse_lower(*host->node_id, node_id_str);

444

280 - rc = sqlite3_bind_blob(res, 1, &wc->host->host_uuid, sizeof(wc->host->host_uuid), SQLITE_STATIC);

281 - if (unlikely(rc != SQLITE_OK)) {

282 - error_report("Failed to bind host_id for pushing alert event.");

283 - goto done;

284 - }

445 + struct alarm_log_entry alarm_log;

446 + alarm_log.node_id = node_id_str;

447 + alarm_log.claim_id = claim_id;

448

286 - uint64_t first_sequence_id = 0;

287 - uint64_t last_sequence_id = 0;

449 + int64_t first_id = 0;

450 + int64_t last_id = 0;

451

452 + param = 0;

453 + RRDCALC_STATUS status;

454 + struct aclk_sync_cfg_t *wc = host->aclk_config;

455 while (sqlite3_step_monitored(res) == SQLITE_ROW) {

290 - struct alarm_log_entry alarm_log;

291 - char old_value_string[100 + 1];

292 - char new_value_string[100 + 1];

293 -

294 - alarm_log.node_id = wc->node_id;

295 - alarm_log.claim_id = claim_id;

296 - alarm_log.chart = strdupz((char *)sqlite3_column_text(res, 12));

297 - alarm_log.name = strdupz((char *)sqlite3_column_text(res, 11));

298 - alarm_log.when = (time_t) sqlite3_column_int64(res, 5);

299 - alarm_log.config_hash = sqlite3_uuid_unparse_strdupz(res, 3);

300 - alarm_log.utc_offset = wc->host->utc_offset;

301 - alarm_log.timezone = strdupz(rrdhost_abbrev_timezone(wc->host));

302 - alarm_log.exec_path = sqlite3_column_bytes(res, 13) > 0 ? strdupz((char *)sqlite3_column_text(res, 13)) :

303 - strdupz((char *)string2str(wc->host->health.health_default_exec));

304 - alarm_log.conf_source = sqlite3_column_bytes(res, 15) > 0 ? strdupz((char *)sqlite3_column_text(res, 15)) : strdupz("");

305 -

306 - char *edit_command = sqlite3_column_bytes(res, 15) > 0 ?

307 - health_edit_command_from_source((char *)sqlite3_column_text(res, 15)) :

308 - strdupz("UNKNOWN=0=UNKNOWN");

309 - alarm_log.command = strdupz(edit_command);

310 -

311 - time_t duration = (time_t) sqlite3_column_int64(res, 6);

312 - alarm_log.duration = (duration > 0) ? duration : 0;

313 - alarm_log.non_clear_duration = (time_t) sqlite3_column_int64(res, 7);

314 - alarm_log.status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 19));

315 - alarm_log.old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 20));

316 - alarm_log.delay = (int) sqlite3_column_int(res, 21);

317 - alarm_log.delay_up_to_timestamp = (time_t) sqlite3_column_int64(res, 10);

318 - alarm_log.last_repeat = (time_t) sqlite3_column_int64(res, 24);

319 - alarm_log.silenced = ((sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_SILENCED) ||

320 - (sqlite3_column_type(res, 14) != SQLITE_NULL &&

321 - !strncmp((char *)sqlite3_column_text(res, 14), "silent", 6))) ?

322 - 1 :

323 - 0;

324 - alarm_log.value_string =

325 - sqlite3_column_type(res, 22) == SQLITE_NULL ?

326 - strdupz((char *)"-") :

327 - strdupz((char *)format_value_and_unit(

328 - new_value_string, 100, sqlite3_column_double(res, 22), (char *)sqlite3_column_text(res, 16), -1));

329 - alarm_log.old_value_string =

330 - sqlite3_column_type(res, 23) == SQLITE_NULL ?

331 - strdupz((char *)"-") :

332 - strdupz((char *)format_value_and_unit(

333 - old_value_string, 100, sqlite3_column_double(res, 23), (char *)sqlite3_column_text(res, 16), -1));

334 - alarm_log.value = (NETDATA_DOUBLE) sqlite3_column_double(res, 22);

335 - alarm_log.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 23);

336 - alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;

337 - alarm_log.rendered_info = sqlite3_text_strdupz_empty(res, 17);

338 - alarm_log.chart_context = sqlite3_text_strdupz_empty(res, 25);

339 - alarm_log.transition_id = sqlite3_uuid_unparse_strdupz(res, 26);

340 - alarm_log.event_id = (time_t) sqlite3_column_int64(res, 27);

341 - alarm_log.chart_name = sqlite3_text_strdupz_empty(res, 28);

342 - alarm_log.summary = sqlite3_text_strdupz_empty(res, 29);

343 -

456 + health_alarm_log_populate(&alarm_log, res, host, &status);

457 aclk_send_alarm_log_entry(&alarm_log);

458 + wc->alert_count++;

459

346 - if (first_sequence_id == 0)

347 - first_sequence_id = (uint64_t) sqlite3_column_int64(res, 0);

348 -

349 - if (wc->alerts_log_first_sequence_id == 0)

350 - wc->alerts_log_first_sequence_id = (uint64_t) sqlite3_column_int64(res, 0);

460 + last_id = alarm_log.sequence_id;

461 + if (first_id == 0)

462 + first_id = last_id;

463

352 - last_sequence_id = (uint64_t) sqlite3_column_int64(res, 0);

353 - wc->alerts_log_last_sequence_id = (uint64_t) sqlite3_column_int64(res, 0);

464 + sql_update_alert_version(alarm_log.health_log_id, alarm_log.unique_id, status, alarm_log.version);

465

466 destroy_alarm_log_entry(&alarm_log);

356 - freez(edit_command);

467 }

468

359 - if (first_sequence_id) {

360 - buffer_flush(sql);

361 - buffer_sprintf(

362 - sql,

363 - "UPDATE aclk_alert_%s SET date_submitted=unixepoch() "

364 - "WHERE +date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64,

365 - wc->uuid_str,

366 - first_sequence_id,

367 - last_sequence_id);

368 -

369 - if (unlikely(db_execute(db_meta, buffer_tostring(sql))))

370 - error_report("Failed to mark ACLK alert entries as submitted for host %s", rrdhost_hostname(wc->host));

371 -

469 + if (first_id) {

470 + nd_log(

471 + NDLS_ACCESS,

472 + NDLP_DEBUG,

473 + "ACLK RES [%s (%s)]: ALERTS SENT from %ld - %ld",

474 + node_id_str,

475 + rrdhost_hostname(host),

476 + first_id,

477 + last_id);

478 +

479 + delete_alert_from_submit_queue(host, first_id, last_id);

480 // Mark to do one more check

373 - rrdhost_flag_set(wc->host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

374 -

375 - } else {

376 - if (wc->alerts_log_first_sequence_id)

377 - nd_log(NDLS_ACCESS, NDLP_DEBUG,

378 - "ACLK RES [%s (%s)]: ALERTS SENT from %" PRIu64 " to %" PRIu64 "",

379 - wc->node_id,

380 - wc->host ? rrdhost_hostname(wc->host) : "N/A",

381 - wc->alerts_log_first_sequence_id,

382 - wc->alerts_log_last_sequence_id);

383 - wc->alerts_log_first_sequence_id = 0;

384 - wc->alerts_log_last_sequence_id = 0;

481 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

482 }

483

484 done:

485 + REPORT_BIND_FAIL(res, param);

486 SQLITE_FINALIZE(res);

487

488 freez(claim_id);

391 - buffer_free(sql);

392 -#endif

489 }

490

395 -void aclk_push_alert_events_for_all_hosts(void)

491 +#define SQL_DELETE_PROCESSED_ROWS \

492 + "DELETE FROM alert_queue WHERE host_id = @host_id AND rowid between @row1 AND @row2"

493 +

494 +static void delete_alert_from_pending_queue(RRDHOST *host, int64_t row1, int64_t row2)

495 {

397 - RRDHOST *host;

496 + static __thread sqlite3_stmt *res = NULL;

497

399 - dfe_start_reentrant(rrdhost_root_index, host) {

400 - if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) ||

401 - !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS))

402 - continue;

498 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_DELETE_PROCESSED_ROWS, &res))

499 + return;

500

404 - rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

501 + int param = 0;

502 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

503 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, row1));

504 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, row2));

505

406 - struct aclk_sync_cfg_t *wc = host->aclk_config;

407 - if (likely(wc))

408 - aclk_push_alert_event(wc);

409 - }

410 - dfe_done(host);

506 + param = 0;

507 + int rc = sqlite3_step_monitored(res);

508 + if (rc != SQLITE_DONE)

509 + error_report("Failed to delete processed rows, rc = %d", rc);

510 +

511 +done:

512 + REPORT_BIND_FAIL(res, param);

513 + SQLITE_RESET(res);

514 }

515

413 -void sql_queue_existing_alerts_to_aclk(RRDHOST *host)

516 +#define SQL_REBUILD_HOST_ALERT_VERSION_TABLE \

517 + "INSERT INTO alert_version (health_log_id, unique_id, status, version, date_submitted) " \

518 + " SELECT hl.health_log_id, hld.unique_id, hld.new_status, hld.when_key, UNIXEPOCH() " \

519 + " FROM health_log hl, health_log_detail hld WHERE " \

520 + " hl.host_id = @host_id AND hld.health_log_id = hl.health_log_id AND hld.transition_id = hl.last_transition_id"

521 +

522 +#define SQL_DELETE_HOST_ALERT_VERSION_TABLE \

523 + "DELETE FROM alert_version WHERE health_log_id IN (SELECT health_log_id FROM health_log WHERE host_id = @host_id)"

524 +

525 +void rebuild_host_alert_version_table(RRDHOST *host)

526 {

527 sqlite3_stmt *res = NULL;

416 - int rc;

528

418 - struct aclk_sync_cfg_t *wc = host->aclk_config;

529 + if (!PREPARE_STATEMENT(db_meta, SQL_DELETE_HOST_ALERT_VERSION_TABLE, &res))

530 + return;

531

420 - BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);

532 + int param = 0;

533 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

534

422 - rw_spinlock_write_lock(&host->health_log.spinlock);

535 + param = 0;

536 + int rc = execute_insert(res);

537 + if (rc != SQLITE_DONE) {

538 + netdata_log_error("Failed to delete the host alert version table");

539 + goto done;

540 + }

541 +

542 + SQLITE_FINALIZE(res);

543 + if (!PREPARE_STATEMENT(db_meta, SQL_REBUILD_HOST_ALERT_VERSION_TABLE, &res))

544 + return;

545

424 - buffer_sprintf(sql, "DELETE FROM aclk_alert_%s", wc->uuid_str);

425 - if (unlikely(db_execute(db_meta, buffer_tostring(sql))))

426 - goto skip;

546 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

547 +

548 + param = 0;

549 + rc = execute_insert(res);

550 + if (rc != SQLITE_DONE)

551 + netdata_log_error("Failed to rebuild the host alert version table");

552 +

553 +done:

554 + REPORT_BIND_FAIL(res, param);

555 + SQLITE_FINALIZE(res);

556 +}

557

428 - buffer_flush(sql);

558 +#define SQL_PROCESS_ALERT_PENDING_QUEUE \

559 + "SELECT health_log_id, unique_id, status, rowid" \

560 + " FROM alert_queue WHERE host_id = @host_id AND date_scheduled <= UNIXEPOCH() ORDER BY rowid ASC"

561

430 - buffer_sprintf(

431 - sql,

432 - "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) "

433 - "SELECT hld.unique_id alert_unique_id, unixepoch(), hld.unique_id alert_unique_id FROM health_log_detail hld, health_log hl "

434 - "WHERE hld.new_status <> 0 AND hld.new_status <> -2 AND hl.health_log_id = hld.health_log_id AND hl.config_hash_id IS NOT NULL "

435 - "AND hld.updated_by_id = 0 AND hl.host_id = @host_id ORDER BY hld.unique_id ASC ON CONFLICT (alert_unique_id) DO NOTHING",

436 - wc->uuid_str);

562 +bool process_alert_pending_queue(RRDHOST *host)

563 +{

564 + static __thread sqlite3_stmt *res = NULL;

565

438 - if (!PREPARE_STATEMENT(db_meta, buffer_tostring(sql), &res))

439 - goto skip;

566 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_PROCESS_ALERT_PENDING_QUEUE, &res))

567 + return false;

568

569 int param = 0;

570 + int added =0, count = 0;

571 SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

572

573 param = 0;

445 - rc = execute_insert(res);

446 - if (unlikely(rc != SQLITE_DONE))

447 - error_report("Failed to queue existing alerts, rc = %d", rc);

448 - else

449 - rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

574 + int64_t start_row = 0;

575 + int64_t end_row = 0;

576 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

577 +

578 + int64_t health_log_id = sqlite3_column_int64(res, 0);

579 + uint32_t unique_id = sqlite3_column_int64(res, 1);

580 + RRDCALC_STATUS new_status = sqlite3_column_int(res, 2);

581 + int64_t row = sqlite3_column_int64(res, 3);

582 +

583 + if (host->aclk_config) {

584 + int ret = insert_alert_to_submit_queue(host, health_log_id, unique_id, new_status);

585 + if (ret == 0)

586 + added++;

587 + }

588 +

589 + if (!start_row)

590 + start_row = row;

591 + end_row = row;

592

593 + count++;

594 + }

595 +

596 + if (start_row)

597 + delete_alert_from_pending_queue(host, start_row, end_row);

598 +

599 + if(count)

600 + nd_log(NDLS_ACCESS, NDLP_NOTICE, "ACLK STA [%s (N/A)]: Processed %d entries, queued %d", rrdhost_hostname(host), count, added);

601 done:

602 REPORT_BIND_FAIL(res, param);

453 - SQLITE_FINALIZE(res);

603 + SQLITE_RESET(res);

604 + return added > 0;

605 +}

606 +

607 +void aclk_push_alert_events_for_all_hosts(void)

608 +{

609 + RRDHOST *host;

610 +

611 + // Checking if we shutting down

612 + if (!service_running(SERVICE_ACLK))

613 + return;

614 +

615 + dfe_start_reentrant(rrdhost_root_index, host) {

616 + if (!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS) ||

617 + rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))

618 + continue;

619

455 -skip:

456 - rw_spinlock_write_unlock(&host->health_log.spinlock);

457 - buffer_free(sql);

620 + rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

621 +

622 + struct aclk_sync_cfg_t *wc = host->aclk_config;

623 + if (!wc || false == wc->stream_alerts || rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {

624 + (void)process_alert_pending_queue(host);

625 + commit_alert_events(host);

626 + continue;

627 + }

628 +

629 + if (wc->send_snapshot) {

630 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

631 + if (wc->send_snapshot == 1)

632 + continue;

633 + (void)process_alert_pending_queue(host);

634 + commit_alert_events(host);

635 + rebuild_host_alert_version_table(host);

636 + send_alert_snapshot_to_cloud(host);

637 + wc->snapshot_count++;

638 + wc->send_snapshot = 0;

639 + }

640 + else

641 + aclk_push_alert_event(host);

642 + }

643 + dfe_done(host);

644 }

645

460 -void aclk_send_alarm_configuration(char *config_hash)

646 +void aclk_send_alert_configuration(char *config_hash)

647 {

648 if (unlikely(!config_hash))

649 return;

@@ -484,7 +670,6 @@ void aclk_send_alarm_configuration(char *config_hash)

670

671 void aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_hash __maybe_unused)

672 {

487 -#ifdef ENABLE_ACLK

673 sqlite3_stmt *res = NULL;

674 struct aclk_sync_cfg_t *wc;

675

@@ -586,91 +771,60 @@ done:

771 SQLITE_FINALIZE(res);

772 freez(config_hash);

773 freez(node_id);

589 -#endif

774 }

775

776 +#define SQL_ALERT_VERSION_CALC \

777 + "SELECT SUM(version) FROM health_log hl, alert_version av" \

778 + " WHERE hl.host_id = @host_uuid AND hl.health_log_id = av.health_log_id AND av.status <> -2"

779

593 -// Start streaming alerts

594 -void aclk_start_alert_streaming(char *node_id, bool resets)

780 +static uint64_t calculate_node_alert_version(RRDHOST *host)

781 {

596 - nd_uuid_t node_uuid;

597 -

598 - if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))

599 - return;

600 -

601 - struct aclk_sync_cfg_t *wc;

602 -

603 - RRDHOST *host = find_host_by_node_id(node_id);

604 - if (unlikely(!host || !(wc = host->aclk_config)))

605 - return;

606 -

607 - if (unlikely(!host->health.health_enabled)) {

608 - nd_log(NDLS_ACCESS, NDLP_NOTICE, "ACLK STA [%s (N/A)]: Ignoring request to stream alert state changes, health is disabled.", node_id);

609 - return;

610 - }

611 -

612 - if (resets) {

613 - nd_log(NDLS_ACCESS, NDLP_DEBUG, "ACLK REQ [%s (%s)]: STREAM ALERTS ENABLED (RESET REQUESTED)", node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");

614 - sql_queue_existing_alerts_to_aclk(host);

615 - } else

616 - nd_log(NDLS_ACCESS, NDLP_DEBUG, "ACLK REQ [%s (%s)]: STREAM ALERTS ENABLED", node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");

617 -

618 - wc->alert_updates = 1;

619 - wc->alert_queue_removed = SEND_REMOVED_AFTER_HEALTH_LOOPS;

620 -}

621 -

622 -#define SQL_QUEUE_REMOVE_ALERTS \

623 - "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \

624 - "SELECT hld.unique_id alert_unique_id, UNIXEPOCH(), hld.unique_id alert_unique_id FROM health_log hl, health_log_detail hld " \

625 - "WHERE hl.host_id = @host_id AND hl.health_log_id = hld.health_log_id AND hld.new_status = -2 AND hld.updated_by_id = 0 " \

626 - "AND hld.unique_id NOT IN (SELECT alert_unique_id FROM aclk_alert_%s) " \

627 - "AND hl.config_hash_id NOT IN (SELECT hash_id FROM alert_hash WHERE warn IS NULL AND crit IS NULL) " \

628 - "AND hl.name || hl.chart NOT IN (select name || chart FROM health_log WHERE name = hl.name AND " \

629 - "chart = hl.chart AND alarm_id > hl.alarm_id AND host_id = hl.host_id) " \

630 - "ORDER BY hld.unique_id ASC ON CONFLICT (alert_unique_id) DO NOTHING"

631 -

632 -void sql_process_queue_removed_alerts_to_aclk(char *node_id)

633 -{

634 - struct aclk_sync_cfg_t *wc;

635 - RRDHOST *host = find_host_by_node_id(node_id);

636 - freez(node_id);

637 -

638 - if (unlikely(!host || !(wc = host->aclk_config)))

639 - return;

640 -

641 - sqlite3_stmt *res = NULL;

782 + static __thread sqlite3_stmt *res = NULL;

783

643 - CLEAN_BUFFER *wb = buffer_create(1024, NULL); // Note buffer auto free on function return

644 - buffer_sprintf(wb, SQL_QUEUE_REMOVE_ALERTS, wc->uuid_str, wc->uuid_str);

645 -

646 - if (!PREPARE_STATEMENT(db_meta, buffer_tostring(wb), &res))

647 - return;

784 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_ALERT_VERSION_CALC, &res))

785 + return 0;

786

787 + uint64_t version = 0;

788 int param = 0;

789 SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

790

791 param = 0;

653 - int rc = execute_insert(res);

654 - if (likely(rc == SQLITE_DONE)) {

655 - nd_log(NDLS_ACCESS, NDLP_DEBUG, "ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS", wc->node_id, rrdhost_hostname(wc->host));

656 - rrdhost_flag_set(wc->host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

657 - wc->alert_queue_removed = 0;

792 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

793 + version = (uint64_t)sqlite3_column_int64(res, 0);

794 }

795

796 done:

797 REPORT_BIND_FAIL(res, param);

662 - SQLITE_FINALIZE(res);

798 + SQLITE_RESET(res);

799 + return version;

800 }

801

665 -void sql_queue_removed_alerts_to_aclk(RRDHOST *host)

802 +static void schedule_alert_snapshot_if_needed(struct aclk_sync_cfg_t *wc, uint64_t cloud_version)

803 {

667 - if (unlikely(!host->aclk_config || !claimed() || !host->node_id))

668 - return;

669 -

670 - char node_id[UUID_STR_LEN];

671 - uuid_unparse_lower(*host->node_id, node_id);

804 + uint64_t local_version = calculate_node_alert_version(wc->host);

805 + if (local_version != cloud_version) {

806 + nd_log(

807 + NDLS_ACCESS,

808 + NDLP_NOTICE,

809 + "Scheduling alert snapshot for host \"%s\", node \"%s\" (version: cloud %zu, local %zu)",

810 + rrdhost_hostname(wc->host),

811 + wc->node_id,

812 + cloud_version,

813 + local_version);

814

673 - aclk_push_node_removed_alerts(node_id);

815 + wc->send_snapshot = 1;

816 + rrdhost_flag_set(wc->host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

817 + }

818 + else

819 + nd_log(

820 + NDLS_ACCESS,

821 + NDLP_DEBUG,

822 + "Alert check on \"%s\", node \"%s\" (version: cloud %zu, local %zu)",

823 + rrdhost_hostname(wc->host),

824 + wc->node_id,

825 + cloud_version,

826 + local_version);

827 + wc->checkpoint_count++;

828 }

829

830 void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id __maybe_unused, char *snapshot_uuid)

@@ -699,371 +853,203 @@ void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id __maybe_unus

853

854 wc->alerts_snapshot_uuid = strdupz(snapshot_uuid);

855

702 - aclk_push_node_alert_snapshot(node_id);

856 + wc->send_snapshot = 1;

857 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

858 }

859

705 -#ifdef ENABLE_ACLK

706 -void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_ENTRY *ae, RRDHOST *host)

707 -{

708 - char *edit_command = ae->source ? health_edit_command_from_source(ae_source(ae)) : strdupz("UNKNOWN=0=UNKNOWN");

709 - char config_hash_id[UUID_STR_LEN];

710 - uuid_unparse_lower(ae->config_hash_id, config_hash_id);

711 - char transition_id[UUID_STR_LEN];

712 - uuid_unparse_lower(ae->transition_id, transition_id);

713 -

714 - alarm_log->chart = strdupz(ae_chart_id(ae));

715 - alarm_log->name = strdupz(ae_name(ae));

716 -

717 - alarm_log->when = ae->when;

860 +#define SQL_COUNT_SNAPSHOT_ENTRIES \

861 + "SELECT COUNT(1) FROM alert_version av, health_log hl " \

862 + "WHERE hl.host_id = @host_id AND hl.health_log_id = av.health_log_id"

863

719 - alarm_log->config_hash = strdupz((char *)config_hash_id);

720 -

721 - alarm_log->utc_offset = host->utc_offset;

722 - alarm_log->timezone = strdupz(rrdhost_abbrev_timezone(host));

723 - alarm_log->exec_path = ae->exec ? strdupz(ae_exec(ae)) : strdupz((char *)string2str(host->health.health_default_exec));

724 - alarm_log->conf_source = ae->source ? strdupz(ae_source(ae)) : strdupz((char *)"");

725 -

726 - alarm_log->command = strdupz((char *)edit_command);

727 -

728 - alarm_log->duration = (time_t)ae->duration;

729 - alarm_log->non_clear_duration = (time_t)ae->non_clear_duration;

730 - alarm_log->status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS)ae->new_status);

731 - alarm_log->old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS)ae->old_status);

732 - alarm_log->delay = ae->delay;

733 - alarm_log->delay_up_to_timestamp = (time_t)ae->delay_up_to_timestamp;

734 - alarm_log->last_repeat = (time_t)ae->last_repeat;

735 -

736 - alarm_log->silenced =

737 - ((ae->flags & HEALTH_ENTRY_FLAG_SILENCED) || (ae->recipient && !strncmp(ae_recipient(ae), "silent", 6))) ?

738 - 1 :

739 - 0;

864 +static int calculate_alert_snapshot_entries(nd_uuid_t *host_uuid)

865 +{

866 + int count = 0;

867

741 - alarm_log->value_string = strdupz(ae_new_value_string(ae));

742 - alarm_log->old_value_string = strdupz(ae_old_value_string(ae));

868 + sqlite3_stmt *res = NULL;

869

744 - alarm_log->value = (!isnan(ae->new_value)) ? (NETDATA_DOUBLE)ae->new_value : 0;

745 - alarm_log->old_value = (!isnan(ae->old_value)) ? (NETDATA_DOUBLE)ae->old_value : 0;

870 + if (!PREPARE_STATEMENT(db_meta, SQL_COUNT_SNAPSHOT_ENTRIES, &res))

871 + return 0;

872

747 - alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;

748 - alarm_log->rendered_info = strdupz(ae_info(ae));

749 - alarm_log->chart_context = strdupz(ae_chart_context(ae));

750 - alarm_log->chart_name = strdupz(ae_chart_name(ae));

873 + int param = 0;

874 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC));

875

752 - alarm_log->transition_id = strdupz((char *)transition_id);

753 - alarm_log->event_id = (uint64_t) ae->alarm_event_id;

876 + param = 0;

877 + int rc = sqlite3_step_monitored(res);

878 + if (rc == SQLITE_ROW)

879 + count = sqlite3_column_int(res, 0);

880 + else

881 + error_report("Failed to select snapshot count");

882

755 - alarm_log->summary = strdupz(ae_summary(ae));

883 +done:

884 + REPORT_BIND_FAIL(res, param);

885 + SQLITE_FINALIZE(res);

886

757 - freez(edit_command);

887 + return count;

888 }

759 -#endif

760 -

761 -#ifdef ENABLE_ACLK

762 -static bool have_recent_alarm_unsafe(RRDHOST *host, int64_t alarm_id, int64_t mark)

763 -{

764 - ALARM_ENTRY *ae = host->health_log.alarms;

765 -

766 - while (ae) {

767 - if (ae->alarm_id == alarm_id && ae->unique_id >mark &&

768 - (ae->new_status != RRDCALC_STATUS_WARNING && ae->new_status != RRDCALC_STATUS_CRITICAL))

769 - return true;

770 - ae = ae->next;

771 - }

889

773 - return false;

774 -}

775 -#endif

890 +#define SQL_GET_SNAPSHOT_ENTRIES \

891 + " SELECT 0, hld.unique_id, hld.alarm_id, hl.config_hash_id, hld.updated_by_id, hld.when_key, " \

892 + " hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, hld.delay_up_to_timestamp, hl.name, " \

893 + " hl.chart, hl.exec, hl.recipient, ah.source, hl.units, hld.info, hld.exec_code, hld.new_status, " \

894 + " hld.old_status, hld.delay, hld.new_value, hld.old_value, hld.last_repeat, hl.chart_context, hld.transition_id, " \

895 + " hld.alarm_event_id, hl.chart_name, hld.summary, hld.health_log_id, av.version " \

896 + " FROM health_log hl, alert_hash ah, health_log_detail hld, alert_version av " \

897 + " WHERE hl.config_hash_id = ah.hash_id" \

898 + " AND hl.host_id = @host_id AND hl.health_log_id = hld.health_log_id " \

899 + " AND hld.health_log_id = av.health_log_id AND av.unique_id = hld.unique_id"

900

901 #define ALARM_EVENTS_PER_CHUNK 1000

778 -void aclk_push_alert_snapshot_event(char *node_id __maybe_unused)

902 +void send_alert_snapshot_to_cloud(RRDHOST *host __maybe_unused)

903 {

780 -#ifdef ENABLE_ACLK

781 - RRDHOST *host = find_host_by_node_id(node_id);

904 + struct aclk_sync_cfg_t *wc = host->aclk_config;

905

906 if (unlikely(!host)) {

784 - nd_log(NDLS_ACCESS, NDLP_WARNING, "AC [%s (N/A)]: Node id not found", node_id);

785 - freez(node_id);

907 + nd_log(NDLS_ACCESS, NDLP_WARNING, "AC [%s (N/A)]: Node id not found", wc->node_id);

908 return;

909 }

788 - freez(node_id);

789 -

790 - struct aclk_sync_cfg_t *wc = host->aclk_config;

910

792 - // we perhaps we don't need this for snapshots

793 - if (unlikely(!wc->alert_updates)) {

794 - nd_log(NDLS_ACCESS, NDLP_NOTICE,

795 - "ACLK STA [%s (%s)]: Ignoring alert snapshot event, updates have been turned off for this node.",

796 - wc->node_id,

797 - wc->host ? rrdhost_hostname(wc->host) : "N/A");

911 + char *claim_id = get_agent_claimid();

912 + if (unlikely(!claim_id))

913 return;

799 - }

914

801 - if (unlikely(!wc->alerts_snapshot_uuid))

915 + // Check database for this node to see how many alerts we will need to put in the snapshot

916 + int cnt = calculate_alert_snapshot_entries(&host->host_uuid);

917 + if (!cnt) {

918 + freez(claim_id);

919 return;

920 + }

921

804 - char *claim_id = get_agent_claimid();

805 - if (unlikely(!claim_id))

922 + sqlite3_stmt *res = NULL;

923 + if (!PREPARE_STATEMENT(db_meta, SQL_GET_SNAPSHOT_ENTRIES, &res))

924 return;

925

808 - nd_log(NDLS_ACCESS, NDLP_DEBUG, "ACLK REQ [%s (%s)]: Sending alerts snapshot, snapshot_uuid %s", wc->node_id, rrdhost_hostname(wc->host), wc->alerts_snapshot_uuid);

926 + int param = 0;

927 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

928

810 - uint32_t cnt = 0;

929 + nd_uuid_t local_snapshot_uuid;

930 + char snapshot_uuid_str[UUID_STR_LEN];

931 + uuid_generate_random(local_snapshot_uuid);

932 + uuid_unparse_lower(local_snapshot_uuid, snapshot_uuid_str);

933 + char *snapshot_uuid = &snapshot_uuid_str[0];

934

812 - rw_spinlock_read_lock(&host->health_log.spinlock);

935 + nd_log(NDLS_ACCESS, NDLP_DEBUG,

936 + "ACLK REQ [%s (%s)]: Sending %d alerts snapshot, snapshot_uuid %s", wc->node_id, rrdhost_hostname(host),

937 + cnt, snapshot_uuid);

938

814 - ALARM_ENTRY *ae = host->health_log.alarms;

939 + uint32_t chunks;

940 + chunks = (cnt / ALARM_EVENTS_PER_CHUNK) + (cnt % ALARM_EVENTS_PER_CHUNK != 0);

941

816 - for (; ae; ae = ae->next) {

817 - if (likely(ae->updated_by_id))

818 - continue;

942 + alarm_snapshot_proto_ptr_t snapshot_proto = NULL;

943 + struct alarm_snapshot alarm_snap;

944 + struct alarm_log_entry alarm_log;

945

820 - if (unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED))

821 - continue;

946 + alarm_snap.node_id = wc->node_id;

947 + alarm_snap.claim_id = claim_id;

948 + alarm_snap.snapshot_uuid = snapshot_uuid;

949 + alarm_snap.chunks = chunks;

950 + alarm_snap.chunk = 1;

951

823 - if (have_recent_alarm_unsafe(host, ae->alarm_id, ae->unique_id))

824 - continue;

952 + alarm_log.node_id = wc->node_id;

953 + alarm_log.claim_id = claim_id;

954

826 - if (is_event_from_alert_variable_config(ae->unique_id, &host->host_uuid))

827 - continue;

955

956 + cnt = 0;

957 + param = 0;

958 + uint64_t version = 0;

959 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

960 cnt++;

830 - }

831 -

832 - if (cnt) {

833 - uint32_t chunks;

834 -

835 - chunks = (cnt / ALARM_EVENTS_PER_CHUNK) + (cnt % ALARM_EVENTS_PER_CHUNK != 0);

836 - ae = host->health_log.alarms;

961

838 - cnt = 0;

839 - struct alarm_snapshot alarm_snap;

840 - alarm_snap.node_id = wc->node_id;

841 - alarm_snap.claim_id = claim_id;

842 - alarm_snap.snapshot_uuid = wc->alerts_snapshot_uuid;

843 - alarm_snap.chunks = chunks;

844 - alarm_snap.chunk = 1;

962 + if (!snapshot_proto)

963 + snapshot_proto = generate_alarm_snapshot_proto(&alarm_snap);

964

846 - alarm_snapshot_proto_ptr_t snapshot_proto = NULL;

965 + health_alarm_log_populate(&alarm_log, res, host, NULL);

966

848 - for (; ae; ae = ae->next) {

849 - if (likely(ae->updated_by_id) || unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED))

850 - continue;

851 -

852 - if (have_recent_alarm_unsafe(host, ae->alarm_id, ae->unique_id))

853 - continue;

854 -

855 - if (is_event_from_alert_variable_config(ae->unique_id, &host->host_uuid))

856 - continue;

857 -

858 - cnt++;

859 -

860 - struct alarm_log_entry alarm_log;

861 - alarm_log.node_id = wc->node_id;

862 - alarm_log.claim_id = claim_id;

863 -

864 - if (!snapshot_proto)

865 - snapshot_proto = generate_alarm_snapshot_proto(&alarm_snap);

967 + //log_alarm_log(&alarm_log);

968

867 - health_alarm_entry2proto_nolock(&alarm_log, ae, host);

868 - add_alarm_log_entry2snapshot(snapshot_proto, &alarm_log);

969 + add_alarm_log_entry2snapshot(snapshot_proto, &alarm_log);

970 + version += alarm_log.version;

971

870 - if (cnt == ALARM_EVENTS_PER_CHUNK) {

972 + if (cnt == ALARM_EVENTS_PER_CHUNK) {

973 + if (aclk_connected)

974 aclk_send_alarm_snapshot(snapshot_proto);

872 - cnt = 0;

873 - if (alarm_snap.chunk < chunks) {

874 - alarm_snap.chunk++;

875 - snapshot_proto = generate_alarm_snapshot_proto(&alarm_snap);

876 - }

975 + cnt = 0;

976 + if (alarm_snap.chunk < chunks) {

977 + alarm_snap.chunk++;

978 + snapshot_proto = generate_alarm_snapshot_proto(&alarm_snap);

979 }

878 - destroy_alarm_log_entry(&alarm_log);

980 }

880 - if (cnt)

881 - aclk_send_alarm_snapshot(snapshot_proto);

981 + destroy_alarm_log_entry(&alarm_log);

982 }

983 + if (cnt)

984 + aclk_send_alarm_snapshot(snapshot_proto);

985

884 - rw_spinlock_read_unlock(&host->health_log.spinlock);

885 - wc->alerts_snapshot_uuid = NULL;

886 -

887 - freez(claim_id);

888 -#endif

889 -}

890 -

891 -#define SQL_DELETE_ALERT_ENTRIES "DELETE FROM aclk_alert_%s WHERE date_created < UNIXEPOCH() - @period"

892 -

893 -void sql_aclk_alert_clean_dead_entries(RRDHOST *host)

894 -{

895 - struct aclk_sync_cfg_t *wc = host->aclk_config;

896 - if (unlikely(!wc))

897 - return;

898 -

899 - char sql[ACLK_SYNC_QUERY_SIZE];

900 - snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_ALERT_ENTRIES, wc->uuid_str);

901 -

902 - sqlite3_stmt *res = NULL;

903 -

904 - if (!PREPARE_STATEMENT(db_meta, sql, &res))

905 - return;

906 -

907 - int param = 0;

908 - SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, MAX_REMOVED_PERIOD));

909 -

910 - param = 0;

911 - int rc = sqlite3_step_monitored(res);

912 - if (rc != SQLITE_DONE)

913 - error_report("Failed to execute DELETE query for cleaning stale ACLK alert entries.");

986 + nd_log(

987 + NDLS_ACCESS,

988 + NDLP_DEBUG,

989 + "ACLK REQ [%s (%s)]: Sent! %d alerts snapshot, snapshot_uuid %s (version = %zu)",

990 + wc->node_id,

991 + rrdhost_hostname(host),

992 + cnt,

993 + snapshot_uuid,

994 + version);

995

996 done:

997 REPORT_BIND_FAIL(res, param);

998 SQLITE_FINALIZE(res);

918 -}

919 -

920 -#define SQL_GET_MIN_MAX_ALERT_SEQ "SELECT MIN(sequence_id), MAX(sequence_id), " \

921 - "(SELECT MAX(sequence_id) FROM aclk_alert_%s WHERE date_submitted IS NOT NULL) " \

922 - "FROM aclk_alert_%s WHERE date_submitted IS NULL"

923 -int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert_status)

924 -{

925 -

926 - struct aclk_sync_cfg_t *wc = host->aclk_config;

927 - if (!wc)

928 - return 1;

929 -

930 - proto_alert_status->alert_updates = wc->alert_updates;

931 -

932 - char sql[ACLK_SYNC_QUERY_SIZE];

933 - snprintfz(sql, sizeof(sql) - 1, SQL_GET_MIN_MAX_ALERT_SEQ, wc->uuid_str, wc->uuid_str);

934 -

935 - sqlite3_stmt *res = NULL;

936 - if (!PREPARE_STATEMENT(db_meta, sql, &res))

937 - return 1;

938 -

939 - while (sqlite3_step_monitored(res) == SQLITE_ROW) {

940 - proto_alert_status->pending_min_sequence_id =

941 - sqlite3_column_bytes(res, 0) > 0 ? (uint64_t)sqlite3_column_int64(res, 0) : 0;

942 - proto_alert_status->pending_max_sequence_id =

943 - sqlite3_column_bytes(res, 1) > 0 ? (uint64_t)sqlite3_column_int64(res, 1) : 0;

944 - proto_alert_status->last_submitted_sequence_id =

945 - sqlite3_column_bytes(res, 2) > 0 ? (uint64_t)sqlite3_column_int64(res, 2) : 0;

946 - }

947 -

948 - SQLITE_FINALIZE(res);

999

950 - return 0;

1000 + freez(claim_id);

1001 }

1002

953 -void aclk_send_alarm_checkpoint(char *node_id, char *claim_id __maybe_unused)

1003 +// Start streaming alerts

1004 +void aclk_start_alert_streaming(char *node_id, uint64_t cloud_version)

1005 {

955 - if (unlikely(!node_id))

1006 + nd_uuid_t node_uuid;

1007 +

1008 + if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))

1009 return;

1010

1011 struct aclk_sync_cfg_t *wc;

1012 RRDHOST *host = find_host_by_node_id(node_id);

1013

961 - if (unlikely(!host || !(wc = host->aclk_config)))

962 - nd_log(NDLS_ACCESS, NDLP_WARNING, "ACLK REQ [%s (N/A)]: ALERTS CHECKPOINT REQUEST RECEIVED FOR INVALID NODE", node_id);

963 - else {

964 - nd_log(NDLS_ACCESS, NDLP_DEBUG, "ACLK REQ [%s (%s)]: ALERTS CHECKPOINT REQUEST RECEIVED", node_id, rrdhost_hostname(host));

965 - wc->alert_checkpoint_req = SEND_CHECKPOINT_AFTER_HEALTH_LOOPS;

966 - }

967 -}

968 -

969 -typedef struct active_alerts {

970 - char *name;

971 - char *chart;

972 - RRDCALC_STATUS status;

973 -} active_alerts_t;

974 -

975 -static inline int compare_active_alerts(const void *a, const void *b)

976 -{

977 - active_alerts_t *active_alerts_a = (active_alerts_t *)a;

978 - active_alerts_t *active_alerts_b = (active_alerts_t *)b;

979 -

980 - if (!(strcmp(active_alerts_a->name, active_alerts_b->name))) {

981 - return strcmp(active_alerts_a->chart, active_alerts_b->chart);

982 - } else

983 - return strcmp(active_alerts_a->name, active_alerts_b->name);

984 -}

985 -

986 -#define BATCH_ALLOCATED 10

987 -void aclk_push_alarm_checkpoint(RRDHOST *host __maybe_unused)

988 -{

989 -#ifdef ENABLE_ACLK

990 - struct aclk_sync_cfg_t *wc = host->aclk_config;

991 - if (unlikely(!wc)) {

992 - nd_log(NDLS_ACCESS, NDLP_WARNING, "ACLK REQ [%s (N/A)]: ALERTS CHECKPOINT REQUEST RECEIVED FOR INVALID NODE", rrdhost_hostname(host));

1014 + if (unlikely(!host || !(wc = host->aclk_config))) {

1015 + nd_log(NDLS_ACCESS, NDLP_NOTICE, "ACLK STA [%s (N/A)]: Ignoring request to stream alert state changes, invalid node.", node_id);

1016 return;

1017 }

1018

996 - if (rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS)) {

997 - //postpone checkpoint send

998 - wc->alert_checkpoint_req += 3;

999 - nd_log(NDLS_ACCESS, NDLP_NOTICE, "ACLK REQ [%s (N/A)]: ALERTS CHECKPOINT POSTPONED", rrdhost_hostname(host));

1019 + if (unlikely(!host->health.health_enabled)) {

1020 + nd_log(NDLS_ACCESS, NDLP_NOTICE, "ACLK STA [%s (N/A)]: Ignoring request to stream alert state changes, health is disabled.", node_id);

1021 return;

1022 }

1023

1003 - RRDCALC *rc;

1004 - uint32_t cnt = 0;

1005 - size_t len = 0;

1006 -

1007 - active_alerts_t *active_alerts = callocz(BATCH_ALLOCATED, sizeof(active_alerts_t));

1008 - foreach_rrdcalc_in_rrdhost_read(host, rc) {

1009 - if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))

1010 - continue;

1024 + nd_log(NDLS_ACCESS, NDLP_DEBUG, "ACLK REQ [%s (%s)]: STREAM ALERTS ENABLED", node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");

1025 + schedule_alert_snapshot_if_needed(wc, cloud_version);

1026 + wc->stream_alerts = true;

1027 +}

1028

1012 - if (rc->status == RRDCALC_STATUS_WARNING ||

1013 - rc->status == RRDCALC_STATUS_CRITICAL) {

1029 +// Do checkpoint alert version check

1030 +void aclk_alert_version_check(char *node_id, char *claim_id, uint64_t cloud_version)

1031 +{

1032 + nd_uuid_t node_uuid;

1033

1015 - if (cnt && !(cnt % BATCH_ALLOCATED)) {

1016 - active_alerts = reallocz(active_alerts, (BATCH_ALLOCATED * ((cnt / BATCH_ALLOCATED) + 1)) * sizeof(active_alerts_t));

1017 - }

1034 + if (unlikely(!node_id || !claim_id || !claimed() || uuid_parse(node_id, node_uuid)))

1035 + return;

1036

1019 - active_alerts[cnt].name = (char *)rrdcalc_name(rc);

1020 - len += string_strlen(rc->config.name);

1021 - active_alerts[cnt].chart = (char *)rrdcalc_chart_name(rc);

1022 - len += string_strlen(rc->chart);

1023 - active_alerts[cnt].status = rc->status;

1024 - len++;

1025 - cnt++;

1026 - }

1027 - }

1028 - foreach_rrdcalc_in_rrdhost_done(rc);

1029 -

1030 - BUFFER *alarms_to_hash;

1031 - if (cnt) {

1032 - qsort(active_alerts, cnt, sizeof(active_alerts_t), compare_active_alerts);

1033 -

1034 - alarms_to_hash = buffer_create(len, NULL);

1035 - for (uint32_t i = 0; i < cnt; i++) {

1036 - buffer_strcat(alarms_to_hash, active_alerts[i].name);

1037 - buffer_strcat(alarms_to_hash, active_alerts[i].chart);

1038 - if (active_alerts[i].status == RRDCALC_STATUS_WARNING)

1039 - buffer_fast_strcat(alarms_to_hash, "W", 1);

1040 - else if (active_alerts[i].status == RRDCALC_STATUS_CRITICAL)

1041 - buffer_fast_strcat(alarms_to_hash, "C", 1);

1042 - }

1043 - } else {

1044 - alarms_to_hash = buffer_create(1, NULL);

1045 - buffer_strcat(alarms_to_hash, "");

1046 - len = 0;

1037 + char *agent_claim_id = get_agent_claimid();

1038 + if (claim_id && agent_claim_id && strcmp(agent_claim_id, claim_id) != 0) {

1039 + nd_log(NDLS_ACCESS, NDLP_NOTICE, "ACLK REQ [%s (N/A)]: ALERTS CHECKPOINT VALIDATION REQUEST RECEIVED WITH INVALID CLAIM ID", node_id);

1040 + goto done;

1041 }

1048 - freez(active_alerts);

1042

1050 - char hash[SHA256_DIGEST_LENGTH + 1];

1051 - if (hash256_string((const unsigned char *)buffer_tostring(alarms_to_hash), len, hash)) {

1052 - hash[SHA256_DIGEST_LENGTH] = 0;

1043 + struct aclk_sync_cfg_t *wc;

1044 + RRDHOST *host = find_host_by_node_id(node_id);

1045

1054 - struct alarm_checkpoint alarm_checkpoint;

1055 - char *claim_id = get_agent_claimid();

1056 - alarm_checkpoint.claim_id = claim_id;

1057 - alarm_checkpoint.node_id = wc->node_id;

1058 - alarm_checkpoint.checksum = (char *)hash;

1046 + if ((!host || !(wc = host->aclk_config)))

1047 + nd_log(NDLS_ACCESS, NDLP_NOTICE, "ACLK REQ [%s (N/A)]: ALERTS CHECKPOINT VALIDATION REQUEST RECEIVED FOR INVALID NODE", node_id);

1048 + else

1049 + schedule_alert_snapshot_if_needed(wc, cloud_version);

1050

1060 - aclk_send_provide_alarm_checkpoint(&alarm_checkpoint);

1061 - freez(claim_id);

1062 - nd_log(NDLS_ACCESS, NDLP_DEBUG, "ACLK RES [%s (%s)]: ALERTS CHECKPOINT SENT", wc->node_id, rrdhost_hostname(host));

1063 - } else

1064 - nd_log(NDLS_ACCESS, NDLP_ERR, "ACLK RES [%s (%s)]: FAILED TO CREATE ALERTS CHECKPOINT HASH", wc->node_id, rrdhost_hostname(host));

1051 +done:

1052 + freez(agent_claim_id);

1053 +}

1054

1066 - wc->alert_checkpoint_req = 0;

1067 - buffer_free(alarms_to_hash);

1055 #endif

1069 -}

src/database/sqlite/sqlite_aclk_alert.h

+5 -19

@@ -5,28 +5,14 @@

6 extern sqlite3 *db_meta;

8 -#define SEND_REMOVED_AFTER_HEALTH_LOOPS 3

9 -#define SEND_CHECKPOINT_AFTER_HEALTH_LOOPS 4

10 -

11 -struct proto_alert_status {

12 - int alert_updates;

13 - uint64_t pending_min_sequence_id;

14 - uint64_t pending_max_sequence_id;

15 - uint64_t last_submitted_sequence_id;

16 -};

17 -

18 -void aclk_send_alarm_configuration (char *config_hash);

8 +void aclk_send_alert_configuration(char *config_hash);

9 void aclk_push_alert_config_event(char *node_id, char *config_hash);

20 -void aclk_start_alert_streaming(char *node_id, bool resets);

21 -void sql_queue_removed_alerts_to_aclk(RRDHOST *host);

22 -void sql_process_queue_removed_alerts_to_aclk(char *node_id);

23 -void aclk_send_alarm_checkpoint(char *node_id, char *claim_id);

24 -void aclk_push_alarm_checkpoint(RRDHOST *host);

10 +void aclk_start_alert_streaming(char *node_id, uint64_t cloud_version);

11 +void aclk_alert_version_check(char *node_id, char *claim_id, uint64_t cloud_version);

26 -void aclk_push_alert_snapshot_event(char *node_id);

13 +void send_alert_snapshot_to_cloud(RRDHOST *host __maybe_unused);

14 void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, char *snapshot_uuid);

28 -int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert_status);

29 -void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, bool skip_filter);

15 +bool process_alert_pending_queue(RRDHOST *host);

16 void aclk_push_alert_events_for_all_hosts(void);

18 #endif //NETDATA_SQLITE_ACLK_ALERT_H

src/database/sqlite/sqlite_health.c

+180 -81

@@ -55,15 +55,137 @@ done:

55 }

57 /* Health related SQL queries

58 - Inserts an entry in the table

58 + *

59 + * Inserts an entry in the tables

60 + * alert_queue

61 + * health_log

62 + * health_log_detail

63 + *

64 */

66 +int calculate_delay(RRDCALC_STATUS old_status, RRDCALC_STATUS new_status)

67 +{

68 + int delay = ALERT_TRANSITION_DELAY_NONE;

69 + switch(old_status) {

70 + case RRDCALC_STATUS_REMOVED:

71 + switch (new_status) {

72 + case RRDCALC_STATUS_UNINITIALIZED:

73 + delay = ALERT_TRANSITION_DELAY_LONG;

74 + break;

75 + case RRDCALC_STATUS_CLEAR:

76 + delay = ALERT_TRANSITION_DELAY_SHORT;

77 + break;

78 + default:

79 + delay = ALERT_TRANSITION_DELAY_NONE;

80 + break;

81 + }

82 + break;

83 + case RRDCALC_STATUS_UNDEFINED:

84 + case RRDCALC_STATUS_UNINITIALIZED:

85 + switch (new_status) {

86 + case RRDCALC_STATUS_REMOVED:

87 + case RRDCALC_STATUS_UNINITIALIZED:

88 + case RRDCALC_STATUS_UNDEFINED:

89 + delay = ALERT_TRANSITION_DELAY_LONG;

90 + break;

91 + case RRDCALC_STATUS_CLEAR:

92 + delay = ALERT_TRANSITION_DELAY_SHORT;

93 + break;

94 + default:

95 + delay = ALERT_TRANSITION_DELAY_NONE;

96 + break;

97 + }

98 + break;

99 + case RRDCALC_STATUS_CLEAR:

100 + switch (new_status) {

101 + case RRDCALC_STATUS_REMOVED:

102 + case RRDCALC_STATUS_UNINITIALIZED:

103 + case RRDCALC_STATUS_UNDEFINED:

104 + delay = ALERT_TRANSITION_DELAY_LONG;

105 + break;

106 + case RRDCALC_STATUS_WARNING:

107 + case RRDCALC_STATUS_CRITICAL:

108 + default:

109 + delay = ALERT_TRANSITION_DELAY_NONE;

110 + break;

111 +

112 + }

113 + break;

114 + case RRDCALC_STATUS_WARNING:

115 + case RRDCALC_STATUS_CRITICAL:

116 + switch (new_status) {

117 + case RRDCALC_STATUS_REMOVED:

118 + case RRDCALC_STATUS_UNINITIALIZED:

119 + case RRDCALC_STATUS_UNDEFINED:

120 + delay = ALERT_TRANSITION_DELAY_LONG;

121 + break;

122 + case RRDCALC_STATUS_CLEAR:

123 + delay = ALERT_TRANSITION_DELAY_SHORT;

124 + break;

125 + default:

126 + delay = ALERT_TRANSITION_DELAY_NONE;

127 + break;

128 + }

129 + break;

130 + default:

131 + delay = ALERT_TRANSITION_DELAY_NONE;

132 + break;

133 + }

134 + return delay;

135 +}

136 +

137 +#ifdef ENABLE_ACLK

138 +#define SQL_INSERT_ALERT_PENDING_QUEUE \

139 + "INSERT INTO alert_queue (host_id, health_log_id, unique_id, alarm_id, status, date_scheduled)" \

140 + " VALUES (@host_id, @health_log_id, @unique_id, @alarm_id, @new_status, UNIXEPOCH() + @delay)" \

141 + " ON CONFLICT (host_id, health_log_id, alarm_id)" \

142 + " DO UPDATE SET status = excluded.status, unique_id = excluded.unique_id, " \

143 + " date_scheduled = MIN(date_scheduled, excluded.date_scheduled)"

144 +

145 +static void insert_alert_queue(

146 + RRDHOST *host,

147 + uint64_t health_log_id,

148 + int64_t unique_id,

149 + uint32_t alarm_id,

150 + RRDCALC_STATUS old_status,

151 + RRDCALC_STATUS new_status)

152 +{

153 + static __thread sqlite3_stmt *res = NULL;

154 + int rc;

155 +

156 + if (!host->aclk_config)

157 + return;

158 +

159 + if (!PREPARE_COMPILED_STATEMENT(db_meta, SQL_INSERT_ALERT_PENDING_QUEUE, &res))

160 + return;

161 +

162 + int submit_delay = calculate_delay(old_status, new_status);

163 +

164 + int param = 0;

165 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

166 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (sqlite3_int64)health_log_id));

167 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, unique_id));

168 + SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, alarm_id));

169 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(res, ++param, new_status));

170 + SQLITE_BIND_FAIL(done, sqlite3_bind_int(res, ++param, submit_delay));

171 +

172 + param = 0;

173 + rc = execute_insert(res);

174 + if (rc != SQLITE_DONE)

175 + error_report(

176 + "HEALTH [%s]: Failed to execute insert_alert_queue, rc = %d", rrdhost_hostname(host), rc);

177 +

178 +done:

179 + REPORT_BIND_FAIL(res, param);

180 + SQLITE_RESET(res);

181 +}

182 +#endif

183

184 #define SQL_INSERT_HEALTH_LOG_DETAIL \

185 "INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, " \

186 "updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, " \

187 "info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, transition_id, global_id, summary) " \

66 - "VALUES (@health_log_id,@unique_id,@alarm_id,@alarm_event_id,@updated_by_id,@updates_id,@when_key,@duration," \

188 + " VALUES (@health_log_id,@unique_id,@alarm_id,@alarm_event_id,@updated_by_id,@updates_id,@when_key,@duration," \

189 "@non_clear_duration,@flags,@exec_run_timestamp,@delay_up_to_timestamp, @info,@exec_code,@new_status,@old_status," \

190 "@delay,@new_value,@old_value,@last_repeat,@transition_id,@global_id,@summary)"

191

@@ -150,6 +272,11 @@ static void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae)

272 if (rc == SQLITE_ROW) {

273 health_log_id = (size_t)sqlite3_column_int64(res, 0);

274 sql_health_alarm_log_insert_detail(host, health_log_id, ae);

275 +#ifdef ENABLE_ACLK

276 + if (netdata_cloud_enabled)

277 + insert_alert_queue(

278 + host, health_log_id, (int64_t)ae->unique_id, (int64_t)ae->alarm_id, ae->old_status, ae->new_status);

279 +#endif

280 } else

281 error_report("HEALTH [%s]: Failed to execute SQL_INSERT_HEALTH_LOG, rc = %d", rrdhost_hostname(host), rc);

282

@@ -162,14 +289,8 @@ void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae)

289 {

290 if (ae->flags & HEALTH_ENTRY_FLAG_SAVED)

291 sql_health_alarm_log_update(host, ae);

165 - else {

292 + else

293 sql_health_alarm_log_insert(host, ae);

167 -#ifdef ENABLE_ACLK

168 - if (netdata_cloud_enabled) {

169 - sql_queue_alarm_to_aclk(host, ae, false);

170 - }

171 -#endif

172 - }

294 }

295

296 /*

@@ -179,44 +300,18 @@ void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae)

300 *

301 */

302

182 -#define SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED \

303 +#define SQL_CLEANUP_HEALTH_LOG_DETAIL \

304 "DELETE FROM health_log_detail WHERE health_log_id IN " \

184 - "(SELECT health_log_id FROM health_log WHERE host_id = @host_id) AND when_key < UNIXEPOCH() - @history " \

185 - "AND updated_by_id <> 0 AND transition_id NOT IN " \

186 - "(SELECT last_transition_id FROM health_log hl WHERE hl.host_id = @host_id)"

187 -

188 -#define SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(guid) \

189 - "DELETE from health_log_detail WHERE unique_id NOT IN " \

190 - "(SELECT filtered_alert_unique_id FROM aclk_alert_%s) " \

191 - "AND unique_id IN (SELECT hld.unique_id FROM health_log hl, health_log_detail hld WHERE " \

192 - "hl.host_id = @host_id AND hl.health_log_id = hld.health_log_id) " \

193 - "AND health_log_id IN (SELECT health_log_id FROM health_log WHERE host_id = @host_id) " \

194 - "AND when_key < unixepoch() - @history " \

195 - "AND updated_by_id <> 0 AND transition_id NOT IN " \

196 - "(SELECT last_transition_id FROM health_log hl WHERE hl.host_id = @host_id)", \

197 - guid

198 -

199 -void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed) {

305 + " (SELECT health_log_id FROM health_log WHERE host_id = @host_id) AND when_key < UNIXEPOCH() - @history " \

306 + " AND updated_by_id <> 0 AND transition_id NOT IN " \

307 + " (SELECT last_transition_id FROM health_log hl WHERE hl.host_id = @host_id)"

308 +

309 +void sql_health_alarm_log_cleanup(RRDHOST *host)

310 +{

311 sqlite3_stmt *res = NULL;

312 int rc;

202 - char command[MAX_HEALTH_SQL_SIZE + 1];

203 -

204 - REQUIRE_DB(db_meta);

205 -

206 - char uuid_str[UUID_STR_LEN];

207 - uuid_unparse_lower_fix(&host->host_uuid, uuid_str);

208 - snprintfz(command, sizeof(command) - 1, "aclk_alert_%s", uuid_str);

313

210 - bool aclk_table_exists = table_exists_in_database(db_meta, command);

211 -

212 - char *sql = SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED;

213 -

214 - if (claimed && aclk_table_exists) {

215 - snprintfz(command, sizeof(command) - 1, SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(uuid_str));

216 - sql = command;

217 - }

218 -

219 - if (!PREPARE_STATEMENT(db_meta, sql, &res))

314 + if (!PREPARE_STATEMENT(db_meta, SQL_CLEANUP_HEALTH_LOG_DETAIL, &res))

315 return;

316

317 int param = 0;

@@ -228,33 +323,21 @@ void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed) {

323 if (unlikely(rc != SQLITE_DONE))

324 error_report("Failed to cleanup health log detail table, rc = %d", rc);

325

231 - if (aclk_table_exists)

232 - sql_aclk_alert_clean_dead_entries(host);

233 -

326 done:

327 REPORT_BIND_FAIL(res, param);

328 SQLITE_FINALIZE(res);

329 }

330

239 -#define SQL_INJECT_REMOVED \

240 - "INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, updated_by_id, updates_id, when_key, " \

241 - "duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, info, exec_code, new_status, old_status, " \

242 - "delay, new_value, old_value, last_repeat, transition_id, global_id, summary) " \

243 - "SELECT health_log_id, ?1, ?2, ?3, 0, ?4, UNIXEPOCH(), 0, 0, flags, exec_run_timestamp, UNIXEPOCH(), info, exec_code, -2, " \

244 - "new_status, delay, NULL, new_value, 0, ?5, NOW_USEC(0), summary FROM health_log_detail WHERE unique_id = ?6 AND transition_id = ?7"

245 -

246 -#define SQL_INJECT_REMOVED_UPDATE_DETAIL \

247 - "UPDATE health_log_detail SET flags = flags | ?1, updated_by_id = ?2 WHERE unique_id = ?3 AND transition_id = ?4"

248 -

249 -#define SQL_INJECT_REMOVED_UPDATE_LOG \

250 - "UPDATE health_log SET last_transition_id = ?1 WHERE alarm_id = ?2 AND last_transition_id = ?3 AND host_id = ?4"

331 +#define SQL_UPDATE_TRANSITION_IN_HEALTH_LOG \

332 + "UPDATE health_log SET last_transition_id = @transition WHERE alarm_id = @alarm_id AND " \

333 + " last_transition_id = @prev_trans AND host_id = @host_id"

334

252 -bool sql_update_removed_in_health_log(RRDHOST *host, uint32_t alarm_id, nd_uuid_t *transition_id, nd_uuid_t *last_transition)

335 +bool sql_update_transition_in_health_log(RRDHOST *host, uint32_t alarm_id, nd_uuid_t *transition_id, nd_uuid_t *last_transition)

336 {

337 int rc = 0;

338 sqlite3_stmt *res;

339

257 - if (!PREPARE_STATEMENT(db_meta, SQL_INJECT_REMOVED_UPDATE_LOG, &res))

340 + if (!PREPARE_STATEMENT(db_meta, SQL_UPDATE_TRANSITION_IN_HEALTH_LOG, &res))

341 return false;

342

343 int param = 0;

@@ -275,12 +358,16 @@ done:

358 return (param == 0 && rc == SQLITE_DONE);

359 }

360

278 -bool sql_update_removed_in_health_log_detail(uint32_t unique_id, uint32_t max_unique_id, nd_uuid_t *prev_transition_id)

361 +#define SQL_SET_UPDATED_BY_IN_HEALTH_LOG_DETAIL \

362 + "UPDATE health_log_detail SET flags = flags | @flag, updated_by_id = @updated_by WHERE" \

363 + " unique_id = @unique_id AND transition_id = @transition_id"

364 +

365 +bool sql_set_updated_by_in_health_log_detail(uint32_t unique_id, uint32_t max_unique_id, nd_uuid_t *prev_transition_id)

366 {

367 int rc = 0;

368 sqlite3_stmt *res;

369

283 - if (!PREPARE_STATEMENT(db_meta, SQL_INJECT_REMOVED_UPDATE_DETAIL, &res))

370 + if (!PREPARE_STATEMENT(db_meta, SQL_SET_UPDATED_BY_IN_HEALTH_LOG_DETAIL, &res))

371 return false;

372

373 int param = 0;

@@ -301,7 +388,16 @@ done:

388 return (param == 0 && rc == SQLITE_DONE);

389 }

390

304 -void sql_inject_removed_status(

391 +#define SQL_INJECT_REMOVED \

392 + "INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, updated_by_id, updates_id, when_key, " \

393 + "duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, info, exec_code, new_status, old_status, " \

394 + "delay, new_value, old_value, last_repeat, transition_id, global_id, summary) " \

395 + "SELECT health_log_id, @max_unique_id, @alarm_id, @alarm_event_id, 0, @unique_id, UNIXEPOCH(), 0, 0, flags, " \

396 + " exec_run_timestamp, UNIXEPOCH(), info, exec_code, -2, " \

397 + " new_status, delay, NULL, new_value, 0, @transition_id, NOW_USEC(0), summary FROM health_log_detail " \

398 + " WHERE unique_id = @unique_id AND transition_id = @last_transition_id RETURNING health_log_id, old_status"

399 +

400 +static void sql_inject_removed_status(

401 RRDHOST *host,

402 uint32_t alarm_id,

403 uint32_t alarm_event_id,

@@ -326,19 +422,27 @@ void sql_inject_removed_status(

422 SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (sqlite3_int64) alarm_event_id + 1));

423 SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (sqlite3_int64) unique_id));

424 SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &transition_id, sizeof(transition_id), SQLITE_STATIC));

329 - SQLITE_BIND_FAIL(done, sqlite3_bind_int64(res, ++param, (sqlite3_int64) unique_id));

425 SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, last_transition, sizeof(*last_transition), SQLITE_STATIC));

426

427 param = 0;

333 - int rc = execute_insert(res);

334 - if (rc == SQLITE_DONE) {

428 + //int rc = execute_insert(res);

429 + while (sqlite3_step_monitored(res) == SQLITE_ROW) {

430 //update the old entry in health_log_detail

336 - sql_update_removed_in_health_log_detail(unique_id, max_unique_id, last_transition);

431 + sql_set_updated_by_in_health_log_detail(unique_id, max_unique_id, last_transition);

432 //update the old entry in health_log

338 - sql_update_removed_in_health_log(host, alarm_id, &transition_id, last_transition);

433 + sql_update_transition_in_health_log(host, alarm_id, &transition_id, last_transition);

434 +

435 +#ifdef ENABLE_ACLK

436 + if (netdata_cloud_enabled) {

437 + int64_t health_log_id = sqlite3_column_int64(res, 0);

438 + RRDCALC_STATUS old_status = (RRDCALC_STATUS)sqlite3_column_double(res, 1);

439 + insert_alert_queue(

440 + host, health_log_id, (int64_t)unique_id, (int64_t)alarm_id, old_status, RRDCALC_STATUS_REMOVED);

441 + }

442 +#endif

443 }

340 - else

341 - error_report("HEALTH [N/A]: Failed to execute SQL_INJECT_REMOVED, rc = %d", rc);

444 + //else

445 + // error_report("HEALTH [N/A]: Failed to execute SQL_INJECT_REMOVED, rc = %d", rc);

446

447 done:

448 REPORT_BIND_FAIL(res, param);

@@ -489,7 +593,6 @@ void sql_alert_cleanup(bool cli)

593 void sql_health_alarm_log_load(RRDHOST *host)

594 {

595 sqlite3_stmt *res = NULL;

492 - int ret;

596 ssize_t errored = 0, loaded = 0;

597

598 if (!REQUIRE_DB(db_meta))

@@ -500,21 +603,19 @@ void sql_health_alarm_log_load(RRDHOST *host)

603 if (!PREPARE_STATEMENT(db_meta, SQL_LOAD_HEALTH_LOG, &res))

604 return;

605

503 - ret = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);

504 - if (unlikely(ret != SQLITE_OK)) {

505 - error_report("Failed to bind host_id parameter for SQL_LOAD_HEALTH_LOG.");

506 - SQLITE_FINALIZE(res);

507 - return;

508 - }

606 + int param = 0;

607 + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC));

608

609 DICTIONARY *all_rrdcalcs = dictionary_create(

610 DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE);

611 +

612 RRDCALC *rc;

613 foreach_rrdcalc_in_rrdhost_read(host, rc) {

614 dictionary_set(all_rrdcalcs, rrdcalc_name(rc), rc, sizeof(*rc));

615 }

616 foreach_rrdcalc_in_rrdhost_done(rc);

617

618 + param = 0;

619 rw_spinlock_read_lock(&host->health_log.spinlock);

620

621 while (sqlite3_step_monitored(res) == SQLITE_ROW) {

@@ -618,8 +719,6 @@ void sql_health_alarm_log_load(RRDHOST *host)

719 ae->summary = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 33);

720

721 char value_string[100 + 1];

621 - string_freez(ae->old_value_string);

622 - string_freez(ae->new_value_string);

722 ae->old_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae_units(ae), -1));

723 ae->new_value_string = string_strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae_units(ae), -1));

724

@@ -631,7 +730,6 @@ void sql_health_alarm_log_load(RRDHOST *host)

730

731 if(unlikely(ae->alarm_id >= host->health_max_alarm_id))

732 host->health_max_alarm_id = ae->alarm_id;

634 -

733 loaded++;

734 }

735

@@ -650,7 +748,8 @@ void sql_health_alarm_log_load(RRDHOST *host)

748 nd_log(NDLS_DAEMON, errored ? NDLP_WARNING : NDLP_DEBUG,

749 "[%s]: Table health_log, loaded %zd alarm entries, errors in %zd entries.",

750 rrdhost_hostname(host), loaded, errored);

653 -

751 +done:

752 + REPORT_BIND_FAIL(res, param);

753 SQLITE_FINALIZE(res);

754 }

755

src/database/sqlite/sqlite_health.h

+5 -2

@@ -6,14 +6,17 @@

6 #include "daemon/common.h"

7 #include "sqlite3.h"

9 +#define ALERT_TRANSITION_DELAY_LONG (600)

10 +#define ALERT_TRANSITION_DELAY_SHORT (10)

11 +#define ALERT_TRANSITION_DELAY_NONE (0)

12 +

13 struct sql_alert_transition_data;

14 struct sql_alert_config_data;

15 struct rrd_alert_prototype;

16 void sql_health_alarm_log_load(RRDHOST *host);

17 void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);

14 -void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed);

18 +void sql_health_alarm_log_cleanup(RRDHOST *host);

19 void sql_alert_store_config(struct rrd_alert_prototype *ap);

16 -void sql_aclk_alert_clean_dead_entries(RRDHOST *host);

20 int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_STATUS *last_executed_status);

21 void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, time_t after, const char *chart);

22 int health_migrate_old_health_log_table(char *table);

src/database/sqlite/sqlite_metadata.c

+17 -3

@@ -67,6 +67,18 @@ const char *database_config[] = {

67 "CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)",

68 "CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)",

70 +#ifdef ENABLE_ACLK

71 + "CREATE TABLE IF NOT EXISTS alert_queue "

72 + " (host_id BLOB, health_log_id INT, unique_id INT, alarm_id INT, status INT, date_scheduled INT, "

73 + " UNIQUE(host_id, health_log_id, alarm_id))",

74 +

75 + "CREATE TABLE IF NOT EXISTS alert_version (health_log_id INTEGER PRIMARY KEY, unique_id INT, status INT, "

76 + "version INT, date_submitted INT)",

77 +

78 + "CREATE TABLE IF NOT EXISTS aclk_queue (sequence_id INTEGER PRIMARY KEY, host_id blob, health_log_id INT, "

79 + "unique_id INT, date_created INT, UNIQUE(host_id, health_log_id))",

80 +#endif

81 +

82 NULL

83 };

@@ -251,7 +263,7 @@ static inline void set_host_node_id(RRDHOST *host, nd_uuid_t *node_id)

263 }

264

265 if (unlikely(!wc))

254 - sql_create_aclk_table(host, &host->host_uuid, node_id);

266 + create_aclk_config(host, &host->host_uuid, node_id);

267 else

268 uuid_unparse_lower(*node_id, wc->node_id);

269 }

@@ -1442,11 +1454,10 @@ static void cleanup_health_log(struct metadata_wc *wc)

1454

1455 RRDHOST *host;

1456

1445 - bool is_claimed = claimed();

1457 dfe_start_reentrant(rrdhost_root_index, host){

1458 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))

1459 continue;

1449 - sql_health_alarm_log_cleanup(host, is_claimed);

1460 + sql_health_alarm_log_cleanup(host);

1461 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))

1462 break;

1463 }

@@ -1457,6 +1468,9 @@ static void cleanup_health_log(struct metadata_wc *wc)

1468

1469 (void) db_execute(db_meta,"DELETE FROM health_log WHERE host_id NOT IN (SELECT host_id FROM host)");

1470 (void) db_execute(db_meta,"DELETE FROM health_log_detail WHERE health_log_id NOT IN (SELECT health_log_id FROM health_log)");

1471 +#ifdef ENABLE_ACLK

1472 + (void) db_execute(db_meta,"DELETE FROM alert_version WHERE health_log_id NOT IN (SELECT health_log_id FROM health_log)");

1473 +#endif

1474 }

1475

1476 //

src/health/health_event_loop.c

+66 -131

@@ -101,26 +101,10 @@ static void health_sleep(time_t next_run, unsigned int loop __maybe_unused) {

101 }

102 }

103

104 -static void sql_health_postpone_queue_removed(RRDHOST *host __maybe_unused) {

105 -#ifdef ENABLE_ACLK

106 - if (netdata_cloud_enabled) {

107 - struct aclk_sync_cfg_t *wc = host->aclk_config;

108 - if (unlikely(!wc)) {

109 - return;

110 - }

111 -

112 - if (wc->alert_queue_removed >= 1) {

113 - wc->alert_queue_removed+=6;

114 - }

115 - }

116 -#endif

117 -}

118 -

104 static void health_execute_delayed_initializations(RRDHOST *host) {

105 health_plugin_init();

106

107 RRDSET *st;

123 - bool must_postpone = false;

108

109 if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION)) return;

110 rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);

@@ -131,11 +115,8 @@ static void health_execute_delayed_initializations(RRDHOST *host) {

115

116 worker_is_busy(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET);

117 health_prototype_alerts_for_rrdset_incrementally(st);

134 - must_postpone = true;

118 }

119 rrdset_foreach_done(st);

137 - if (must_postpone)

138 - sql_health_postpone_queue_removed(host);

120 }

121

122 static void health_initialize_rrdhost(RRDHOST *host) {

@@ -179,6 +160,50 @@ static inline int check_if_resumed_from_suspension(void) {

160 return ret;

161 }

162

163 +static void do_eval_expression(

164 + RRDCALC *rc,

165 + EVAL_EXPRESSION *expression,

166 + const char *expression_type __maybe_unused,

167 + size_t job_type,

168 + RRDCALC_FLAGS error_type,

169 + RRDCALC_STATUS *calc_status,

170 + NETDATA_DOUBLE *result)

171 +{

172 + if (!expression || (!calc_status && !result))

173 + return;

174 +

175 + worker_is_busy(job_type);

176 +

177 + if (unlikely(!expression_evaluate(expression))) {

178 + // calculation failed

179 + rc->run_flags |= error_type;

180 + if (result)

181 + *result = NAN;

182 +

183 + netdata_log_debug(D_HEALTH,

184 + "Health on host '%s', alarm '%s.%s': %s expression failed with error: %s",

185 + rrdhost_hostname(rc->rrdset->rrdhost), rrdcalc_chart_name(rc), rrdcalc_name(rc), expression_type,

186 + expression_error_msg(expression)

187 + );

188 + return;

189 + }

190 + rc->run_flags &= ~error_type;

191 + netdata_log_debug(D_HEALTH,

192 + "Health on host '%s', alarm '%s.%s': %s expression gave value "

193 + NETDATA_DOUBLE_FORMAT ": %s (source: %s)",

194 + rrdhost_hostname(rc->rrdset->rrdhost),

195 + rrdcalc_chart_name(rc),

196 + rrdcalc_name(rc),

197 + expression_type,

198 + expression_result(expression),

199 + expression_error_msg(expression),

200 + rrdcalc_source(rc));

201 + if (calc_status)

202 + *calc_status = rrdcalc_value2status(expression_result(expression));

203 + else

204 + *result = expression_result(expression);

205 +}

206 +

207 static void health_event_loop(void) {

208 bool health_running_logged = false;

209

@@ -270,6 +295,15 @@ static void health_event_loop(void) {

295 }

296

297 worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);

298 +#ifdef ENABLE_ACLK

299 + if (netdata_cloud_enabled) {

300 + struct aclk_sync_cfg_t *wc = host->aclk_config;

301 + if (wc && wc->send_snapshot == 2) {

302 + netdata_log_info("DEBUG: %s SKIPPING HEALTH BECAUSE WE WILL SEND SNAPSHOT", rrdhost_hostname(host));

303 + continue;

304 + }

305 + }

306 +#endif

307

308 // the first loop is to lookup values from the db

309 foreach_rrdcalc_in_rrdhost_read(host, rc) {

@@ -314,11 +348,6 @@ static void health_event_loop(void) {

348 rc->last_status_change_value = rc->value;

349 rc->last_updated = now_tmp;

350 rc->value = NAN;

317 -

318 -#ifdef ENABLE_ACLK

319 - if (netdata_cloud_enabled)

320 - sql_queue_alarm_to_aclk(host, ae, true);

321 -#endif

351 }

352 }

353 }

@@ -404,36 +433,7 @@ static void health_event_loop(void) {

433 // ------------------------------------------------------------

434 // if there is calculation expression, run it

435

407 - if (unlikely(rc->config.calculation)) {

408 - worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);

409 -

410 - if (unlikely(!expression_evaluate(rc->config.calculation))) {

411 - // calculation failed

412 - rc->value = NAN;

413 - rc->run_flags |= RRDCALC_FLAG_CALC_ERROR;

414 -

415 - netdata_log_debug(

416 - D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",

417 - rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),

418 - expression_parsed_as(rc->config.calculation), expression_error_msg(rc->config.calculation)

419 - );

420 - }

421 - else {

422 - rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR;

423 -

424 - netdata_log_debug(

425 - D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "

426 - NETDATA_DOUBLE_FORMAT": %s (source: %s)",

427 - rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),

428 - expression_parsed_as(rc->config.calculation),

429 - expression_result(rc->config.calculation),

430 - expression_error_msg(rc->config.calculation),

431 - rrdcalc_source(rc)

432 - );

433 -

434 - rc->value = expression_result(rc->config.calculation);

435 - }

436 - }

436 + do_eval_expression(rc, rc->config.calculation, "calculation", WORKER_HEALTH_JOB_CALC_EVAL, RRDCALC_FLAG_CALC_ERROR, NULL, &rc->value);

437 }

438 foreach_rrdcalc_in_rrdhost_done(rc);

439

@@ -453,65 +453,8 @@ static void health_event_loop(void) {

453 RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;

454 RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;

455

456 - // --------------------------------------------------------

457 - // check the warning expression

458 -

459 - if (likely(rc->config.warning)) {

460 - worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);

461 -

462 - if (unlikely(!expression_evaluate(rc->config.warning))) {

463 - // calculation failed

464 - rc->run_flags |= RRDCALC_FLAG_WARN_ERROR;

465 -

466 - netdata_log_debug(D_HEALTH,

467 - "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",

468 - rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),

469 - expression_error_msg(rc->config.warning)

470 - );

471 - } else {

472 - rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR;

473 - netdata_log_debug(D_HEALTH,

474 - "Health on host '%s', alarm '%s.%s': warning expression gave value "

475 - NETDATA_DOUBLE_FORMAT ": %s (source: %s)",

476 - rrdhost_hostname(host),

477 - rrdcalc_chart_name(rc),

478 - rrdcalc_name(rc),

479 - expression_result(rc->config.warning),

480 - expression_error_msg(rc->config.warning),

481 - rrdcalc_source(rc)

482 - );

483 - warning_status = rrdcalc_value2status(expression_result(rc->config.warning));

484 - }

485 - }

486 -

487 - // --------------------------------------------------------

488 - // check the critical expression

489 -

490 - if (likely(rc->config.critical)) {

491 - worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);

492 -

493 - if (unlikely(!expression_evaluate(rc->config.critical))) {

494 - // calculation failed

495 - rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR;

496 -

497 - netdata_log_debug(D_HEALTH,

498 - "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",

499 - rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),

500 - expression_error_msg(rc->config.critical)

501 - );

502 - } else {

503 - rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR;

504 - netdata_log_debug(D_HEALTH,

505 - "Health on host '%s', alarm '%s.%s': critical expression gave value "

506 - NETDATA_DOUBLE_FORMAT ": %s (source: %s)",

507 - rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),

508 - expression_result(rc->config.critical),

509 - expression_error_msg(rc->config.critical),

510 - rrdcalc_source(rc)

511 - );

512 - critical_status = rrdcalc_value2status(expression_result(rc->config.critical));

513 - }

514 - }

456 + do_eval_expression(rc, rc->config.warning, "warning", WORKER_HEALTH_JOB_WARNING_EVAL, RRDCALC_FLAG_WARN_ERROR, &warning_status, NULL);

457 + do_eval_expression(rc, rc->config.critical, "critical", WORKER_HEALTH_JOB_CRITICAL_EVAL, RRDCALC_FLAG_CRIT_ERROR, &critical_status, NULL);

458

459 // --------------------------------------------------------

460 // decide the final alarm status

@@ -706,26 +649,18 @@ static void health_event_loop(void) {

649 wait_for_all_notifications_to_finish_before_allowing_health_to_be_cleaned_up();

650 break;

651 }

652 + }

653 #ifdef ENABLE_ACLK

710 - if (netdata_cloud_enabled) {

711 - struct aclk_sync_cfg_t *wc = host->aclk_config;

712 - if (unlikely(!wc))

713 - continue;

714 -

715 - if (wc->alert_queue_removed == 1) {

716 - sql_queue_removed_alerts_to_aclk(host);

717 - } else if (wc->alert_queue_removed > 1) {

718 - wc->alert_queue_removed--;

719 - }

720 -

721 - if (wc->alert_checkpoint_req == 1) {

722 - aclk_push_alarm_checkpoint(host);

723 - } else if (wc->alert_checkpoint_req > 1) {

724 - wc->alert_checkpoint_req--;

725 - }

726 - }

727 -#endif

654 + struct aclk_sync_cfg_t *wc = host->aclk_config;

655 + if (wc && wc->send_snapshot == 1) {

656 + wc->send_snapshot = 2;

657 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

658 }

659 + else

660 + if (process_alert_pending_queue(host))

661 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);

662 +#endif

663 +

664 dfe_done(host);

665

666 // wait for all notifications to finish before allowing health to be cleaned up

src/health/health_log.c

+2 -1

@@ -4,7 +4,8 @@

5 // ----------------------------------------------------------------------------

7 -inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {

7 +inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae)

8 +{

9 sql_health_alarm_log_save(host, ae);

10 }

src/health/health_prototypes.c

-9

@@ -687,15 +687,6 @@ void health_apply_prototypes_to_host(RRDHOST *host) {

687 health_prototype_reset_alerts_for_rrdset(st);

688 }

689 rrdset_foreach_done(st);

690 -

691 -#ifdef ENABLE_ACLK

692 - if (netdata_cloud_enabled) {

693 - struct aclk_sync_cfg_t *wc = host->aclk_config;

694 - if (likely(wc)) {

695 - wc->alert_queue_removed = SEND_REMOVED_AFTER_HEALTH_LOOPS;

696 - }

697 - }

698 -#endif

690 }

691

692 void health_apply_prototypes_to_all_hosts(void) {

src/web/api/web_api_v1.c

+4 -1

@@ -1422,10 +1422,13 @@ static int web_client_api_request_v1_aclk_state(RRDHOST *host, struct web_client

1422

1423 BUFFER *wb = w->response.data;

1424 buffer_flush(wb);

1425 -

1425 +#ifdef ENABLE_ACLK

1426 char *str = aclk_state_json();

1427 buffer_strcat(wb, str);

1428 freez(str);

1429 +#else

1430 + buffer_strcat(wb, "{\"aclk-available\":false}");

1431 +#endif

1432

1433 wb->content_type = CT_APPLICATION_JSON;

1434 buffer_no_cacheable(wb);