Revert health to run in a single thread (#14244)
* revert health to single thread * remove getting now * use a health struct * remove commented code * cleanup health log from metdata * dont check for METADATA_UPDATE
Emmanuel Vasilakis committed
Jan 18, 2023 at 10:42 UTC
3d5f9e64a012b8d0f66ccf483e0e0e6eb3186787
19 files changed
+554
-563
aclk/aclk.c
+2
@@ -49,6 +49,8 @@ float last_backoff_value = 0;
49
50
time_t aclk_block_until = 0;
51
52
+int aclk_alert_reloaded = 0; //1 on health log exchange, and again on health_reload
53
+
54
#ifdef ENABLE_ACLK
55
mqtt_wss_client mqttwss_client;
56
aclk/aclk.h
+2
@@ -26,6 +26,8 @@ extern time_t aclk_block_until;
26
27
extern int disconnect_req;
28
29
+extern int aclk_alert_reloaded;
30
+
31
#ifdef ENABLE_ACLK
32
void *aclk_main(void *ptr);
33
daemon/service.c
+1
-1
@@ -201,7 +201,7 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
201
&& (
202
(
203
host->child_last_chart_command
204
- && host->child_last_chart_command + host->health_delay_up_to < now_realtime_sec()
204
+ && host->child_last_chart_command + host->health.health_delay_up_to < now_realtime_sec()
205
)
206
|| (host->child_connect_time + TIME_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT < now_realtime_sec())
207
)
daemon/static_threads.c
+9
@@ -36,6 +36,15 @@ const struct netdata_static_thread static_threads_common[] = {
36
.init_routine = NULL,
37
.start_routine = cpuidlejitter_main
38
},
39
+ {
40
+ .name = "HEALTH",
41
+ .config_section = NULL,
42
+ .config_name = NULL,
43
+ .enabled = 1,
44
+ .thread = NULL,
45
+ .init_routine = NULL,
46
+ .start_routine = health_main
47
+ },
48
{
49
.name = "ANALYTICS",
50
.config_section = NULL,
database/rrd.h
+13
-11
@@ -895,6 +895,17 @@ typedef struct alarm_log {
895
netdata_rwlock_t alarm_log_rwlock;
896
} ALARM_LOG;
897
898
+typedef struct health {
899
+ unsigned int health_enabled; // 1 when this host has health enabled
900
+ time_t health_delay_up_to; // a timestamp to delay alarms processing up to
901
+ STRING *health_default_exec; // the full path of the alarms notifications program
902
+ STRING *health_default_recipient; // the default recipient for all alarms
903
+ char *health_log_filename; // the alarms event log filename
904
+ size_t health_log_entries_written; // the number of alarm events written to the alarms event log
905
+ FILE *health_log_fp; // the FILE pointer to the open alarms event log file
906
+ uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications
907
+ uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications
908
+} HEALTH;
909
910
// ----------------------------------------------------------------------------
911
// RRD HOST
@@ -1012,17 +1023,8 @@ struct rrdhost {
1023
// ------------------------------------------------------------------------
1024
// health monitoring options
1025
1015
- unsigned int health_enabled; // 1 when this host has health enabled
1016
- bool health_spawn; // true when health thread is running
1017
- unsigned int aclk_alert_reloaded; // 1 on thread start and health reload, 0 after removed are sent
1018
- time_t health_delay_up_to; // a timestamp to delay alarms processing up to
1019
- STRING *health_default_exec; // the full path of the alarms notifications program
1020
- STRING *health_default_recipient; // the default recipient for all alarms
1021
- char *health_log_filename; // the alarms event log filename
1022
- size_t health_log_entries_written; // the number of alarm events written to the alarms event log
1023
- FILE *health_log_fp; // the FILE pointer to the open alarms event log file
1024
- uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications
1025
- uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications
1026
+ // health variables
1027
+ HEALTH health;
1028
1029
// all RRDCALCs are primarily allocated and linked here
1030
DICTIONARY *rrdcalc_root_index;
database/rrdcalc.c
+2
-2
@@ -739,7 +739,7 @@ void rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(RRDHOST *host
739
continue;
740
741
if(!rrdlabels_match_simple_pattern_parsed(host->rrdlabels, rc->host_labels_pattern, '=')) {
742
- info("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
742
+ log_health("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
743
rrdcalc_name(rc),
744
rrdhost_hostname(host),
745
rrdcalc_host_labels(rc));
@@ -755,7 +755,7 @@ void rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts() {
755
756
RRDHOST *host;
757
rrdhost_foreach_read(host) {
758
- if (unlikely(!host->health_enabled))
758
+ if (unlikely(!host->health.health_enabled))
759
continue;
760
761
if (host->rrdlabels)
database/rrddimvar.c
+2
-2
@@ -65,7 +65,7 @@ static inline void rrddimvar_free_variables_unsafe(RRDDIMVAR *rs) {
65
66
// HOST VARIABLES FOR THIS DIMENSION
67
68
- if(host->rrdvars && host->health_enabled) {
68
+ if(host->rrdvars && host->health.health_enabled) {
69
rrdvar_release_and_del(host->rrdvars, rs->rrdvar_host_chart_id_dim_id);
70
rs->rrdvar_host_chart_id_dim_id = NULL;
71
@@ -152,7 +152,7 @@ static inline void rrddimvar_update_variables_unsafe(RRDDIMVAR *rs) {
152
// - $chart-name.id
153
// - $chart-name.name
154
155
- if(host->rrdvars && host->health_enabled) {
155
+ if(host->rrdvars && host->health.health_enabled) {
156
rs->rrdvar_host_chart_id_dim_id = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_id_dim_id, rs->type, RRDVAR_FLAG_NONE, rs->value);
157
rs->rrdvar_host_chart_id_dim_name = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_id_dim_name, rs->type, RRDVAR_FLAG_NONE, rs->value);
158
rs->rrdvar_host_chart_name_dim_id = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_name_dim_id, rs->type, RRDVAR_FLAG_NONE, rs->value);
database/rrdhost.c
+10
-17
@@ -281,8 +281,8 @@ int is_legacy = 1;
281
282
rrdhost_init_hostname(host, hostname, false);
283
284
- host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);
285
- host->health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled;
284
+ host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);
285
+ host->health.health_enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled;
286
287
if (likely(!archived)) {
288
rrdfunctions_init(host);
@@ -366,9 +366,6 @@ int is_legacy = 1;
366
rrdcalc_rrdhost_index_init(host);
367
metaqueue_host_update_info(host);
368
369
- if (health_enabled)
370
- health_thread_spawn(host);
371
-
369
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
370
#ifdef ENABLE_DBENGINE
371
char dbenginepath[FILENAME_MAX + 1];
@@ -516,12 +513,12 @@ int is_legacy = 1;
513
, rrdhost_has_rrdpush_sender_enabled(host)?"enabled":"disabled"
514
, host->rrdpush_send_destination?host->rrdpush_send_destination:""
515
, host->rrdpush_send_api_key?host->rrdpush_send_api_key:""
519
- , host->health_enabled?"enabled":"disabled"
516
+ , host->health.health_enabled?"enabled":"disabled"
517
, host->cache_dir
518
, host->varlib_dir
522
- , host->health_log_filename
523
- , string2str(host->health_default_exec)
524
- , string2str(host->health_default_recipient)
519
+ , host->health.health_log_filename
520
+ , string2str(host->health.health_default_exec)
521
+ , string2str(host->health.health_default_recipient)
522
);
523
524
if(!archived)
@@ -566,7 +563,7 @@ static void rrdhost_update(RRDHOST *host
563
564
netdata_spinlock_lock(&host->rrdhost_update_lock);
565
569
- host->health_enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health_enabled;
566
+ host->health.health_enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health_enabled;
567
568
{
569
struct rrdhost_system_info *old = host->system_info;
@@ -651,9 +648,6 @@ static void rrdhost_update(RRDHOST *host
648
info("Host %s is not in archived mode anymore", rrdhost_hostname(host));
649
}
650
654
- if (health_enabled)
655
- health_thread_spawn(host);
656
-
651
netdata_spinlock_unlock(&host->rrdhost_update_lock);
652
}
653
@@ -1175,9 +1169,9 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1169
freez(host->rrdpush_send_api_key);
1170
freez(host->rrdpush_send_destination);
1171
rrdpush_destinations_free(host);
1178
- string_freez(host->health_default_exec);
1179
- string_freez(host->health_default_recipient);
1180
- freez(host->health_log_filename);
1172
+ string_freez(host->health.health_default_exec);
1173
+ string_freez(host->health.health_default_recipient);
1174
+ freez(host->health.health_log_filename);
1175
string_freez(host->registry_hostname);
1176
simple_pattern_free(host->rrdpush_send_charts_matching);
1177
netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock);
@@ -1376,7 +1370,6 @@ void reload_host_labels(void) {
1370
health_label_log_save(localhost);
1371
1372
rrdpush_send_host_labels(localhost);
1379
- health_reload();
1373
}
1374
1375
// ----------------------------------------------------------------------------
database/rrdset.c
+2
-2
@@ -168,7 +168,7 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
168
// chart variables - we need this for data collection to work (collector given chart variables) - not only health
169
rrdsetvar_index_init(st);
170
171
- if (host->health_enabled) {
171
+ if (host->health.health_enabled) {
172
st->rrdfamily = rrdfamily_add_and_acquire(host, rrdset_family(st));
173
st->rrdvars = rrdvariables_create();
174
rrddimvar_index_init(st);
@@ -366,7 +366,7 @@ static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
366
367
st->last_accessed_time_s = now_realtime_sec();
368
369
- if(host->health_enabled && (ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_CHART_ACTIVATED))) {
369
+ if(host->health.health_enabled && (ctr->react_action & (RRDSET_REACT_NEW | RRDSET_REACT_CHART_ACTIVATED))) {
370
rrdset_flag_set(st, RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION);
371
rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION);
372
}
database/rrdsetvar.c
+2
-2
@@ -43,7 +43,7 @@ static inline void rrdsetvar_free_rrdvars_unsafe(RRDSET *st, RRDSETVAR *rs) {
43
// ------------------------------------------------------------------------
44
// HOST
45
46
- if(host->rrdvars && host->health_enabled) {
46
+ if(host->rrdvars && host->health.health_enabled) {
47
rrdvar_release_and_del(host->rrdvars, rs->rrdvar_host_chart_id);
48
rs->rrdvar_host_chart_id = NULL;
49
@@ -93,7 +93,7 @@ static inline void rrdsetvar_update_rrdvars_unsafe(RRDSET *st, RRDSETVAR *rs) {
93
// ------------------------------------------------------------------------
94
// HOST
95
96
- if(host->rrdvars && host->health_enabled) {
96
+ if(host->rrdvars && host->health.health_enabled) {
97
rs->rrdvar_host_chart_id = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_id, rs->type, options, rs->value);
98
rs->rrdvar_host_chart_name = rrdvar_add_and_acquire("host", host->rrdvars, key_chart_name, rs->type, options, rs->value);
99
}
database/sqlite/sqlite_aclk_alert.c
+6
-4
@@ -314,7 +314,7 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
314
alarm_log.utc_offset = wc->host->utc_offset;
315
alarm_log.timezone = strdupz(rrdhost_abbrev_timezone(wc->host));
316
alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) :
317
- strdupz((char *)string2str(wc->host->health_default_exec));
317
+ strdupz((char *)string2str(wc->host->health.health_default_exec));
318
alarm_log.conf_source = strdupz((char *)sqlite3_column_text(res, 16));
319
320
char *edit_command = sqlite3_column_bytes(res, 16) > 0 ?
@@ -531,7 +531,7 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
531
alarm_log.node_id = wc->node_id;
532
alarm_log.log_entries = log_entries;
533
alarm_log.status = wc->alert_updates == 0 ? 2 : 1;
534
- alarm_log.enabled = (int)host->health_enabled;
534
+ alarm_log.enabled = (int)host->health.health_enabled;
535
536
wc->alert_sequence_id = last_sequence;
537
@@ -544,6 +544,8 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
544
545
freez(claim_id);
546
buffer_free(sql);
547
+
548
+ aclk_alert_reloaded = 1;
549
#endif
550
551
return;
@@ -709,7 +711,7 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start
711
(struct aclk_database_worker_config *)host->dbsync_worker :
712
(struct aclk_database_worker_config *)find_inactive_wc_by_node_id(node_id);
713
712
- if (unlikely(!host->health_enabled)) {
714
+ if (unlikely(!host->health.health_enabled)) {
715
log_access("ACLK STA [%s (N/A)]: Ignoring request to stream alert state changes, health is disabled.", node_id);
716
return;
717
}
@@ -849,7 +851,7 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN
851
852
alarm_log->utc_offset = host->utc_offset;
853
alarm_log->timezone = strdupz(rrdhost_abbrev_timezone(host));
852
- alarm_log->exec_path = ae->exec ? strdupz(ae_exec(ae)) : strdupz((char *)string2str(host->health_default_exec));
854
+ alarm_log->exec_path = ae->exec ? strdupz(ae_exec(ae)) : strdupz((char *)string2str(host->health.health_default_exec));
855
alarm_log->conf_source = ae->source ? strdupz(ae_source(ae)) : strdupz((char *)"");
856
857
alarm_log->command = strdupz((char *)edit_command);
database/sqlite/sqlite_health.c
+7
-7
@@ -337,7 +337,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
337
}
338
339
ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
340
- host->health_log_entries_written++;
340
+ host->health.health_log_entries_written++;
341
342
failed:
343
if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
@@ -369,7 +369,7 @@ void sql_health_alarm_log_cleanup(RRDHOST *host) {
369
if(rotate_every < 100) rotate_every = 100;
370
}
371
372
- if(likely(host->health_log_entries_written < rotate_every)) {
372
+ if(likely(host->health.health_log_entries_written < rotate_every)) {
373
return;
374
}
375
@@ -382,7 +382,7 @@ void sql_health_alarm_log_cleanup(RRDHOST *host) {
382
char uuid_str[GUID_LEN + 1];
383
uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
384
385
- snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG(uuid_str, uuid_str, (unsigned long int) (host->health_log_entries_written - rotate_every)));
385
+ snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG(uuid_str, uuid_str, (unsigned long int) (host->health.health_log_entries_written - rotate_every)));
386
387
rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
388
if (unlikely(rc != SQLITE_OK)) {
@@ -398,7 +398,7 @@ void sql_health_alarm_log_cleanup(RRDHOST *host) {
398
if (unlikely(rc != SQLITE_OK))
399
error_report("Failed to finalize the prepared statement to cleanup health log table");
400
401
- host->health_log_entries_written = rotate_every;
401
+ host->health.health_log_entries_written = rotate_every;
402
403
sql_aclk_alert_clean_dead_entries(host);
404
}
@@ -431,13 +431,13 @@ void sql_health_alarm_log_count(RRDHOST *host) {
431
432
rc = sqlite3_step_monitored(res);
433
if (likely(rc == SQLITE_ROW))
434
- host->health_log_entries_written = (size_t) sqlite3_column_int64(res, 0);
434
+ host->health.health_log_entries_written = (size_t) sqlite3_column_int64(res, 0);
435
436
rc = sqlite3_finalize(res);
437
if (unlikely(rc != SQLITE_OK))
438
error_report("Failed to finalize the prepared statement to count health log entries from db");
439
440
- info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", rrdhost_hostname(host), uuid_str, (unsigned long int) host->health_log_entries_written);
440
+ info("HEALTH [%s]: Table health_log_%s, contains %lu entries.", rrdhost_hostname(host), uuid_str, (unsigned long int) host->health.health_log_entries_written);
441
}
442
443
#define SQL_INJECT_REMOVED(guid, guid2) "insert into health_log_%s (hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, " \
@@ -612,7 +612,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
612
ssize_t errored = 0, loaded = 0;
613
char command[MAX_HEALTH_SQL_SIZE + 1];
614
615
- host->health_log_entries_written = 0;
615
+ host->health.health_log_entries_written = 0;
616
617
if (unlikely(!db_meta)) {
618
if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
database/sqlite/sqlite_metadata.c
+12
-1
@@ -328,7 +328,7 @@ static int sql_store_host_info(RRDHOST *host)
328
if (unlikely(rc != SQLITE_OK))
329
goto bind_fail;
330
331
- rc = sqlite3_bind_int(res, ++param, (int ) host->health_enabled);
331
+ rc = sqlite3_bind_int(res, ++param, (int ) host->health.health_enabled);
332
if (unlikely(rc != SQLITE_OK))
333
goto bind_fail;
334
@@ -686,6 +686,16 @@ skip_run:
686
error_report("Failed to finalize the prepared statement when reading dimensions");
687
}
688
689
+static void cleanup_health_log(void)
690
+{
691
+ RRDHOST *host;
692
+ dfe_start_reentrant(rrdhost_root_index, host) {
693
+ if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))
694
+ continue;
695
+ sql_health_alarm_log_cleanup(host);
696
+ }
697
+ dfe_done(host);
698
+}
699
700
//
701
// EVENT LOOP STARTS HERE
@@ -845,6 +855,7 @@ static void start_metadata_cleanup(uv_work_t *req)
855
worker_is_busy(UV_EVENT_METADATA_CLEANUP);
856
struct metadata_wc *wc = req->data;
857
check_dimension_metadata(wc);
858
+ cleanup_health_log();
859
worker_is_idle();
860
}
861
health/health.c
+435
-458
@@ -162,7 +162,7 @@ char *silencers_filename;
162
SIMPLE_PATTERN *conf_enabled_alarms = NULL;
163
164
// the queue of executed alarm notifications that haven't been waited for yet
165
-static __thread struct {
165
+static struct {
166
ALARM_ENTRY *head; // oldest
167
ALARM_ENTRY *tail; // latest
168
} alarm_notifications_in_progress = {NULL, NULL};
@@ -302,7 +302,7 @@ void health_init(void) {
302
* @param host the structure of the host that the function will reload the configuration.
303
*/
304
static void health_reload_host(RRDHOST *host) {
305
- if(unlikely(!host->health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
305
+ if(unlikely(!host->health.health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
306
return;
307
308
log_health("[%s]: Reloading health.", rrdhost_hostname(host));
@@ -346,7 +346,6 @@ static void health_reload_host(RRDHOST *host) {
346
rrdcalctemplate_link_matching_templates_to_rrdset(st);
347
}
348
rrdset_foreach_done(st);
349
- host->aclk_alert_reloaded = 1;
349
}
350
351
/**
@@ -364,6 +363,12 @@ void health_reload(void) {
363
health_reload_host(host);
364
365
rrd_unlock();
366
+
367
+#ifdef ENABLE_ACLK
368
+ if (netdata_cloud_setting) {
369
+ aclk_alert_reloaded = 1;
370
+ }
371
+#endif
372
}
373
374
// ----------------------------------------------------------------------------
@@ -445,8 +450,8 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
450
451
log_health("[%s]: Sending notification for alarm '%s.%s' status %s.", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
452
448
- const char *exec = (ae->exec) ? ae_exec(ae) : string2str(host->health_default_exec);
449
- const char *recipient = (ae->recipient) ? ae_recipient(ae) : string2str(host->health_default_recipient);
453
+ const char *exec = (ae->exec) ? ae_exec(ae) : string2str(host->health.health_default_exec);
454
+ const char *recipient = (ae->recipient) ? ae_recipient(ae) : string2str(host->health.health_default_recipient);
455
456
int n_warn=0, n_crit=0;
457
RRDCALC *rc;
@@ -720,7 +725,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
725
}
726
727
static inline int check_if_resumed_from_suspension(void) {
723
- static __thread usec_t last_realtime = 0, last_monotonic = 0;
728
+ static usec_t last_realtime = 0, last_monotonic = 0;
729
usec_t realtime = now_realtime_usec(), monotonic = now_monotonic_usec();
730
int ret = 0;
731
@@ -736,18 +741,19 @@ static inline int check_if_resumed_from_suspension(void) {
741
return ret;
742
}
743
739
-static void health_thread_cleanup(void *ptr) {
744
+static void health_main_cleanup(void *ptr) {
745
worker_unregister();
746
742
- struct health_state *h = ptr;
743
- h->host->health_spawn = 0;
747
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
748
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
749
+ info("cleaning up...");
750
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
751
745
- log_health("[%s]: Health thread ended.", rrdhost_hostname(h->host));
746
- debug(D_HEALTH, "HEALTH %s: Health thread ended.", rrdhost_hostname(h->host));
752
+ log_health("Health thread ended.");
753
}
754
755
static void initialize_health(RRDHOST *host, int is_localhost) {
750
- if(!host->health_enabled ||
756
+ if(!host->health.health_enabled ||
757
rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH) ||
758
!service_running(SERVICE_HEALTH))
759
return;
@@ -756,8 +762,8 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
762
763
log_health("[%s]: Initializing health.", rrdhost_hostname(host));
764
759
- host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
760
- host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");
765
+ host->health.health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
766
+ host->health.health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");
767
768
host->health_log.next_log_id = 1;
769
host->health_log.next_alarm_id = 1;
@@ -792,13 +798,13 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
798
error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename);
799
}
800
snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir);
795
- host->health_log_filename = strdupz(filename);
801
+ host->health.health_log_filename = strdupz(filename);
802
803
snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir);
798
- host->health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
799
- host->health_default_recipient = string_strdupz("root");
804
+ host->health.health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
805
+ host->health.health_default_recipient = string_strdupz("root");
806
801
- if (!file_is_migrated(host->health_log_filename)) {
807
+ if (!file_is_migrated(host->health.health_log_filename)) {
808
int rc = sql_create_health_log_table(host);
809
if (unlikely(rc)) {
810
log_health("[%s]: Failed to create health log table in the database", rrdhost_hostname(host));
@@ -807,7 +813,7 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
813
}
814
else {
815
health_alarm_log_load(host);
810
- add_migrated_file(host->health_log_filename, 0);
816
+ add_migrated_file(host->health.health_log_filename, 0);
817
}
818
} else {
819
// TODO: This needs to go to the metadata thread
@@ -834,16 +840,14 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
840
841
//Discard alarms with labels that do not apply to host
842
rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
837
-
838
- health_silencers_init();
843
}
844
841
-static void health_sleep(time_t next_run, unsigned int loop __maybe_unused, RRDHOST *host) {
845
+static void health_sleep(time_t next_run, unsigned int loop __maybe_unused) {
846
time_t now = now_realtime_sec();
847
if(now < next_run) {
848
worker_is_idle();
849
debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now));
846
- while (now < next_run && host->health_enabled && service_running(SERVICE_HEALTH)) {
850
+ while (now < next_run && service_running(SERVICE_HEALTH)) {
851
sleep_usec(USEC_PER_SEC);
852
now = now_realtime_sec();
853
}
@@ -1001,534 +1005,522 @@ void *health_main(void *ptr) {
1005
worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET, "rrdset init");
1006
worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM, "rrddim init");
1007
1004
- struct health_state *h = ptr;
1005
- netdata_thread_cleanup_push(health_thread_cleanup, ptr);
1006
-
1007
- RRDHOST *host = h->host;
1008
- initialize_health(host, host == localhost);
1008
+ netdata_thread_cleanup_push(health_main_cleanup, ptr);
1009
1010
int min_run_every = (int)config_get_number(CONFIG_SECTION_HEALTH, "run at least every seconds", 10);
1011
if(min_run_every < 1) min_run_every = 1;
1012
1013
- int cleanup_sql_every_loop = 7200 / min_run_every;
1014
-
1015
- time_t now = now_realtime_sec();
1013
time_t hibernation_delay = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60);
1014
1015
bool health_running_logged = false;
1016
1020
- rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
1017
+ rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts();
1018
1019
unsigned int loop = 0;
1020
#ifdef ENABLE_ACLK
1021
unsigned int marked_aclk_reload_loop = 0;
1022
#endif
1026
- while(service_running(SERVICE_HEALTH) && host->health_enabled) {
1023
+ while(service_running(SERVICE_HEALTH)) {
1024
loop++;
1025
debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
1026
1030
- now = now_realtime_sec();
1027
+ time_t now = now_realtime_sec();
1028
int runnable = 0, apply_hibernation_delay = 0;
1029
time_t next_run = now + min_run_every;
1030
RRDCALC *rc;
1031
+ RRDHOST *host;
1032
1033
if (unlikely(check_if_resumed_from_suspension())) {
1034
apply_hibernation_delay = 1;
1035
1036
log_health(
1039
- "[%s]: Postponing alarm checks for %"PRId64" seconds, "
1037
+ "Postponing alarm checks for %"PRId64" seconds, "
1038
"because it seems that the system was just resumed from suspension.",
1041
- rrdhost_hostname(host),
1039
(int64_t)hibernation_delay);
1040
}
1041
1042
if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) {
1046
- static __thread int logged=0;
1043
+ static int logged=0;
1044
if (!logged) {
1048
- log_health("[%s]: Skipping health checks, because all alarms are disabled via a %s command.",
1049
- rrdhost_hostname(host),
1045
+ log_health("Skipping health checks, because all alarms are disabled via a %s command.",
1046
HEALTH_CMDAPI_CMD_DISABLEALL);
1047
logged = 1;
1048
}
1049
}
1050
1051
#ifdef ENABLE_ACLK
1056
- if (host->aclk_alert_reloaded && !marked_aclk_reload_loop)
1052
+ if (aclk_alert_reloaded && !marked_aclk_reload_loop)
1053
marked_aclk_reload_loop = loop;
1054
#endif
1055
1060
- if (unlikely(apply_hibernation_delay)) {
1061
- log_health(
1062
- "[%s]: Postponing health checks for %"PRId64" seconds.",
1063
- rrdhost_hostname(host),
1064
- (int64_t)hibernation_delay);
1056
+ worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
1057
+ rrd_rdlock();
1058
1066
- host->health_delay_up_to = now + hibernation_delay;
1067
- next_run = now + hibernation_delay;
1068
- health_sleep(next_run, loop, host);
1069
- }
1059
+ rrdhost_foreach_read(host) {
1060
1071
- if (unlikely(host->health_delay_up_to)) {
1072
- if (unlikely(now < host->health_delay_up_to)) {
1073
- next_run = host->health_delay_up_to;
1074
- health_sleep(next_run, loop, host);
1061
+ if (unlikely(!host->health.health_enabled))
1062
continue;
1076
- }
1077
-
1078
- log_health("[%s]: Resuming health checks after delay.", rrdhost_hostname(host));
1079
- host->health_delay_up_to = 0;
1080
- }
1063
1082
- // wait until cleanup of obsolete charts on children is complete
1083
- if (host != localhost) {
1084
- if (unlikely(host->trigger_chart_obsoletion_check == 1)) {
1085
- log_health("[%s]: Waiting for chart obsoletion check.", rrdhost_hostname(host));
1086
- health_sleep(next_run, loop, host);
1087
- continue;
1064
+ if (unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))) {
1065
+ rrd_unlock();
1066
+ initialize_health(host, host == localhost);
1067
+ rrd_rdlock();
1068
}
1089
- }
1090
-
1091
- if (!health_running_logged) {
1092
- log_health("[%s]: Health is running.", rrdhost_hostname(host));
1093
- health_running_logged = true;
1094
- }
1069
1096
- if(likely(!host->health_log_fp) && (loop == 1 || loop % cleanup_sql_every_loop == 0))
1097
- sql_health_alarm_log_cleanup(host);
1070
+ health_execute_delayed_initializations(host);
1071
1099
- health_execute_delayed_initializations(host);
1072
+ rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
1073
1101
- worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
1102
-
1103
- // the first loop is to lookup values from the db
1104
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
1074
+ if (unlikely(apply_hibernation_delay)) {
1075
+ log_health(
1076
+ "[%s]: Postponing health checks for %"PRId64" seconds.",
1077
+ rrdhost_hostname(host),
1078
+ (int64_t)hibernation_delay);
1079
1106
- rrdcalc_update_info_using_rrdset_labels(rc);
1080
+ host->health.health_delay_up_to = now + hibernation_delay;
1081
+ }
1082
1108
- if (update_disabled_silenced(host, rc))
1109
- continue;
1083
+ if (unlikely(host->health.health_delay_up_to)) {
1084
+ if (unlikely(now < host->health.health_delay_up_to)) {
1085
+ continue;
1086
+ }
1087
1111
- // create an alert removed event if the chart is obsolete and
1112
- // has stopped being collected for 60 seconds
1113
- if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
1114
- rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
1115
- now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
1116
- if (!rrdcalc_isrepeating(rc)) {
1117
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1118
- time_t now = now_realtime_sec();
1119
-
1120
- ALARM_ENTRY *ae = health_create_alarm_entry(
1121
- host,
1122
- rc->id,
1123
- rc->next_event_id++,
1124
- rc->config_hash_id,
1125
- now,
1126
- rc->name,
1127
- rc->rrdset->id,
1128
- rc->rrdset->context,
1129
- rc->rrdset->family,
1130
- rc->classification,
1131
- rc->component,
1132
- rc->type,
1133
- rc->exec,
1134
- rc->recipient,
1135
- now - rc->last_status_change,
1136
- rc->value,
1137
- NAN,
1138
- rc->status,
1139
- RRDCALC_STATUS_REMOVED,
1140
- rc->source,
1141
- rc->units,
1142
- rc->info,
1143
- 0,
1144
- rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
1145
-
1146
- if (ae) {
1147
- health_alarm_log_add_entry(host, ae);
1148
- rc->old_status = rc->status;
1149
- rc->status = RRDCALC_STATUS_REMOVED;
1150
- rc->last_status_change = now;
1151
- rc->last_updated = now;
1152
- rc->value = NAN;
1088
+ log_health("[%s]: Resuming health checks after delay.", rrdhost_hostname(host));
1089
+ host->health.health_delay_up_to = 0;
1090
+ }
1091
1154
-#ifdef ENABLE_ACLK
1155
- if (netdata_cloud_setting && likely(!host->aclk_alert_reloaded))
1156
- sql_queue_alarm_to_aclk(host, ae, 1);
1157
-#endif
1158
- }
1092
+ // wait until cleanup of obsolete charts on children is complete
1093
+ if (host != localhost) {
1094
+ if (unlikely(host->trigger_chart_obsoletion_check == 1)) {
1095
+ log_health("[%s]: Waiting for chart obsoletion check.", rrdhost_hostname(host));
1096
+ continue;
1097
}
1098
}
1099
1162
- if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
1163
- if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
1164
- rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
1165
- continue;
1100
+ if (!health_running_logged) {
1101
+ log_health("[%s]: Health is running.", rrdhost_hostname(host));
1102
+ health_running_logged = true;
1103
}
1104
1168
- runnable++;
1169
- rc->old_value = rc->value;
1170
- rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
1105
+ worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
1106
1172
- // ------------------------------------------------------------
1173
- // if there is database lookup, do it
1107
+ // the first loop is to lookup values from the db
1108
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
1109
1175
- if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
1176
- worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
1110
+ rrdcalc_update_info_using_rrdset_labels(rc);
1111
1178
- /* time_t old_db_timestamp = rc->db_before; */
1179
- int value_is_null = 0;
1112
+ if (update_disabled_silenced(host, rc))
1113
+ continue;
1114
1181
- int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
1182
- rc->after, rc->before, rc->group, NULL,
1183
- 0, rc->options,
1184
- &rc->db_after,&rc->db_before,
1185
- NULL, NULL, NULL,
1186
- &value_is_null, NULL, 0, 0,
1187
- QUERY_SOURCE_HEALTH, STORAGE_PRIORITY_LOW);
1115
+ // create an alert removed event if the chart is obsolete and
1116
+ // has stopped being collected for 60 seconds
1117
+ if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
1118
+ rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
1119
+ now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
1120
+ if (!rrdcalc_isrepeating(rc)) {
1121
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1122
+ time_t now = now_realtime_sec();
1123
+
1124
+ ALARM_ENTRY *ae = health_create_alarm_entry(
1125
+ host,
1126
+ rc->id,
1127
+ rc->next_event_id++,
1128
+ rc->config_hash_id,
1129
+ now,
1130
+ rc->name,
1131
+ rc->rrdset->id,
1132
+ rc->rrdset->context,
1133
+ rc->rrdset->family,
1134
+ rc->classification,
1135
+ rc->component,
1136
+ rc->type,
1137
+ rc->exec,
1138
+ rc->recipient,
1139
+ now - rc->last_status_change,
1140
+ rc->value,
1141
+ NAN,
1142
+ rc->status,
1143
+ RRDCALC_STATUS_REMOVED,
1144
+ rc->source,
1145
+ rc->units,
1146
+ rc->info,
1147
+ 0,
1148
+ rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
1149
+
1150
+ if (ae) {
1151
+ health_alarm_log_add_entry(host, ae);
1152
+ rc->old_status = rc->status;
1153
+ rc->status = RRDCALC_STATUS_REMOVED;
1154
+ rc->last_status_change = now;
1155
+ rc->last_updated = now;
1156
+ rc->value = NAN;
1157
1189
- if (unlikely(ret != 200)) {
1190
- // database lookup failed
1191
- rc->value = NAN;
1192
- rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
1158
+#ifdef ENABLE_ACLK
1159
+ if (netdata_cloud_setting && likely(!aclk_alert_reloaded))
1160
+ sql_queue_alarm_to_aclk(host, ae, 1);
1161
+#endif
1162
+ }
1163
+ }
1164
+ }
1165
1194
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
1195
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
1196
- );
1197
- } else
1198
- rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
1199
-
1200
- /* - RRDCALC_FLAG_DB_STALE not currently used
1201
- if (unlikely(old_db_timestamp == rc->db_before)) {
1202
- // database is stale
1203
-
1204
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
1205
-
1206
- if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
1207
- rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
1208
- error("Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
1209
- }
1210
- }
1211
- else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
1212
- rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
1213
- */
1214
-
1215
- if (unlikely(value_is_null)) {
1216
- // collected value is null
1217
- rc->value = NAN;
1218
- rc->run_flags |= RRDCALC_FLAG_DB_NAN;
1219
-
1220
- debug(D_HEALTH,
1221
- "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
1222
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
1223
- );
1224
- } else
1225
- rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
1166
+ if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
1167
+ if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
1168
+ rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
1169
+ continue;
1170
+ }
1171
1227
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
1228
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
1229
- );
1230
- }
1172
+ runnable++;
1173
+ rc->old_value = rc->value;
1174
+ rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
1175
1232
- // ------------------------------------------------------------
1233
- // if there is calculation expression, run it
1176
+ // ------------------------------------------------------------
1177
+ // if there is database lookup, do it
1178
1235
- if (unlikely(rc->calculation)) {
1236
- worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);
1179
+ if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
1180
+ worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
1181
1238
- if (unlikely(!expression_evaluate(rc->calculation))) {
1239
- // calculation failed
1240
- rc->value = NAN;
1241
- rc->run_flags |= RRDCALC_FLAG_CALC_ERROR;
1182
+ /* time_t old_db_timestamp = rc->db_before; */
1183
+ int value_is_null = 0;
1184
1243
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",
1244
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1245
- rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)
1246
- );
1247
- } else {
1248
- rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR;
1185
+ int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
1186
+ rc->after, rc->before, rc->group, NULL,
1187
+ 0, rc->options,
1188
+ &rc->db_after,&rc->db_before,
1189
+ NULL, NULL, NULL,
1190
+ &value_is_null, NULL, 0, 0,
1191
+ QUERY_SOURCE_HEALTH, STORAGE_PRIORITY_LOW);
1192
1250
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
1251
- NETDATA_DOUBLE_FORMAT
1252
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1253
- rc->calculation->parsed_as, rc->calculation->result,
1254
- buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc)
1255
- );
1193
+ if (unlikely(ret != 200)) {
1194
+ // database lookup failed
1195
+ rc->value = NAN;
1196
+ rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
1197
1257
- rc->value = rc->calculation->result;
1258
- }
1259
- }
1260
- }
1261
- foreach_rrdcalc_in_rrdhost_done(rc);
1198
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
1199
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
1200
+ );
1201
+ } else
1202
+ rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
1203
1263
- if (unlikely(runnable && service_running(SERVICE_HEALTH))) {
1264
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
1265
- if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
1266
- continue;
1204
+ if (unlikely(value_is_null)) {
1205
+ // collected value is null
1206
+ rc->value = NAN;
1207
+ rc->run_flags |= RRDCALC_FLAG_DB_NAN;
1208
1268
- if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
1269
- continue;
1209
+ debug(D_HEALTH,
1210
+ "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
1211
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
1212
+ );
1213
+ } else
1214
+ rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
1215
+
1216
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
1217
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
1218
+ );
1219
}
1271
- RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
1272
- RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
1220
1274
- // --------------------------------------------------------
1275
- // check the warning expression
1221
+ // ------------------------------------------------------------
1222
+ // if there is calculation expression, run it
1223
1277
- if (likely(rc->warning)) {
1278
- worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);
1224
+ if (unlikely(rc->calculation)) {
1225
+ worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);
1226
1280
- if (unlikely(!expression_evaluate(rc->warning))) {
1227
+ if (unlikely(!expression_evaluate(rc->calculation))) {
1228
// calculation failed
1282
- rc->run_flags |= RRDCALC_FLAG_WARN_ERROR;
1229
+ rc->value = NAN;
1230
+ rc->run_flags |= RRDCALC_FLAG_CALC_ERROR;
1231
1284
- debug(D_HEALTH,
1285
- "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",
1232
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",
1233
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1287
- buffer_tostring(rc->warning->error_msg)
1234
+ rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)
1235
);
1236
} else {
1290
- rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR;
1291
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
1237
+ rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR;
1238
+
1239
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
1240
NETDATA_DOUBLE_FORMAT
1293
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
1294
- rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc)
1241
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1242
+ rc->calculation->parsed_as, rc->calculation->result,
1243
+ buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc)
1244
);
1296
- warning_status = rrdcalc_value2status(rc->warning->result);
1245
+
1246
+ rc->value = rc->calculation->result;
1247
}
1248
}
1249
+ }
1250
+ foreach_rrdcalc_in_rrdhost_done(rc);
1251
1300
- // --------------------------------------------------------
1301
- // check the critical expression
1302
-
1303
- if (likely(rc->critical)) {
1304
- worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);
1252
+ if (unlikely(runnable && service_running(SERVICE_HEALTH))) {
1253
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
1254
+ if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
1255
+ continue;
1256
1306
- if (unlikely(!expression_evaluate(rc->critical))) {
1307
- // calculation failed
1308
- rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR;
1257
+ if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
1258
+ continue;
1259
+ }
1260
+ RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
1261
+ RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
1262
+
1263
+ // --------------------------------------------------------
1264
+ // check the warning expression
1265
+
1266
+ if (likely(rc->warning)) {
1267
+ worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);
1268
+
1269
+ if (unlikely(!expression_evaluate(rc->warning))) {
1270
+ // calculation failed
1271
+ rc->run_flags |= RRDCALC_FLAG_WARN_ERROR;
1272
+
1273
+ debug(D_HEALTH,
1274
+ "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",
1275
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1276
+ buffer_tostring(rc->warning->error_msg)
1277
+ );
1278
+ } else {
1279
+ rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR;
1280
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
1281
+ NETDATA_DOUBLE_FORMAT
1282
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
1283
+ rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc)
1284
+ );
1285
+ warning_status = rrdcalc_value2status(rc->warning->result);
1286
+ }
1287
+ }
1288
1310
- debug(D_HEALTH,
1311
- "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",
1312
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1313
- buffer_tostring(rc->critical->error_msg)
1314
- );
1315
- } else {
1316
- rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
1317
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
1318
- NETDATA_DOUBLE_FORMAT
1319
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
1320
- rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg),
1321
- rrdcalc_source(rc)
1322
- );
1323
- critical_status = rrdcalc_value2status(rc->critical->result);
1289
+ // --------------------------------------------------------
1290
+ // check the critical expression
1291
+
1292
+ if (likely(rc->critical)) {
1293
+ worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);
1294
+
1295
+ if (unlikely(!expression_evaluate(rc->critical))) {
1296
+ // calculation failed
1297
+ rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR;
1298
+
1299
+ debug(D_HEALTH,
1300
+ "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",
1301
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1302
+ buffer_tostring(rc->critical->error_msg)
1303
+ );
1304
+ } else {
1305
+ rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
1306
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
1307
+ NETDATA_DOUBLE_FORMAT
1308
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
1309
+ rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg),
1310
+ rrdcalc_source(rc)
1311
+ );
1312
+ critical_status = rrdcalc_value2status(rc->critical->result);
1313
+ }
1314
}
1325
- }
1315
1327
- // --------------------------------------------------------
1328
- // decide the final alarm status
1316
+ // --------------------------------------------------------
1317
+ // decide the final alarm status
1318
1330
- RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
1319
+ RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
1320
1332
- switch (warning_status) {
1333
- case RRDCALC_STATUS_CLEAR:
1334
- status = RRDCALC_STATUS_CLEAR;
1335
- break;
1321
+ switch (warning_status) {
1322
+ case RRDCALC_STATUS_CLEAR:
1323
+ status = RRDCALC_STATUS_CLEAR;
1324
+ break;
1325
1337
- case RRDCALC_STATUS_RAISED:
1338
- status = RRDCALC_STATUS_WARNING;
1339
- break;
1326
+ case RRDCALC_STATUS_RAISED:
1327
+ status = RRDCALC_STATUS_WARNING;
1328
+ break;
1329
1341
- default:
1342
- break;
1343
- }
1330
+ default:
1331
+ break;
1332
+ }
1333
1345
- switch (critical_status) {
1346
- case RRDCALC_STATUS_CLEAR:
1347
- if (status == RRDCALC_STATUS_UNDEFINED)
1348
- status = RRDCALC_STATUS_CLEAR;
1349
- break;
1334
+ switch (critical_status) {
1335
+ case RRDCALC_STATUS_CLEAR:
1336
+ if (status == RRDCALC_STATUS_UNDEFINED)
1337
+ status = RRDCALC_STATUS_CLEAR;
1338
+ break;
1339
1351
- case RRDCALC_STATUS_RAISED:
1352
- status = RRDCALC_STATUS_CRITICAL;
1353
- break;
1340
+ case RRDCALC_STATUS_RAISED:
1341
+ status = RRDCALC_STATUS_CRITICAL;
1342
+ break;
1343
1355
- default:
1356
- break;
1357
- }
1344
+ default:
1345
+ break;
1346
+ }
1347
1359
- // --------------------------------------------------------
1360
- // check if the new status and the old differ
1348
+ // --------------------------------------------------------
1349
+ // check if the new status and the old differ
1350
1362
- if (status != rc->status) {
1363
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1364
- int delay = 0;
1351
+ if (status != rc->status) {
1352
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1353
+ int delay = 0;
1354
1366
- // apply trigger hysteresis
1355
+ // apply trigger hysteresis
1356
1368
- if (now > rc->delay_up_to_timestamp) {
1369
- rc->delay_up_current = rc->delay_up_duration;
1370
- rc->delay_down_current = rc->delay_down_duration;
1371
- rc->delay_last = 0;
1372
- rc->delay_up_to_timestamp = 0;
1373
- } else {
1374
- rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier);
1375
- if (rc->delay_up_current > rc->delay_max_duration)
1376
- rc->delay_up_current = rc->delay_max_duration;
1357
+ if (now > rc->delay_up_to_timestamp) {
1358
+ rc->delay_up_current = rc->delay_up_duration;
1359
+ rc->delay_down_current = rc->delay_down_duration;
1360
+ rc->delay_last = 0;
1361
+ rc->delay_up_to_timestamp = 0;
1362
+ } else {
1363
+ rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier);
1364
+ if (rc->delay_up_current > rc->delay_max_duration)
1365
+ rc->delay_up_current = rc->delay_max_duration;
1366
1378
- rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier);
1379
- if (rc->delay_down_current > rc->delay_max_duration)
1380
- rc->delay_down_current = rc->delay_max_duration;
1381
- }
1367
+ rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier);
1368
+ if (rc->delay_down_current > rc->delay_max_duration)
1369
+ rc->delay_down_current = rc->delay_max_duration;
1370
+ }
1371
1383
- if (status > rc->status)
1384
- delay = rc->delay_up_current;
1385
- else
1386
- delay = rc->delay_down_current;
1387
-
1388
- // COMMENTED: because we do need to send raising alarms
1389
- // if(now + delay < rc->delay_up_to_timestamp)
1390
- // delay = (int)(rc->delay_up_to_timestamp - now);
1391
-
1392
- rc->delay_last = delay;
1393
- rc->delay_up_to_timestamp = now + delay;
1394
-
1395
- ALARM_ENTRY *ae = health_create_alarm_entry(
1396
- host,
1397
- rc->id,
1398
- rc->next_event_id++,
1399
- rc->config_hash_id,
1400
- now,
1401
- rc->name,
1402
- rc->rrdset->id,
1403
- rc->rrdset->context,
1404
- rc->rrdset->family,
1405
- rc->classification,
1406
- rc->component,
1407
- rc->type,
1408
- rc->exec,
1409
- rc->recipient,
1410
- now - rc->last_status_change,
1411
- rc->old_value,
1412
- rc->value,
1413
- rc->status,
1414
- status,
1415
- rc->source,
1416
- rc->units,
1417
- rc->info,
1418
- rc->delay_last,
1419
- (
1420
- ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1421
- ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
1422
- (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
1423
- )
1424
- );
1425
-
1426
- health_alarm_log_add_entry(host, ae);
1427
-
1428
- log_health("[%s]: Alert event for [%s.%s], value [%s], status [%s].", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), ae_new_value_string(ae), rrdcalc_status2string(ae->new_status));
1429
-
1430
- rc->last_status_change = now;
1431
- rc->old_status = rc->status;
1432
- rc->status = status;
1433
- }
1372
+ if (status > rc->status)
1373
+ delay = rc->delay_up_current;
1374
+ else
1375
+ delay = rc->delay_down_current;
1376
+
1377
+ // COMMENTED: because we do need to send raising alarms
1378
+ // if(now + delay < rc->delay_up_to_timestamp)
1379
+ // delay = (int)(rc->delay_up_to_timestamp - now);
1380
+
1381
+ rc->delay_last = delay;
1382
+ rc->delay_up_to_timestamp = now + delay;
1383
+
1384
+ ALARM_ENTRY *ae = health_create_alarm_entry(
1385
+ host,
1386
+ rc->id,
1387
+ rc->next_event_id++,
1388
+ rc->config_hash_id,
1389
+ now,
1390
+ rc->name,
1391
+ rc->rrdset->id,
1392
+ rc->rrdset->context,
1393
+ rc->rrdset->family,
1394
+ rc->classification,
1395
+ rc->component,
1396
+ rc->type,
1397
+ rc->exec,
1398
+ rc->recipient,
1399
+ now - rc->last_status_change,
1400
+ rc->old_value,
1401
+ rc->value,
1402
+ rc->status,
1403
+ status,
1404
+ rc->source,
1405
+ rc->units,
1406
+ rc->info,
1407
+ rc->delay_last,
1408
+ (
1409
+ ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1410
+ ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
1411
+ (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
1412
+ )
1413
+ );
1414
1435
- rc->last_updated = now;
1436
- rc->next_update = now + rc->update_every;
1415
+ health_alarm_log_add_entry(host, ae);
1416
1438
- if (next_run > rc->next_update)
1439
- next_run = rc->next_update;
1440
- }
1441
- foreach_rrdcalc_in_rrdhost_done(rc);
1417
+ log_health("[%s]: Alert event for [%s.%s], value [%s], status [%s].", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), ae_new_value_string(ae), rrdcalc_status2string(ae->new_status));
1418
1443
- // process repeating alarms
1444
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
1445
- int repeat_every = 0;
1446
- if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
1447
- if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
1448
- rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
1449
- repeat_every = rc->warn_repeat_every;
1450
- } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
1451
- rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
1452
- repeat_every = rc->crit_repeat_every;
1453
- } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
1454
- if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE)) {
1455
- if(rc->old_status == RRDCALC_STATUS_CRITICAL) {
1456
- repeat_every = 1;
1457
- } else if (rc->old_status == RRDCALC_STATUS_WARNING) {
1458
- repeat_every = 1;
1419
+ rc->last_status_change = now;
1420
+ rc->old_status = rc->status;
1421
+ rc->status = status;
1422
+ }
1423
+
1424
+ rc->last_updated = now;
1425
+ rc->next_update = now + rc->update_every;
1426
+
1427
+ if (next_run > rc->next_update)
1428
+ next_run = rc->next_update;
1429
+ }
1430
+ foreach_rrdcalc_in_rrdhost_done(rc);
1431
+
1432
+ // process repeating alarms
1433
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
1434
+ int repeat_every = 0;
1435
+ if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
1436
+ if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
1437
+ rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
1438
+ repeat_every = rc->warn_repeat_every;
1439
+ } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
1440
+ rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
1441
+ repeat_every = rc->crit_repeat_every;
1442
+ } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
1443
+ if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE)) {
1444
+ if(rc->old_status == RRDCALC_STATUS_CRITICAL) {
1445
+ repeat_every = 1;
1446
+ } else if (rc->old_status == RRDCALC_STATUS_WARNING) {
1447
+ repeat_every = 1;
1448
+ }
1449
}
1450
}
1451
+ } else {
1452
+ continue;
1453
}
1462
- } else {
1463
- continue;
1464
- }
1454
1466
- if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
1467
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1468
- rc->last_repeat = now;
1469
- if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
1470
-
1471
- ALARM_ENTRY *ae = health_create_alarm_entry(
1472
- host,
1473
- rc->id,
1474
- rc->next_event_id++,
1475
- rc->config_hash_id,
1476
- now,
1477
- rc->name,
1478
- rc->rrdset->id,
1479
- rc->rrdset->context,
1480
- rc->rrdset->family,
1481
- rc->classification,
1482
- rc->component,
1483
- rc->type,
1484
- rc->exec,
1485
- rc->recipient,
1486
- now - rc->last_status_change,
1487
- rc->old_value,
1488
- rc->value,
1489
- rc->old_status,
1490
- rc->status,
1491
- rc->source,
1492
- rc->units,
1493
- rc->info,
1494
- rc->delay_last,
1495
- (
1496
- ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1497
- ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
1498
- (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
1499
- )
1500
- );
1501
-
1502
- ae->last_repeat = rc->last_repeat;
1503
- if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
1504
- ae->flags |= HEALTH_ENTRY_RUN_ONCE;
1455
+ if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
1456
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1457
+ rc->last_repeat = now;
1458
+ if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
1459
+
1460
+ ALARM_ENTRY *ae = health_create_alarm_entry(
1461
+ host,
1462
+ rc->id,
1463
+ rc->next_event_id++,
1464
+ rc->config_hash_id,
1465
+ now,
1466
+ rc->name,
1467
+ rc->rrdset->id,
1468
+ rc->rrdset->context,
1469
+ rc->rrdset->family,
1470
+ rc->classification,
1471
+ rc->component,
1472
+ rc->type,
1473
+ rc->exec,
1474
+ rc->recipient,
1475
+ now - rc->last_status_change,
1476
+ rc->old_value,
1477
+ rc->value,
1478
+ rc->old_status,
1479
+ rc->status,
1480
+ rc->source,
1481
+ rc->units,
1482
+ rc->info,
1483
+ rc->delay_last,
1484
+ (
1485
+ ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1486
+ ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
1487
+ (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
1488
+ )
1489
+ );
1490
+
1491
+ ae->last_repeat = rc->last_repeat;
1492
+ if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
1493
+ ae->flags |= HEALTH_ENTRY_RUN_ONCE;
1494
+ }
1495
+ rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
1496
+ health_process_notifications(host, ae);
1497
+ debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
1498
+ health_alarm_wait_for_execution(ae);
1499
+ health_alarm_log_free_one_nochecks_nounlink(ae);
1500
}
1506
- rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
1507
- health_process_notifications(host, ae);
1508
- debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
1509
- health_alarm_wait_for_execution(ae);
1510
- health_alarm_log_free_one_nochecks_nounlink(ae);
1501
}
1502
+ foreach_rrdcalc_in_rrdhost_done(rc);
1503
}
1513
- foreach_rrdcalc_in_rrdhost_done(rc);
1514
- }
1504
1516
- if (unlikely(!service_running(SERVICE_HEALTH)))
1517
- break;
1505
+ if (unlikely(!service_running(SERVICE_HEALTH)))
1506
+ break;
1507
1519
- // execute notifications
1520
- // and cleanup
1521
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
1522
- health_alarm_log_process(host);
1508
+ // execute notifications
1509
+ // and cleanup
1510
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
1511
+ health_alarm_log_process(host);
1512
1524
- if (unlikely(!service_running(SERVICE_HEALTH))) {
1525
- // wait for all notifications to finish before allowing health to be cleaned up
1526
- ALARM_ENTRY *ae;
1527
- while (NULL != (ae = alarm_notifications_in_progress.head)) {
1528
- health_alarm_wait_for_execution(ae);
1513
+ if (unlikely(!service_running(SERVICE_HEALTH))) {
1514
+ // wait for all notifications to finish before allowing health to be cleaned up
1515
+ ALARM_ENTRY *ae;
1516
+ while (NULL != (ae = alarm_notifications_in_progress.head)) {
1517
+ health_alarm_wait_for_execution(ae);
1518
+ }
1519
+ break;
1520
}
1530
- break;
1531
- }
1521
+ } //for each host
1522
+
1523
+ rrd_unlock();
1524
1525
// wait for all notifications to finish before allowing health to be cleaned up
1526
ALARM_ENTRY *ae;
@@ -1537,9 +1529,13 @@ void *health_main(void *ptr) {
1529
}
1530
1531
#ifdef ENABLE_ACLK
1540
- if (netdata_cloud_setting && unlikely(host->aclk_alert_reloaded) && loop > (marked_aclk_reload_loop + 2)) {
1541
- sql_queue_removed_alerts_to_aclk(host);
1542
- host->aclk_alert_reloaded = 0;
1532
+ if (netdata_cloud_setting && unlikely(aclk_alert_reloaded) && loop > (marked_aclk_reload_loop + 2)) {
1533
+ rrdhost_foreach_read(host) {
1534
+ if (unlikely(!host->health.health_enabled))
1535
+ continue;
1536
+ sql_queue_removed_alerts_to_aclk(host);
1537
+ }
1538
+ aclk_alert_reloaded = 0;
1539
marked_aclk_reload_loop = 0;
1540
}
1541
#endif
@@ -1547,7 +1543,7 @@ void *health_main(void *ptr) {
1543
if(unlikely(!service_running(SERVICE_HEALTH)))
1544
break;
1545
1550
- health_sleep(next_run, loop, host);
1546
+ health_sleep(next_run, loop);
1547
1548
} // forever
1549
@@ -1568,22 +1564,3 @@ void health_add_host_labels(void) {
1564
rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_AUTO);
1565
}
1566
1571
-void health_thread_spawn(RRDHOST * host) {
1572
- if(!host->health_spawn) {
1573
- char tag[NETDATA_THREAD_TAG_MAX + 1];
1574
- snprintfz(tag, NETDATA_THREAD_TAG_MAX, "HEALTH[%s]", rrdhost_hostname(host));
1575
- struct health_state *health = callocz(1, sizeof(*health));
1576
- health->host = host;
1577
-
1578
- netdata_thread_t health_thread;
1579
- if(netdata_thread_create(&health_thread, tag, NETDATA_THREAD_OPTION_DEFAULT, health_main, (void *) health)) {
1580
- log_health("[%s]: Failed to create new thread for client.", rrdhost_hostname(host));
1581
- error("HEALTH [%s]: Failed to create new thread for client.", rrdhost_hostname(host));
1582
- }
1583
- else {
1584
- log_health("[%s]: Created new thread for client.", rrdhost_hostname(host));
1585
- host->health_spawn = 1;
1586
- host->aclk_alert_reloaded = 1;
1587
- }
1588
- }
1589
-}
health/health.h
-7
@@ -49,9 +49,6 @@ int health_alarm_log_open(RRDHOST *host);
49
void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
50
void health_alarm_log_load(RRDHOST *host);
51
52
-void health_thread_spawn(RRDHOST *host);
53
-void health_thread_stop(RRDHOST *host);
54
-
52
ALARM_ENTRY* health_create_alarm_entry(
53
RRDHOST *host,
54
uint32_t alarm_id,
@@ -80,10 +77,6 @@ ALARM_ENTRY* health_create_alarm_entry(
77
78
void health_alarm_log_add_entry(RRDHOST *host, ALARM_ENTRY *ae);
79
83
-struct health_state {
84
- RRDHOST *host;
85
-};
86
-
80
void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath);
81
char *health_user_config_dir(void);
82
char *health_stock_config_dir(void);
health/health_config.c
+5
-5
@@ -573,8 +573,8 @@ static int health_readfile(const char *filename, void *data) {
573
rc->old_value = NAN;
574
rc->delay_multiplier = 1.0;
575
rc->old_status = RRDCALC_STATUS_UNINITIALIZED;
576
- rc->warn_repeat_every = host->health_default_warn_repeat_every;
577
- rc->crit_repeat_every = host->health_default_crit_repeat_every;
576
+ rc->warn_repeat_every = host->health.health_default_warn_repeat_every;
577
+ rc->crit_repeat_every = host->health.health_default_crit_repeat_every;
578
if (alert_cfg)
579
alert_config_free(alert_cfg);
580
alert_cfg = callocz(1, sizeof(struct alert_config));
@@ -619,8 +619,8 @@ static int health_readfile(const char *filename, void *data) {
619
rt->green = NAN;
620
rt->red = NAN;
621
rt->delay_multiplier = (float)1.0;
622
- rt->warn_repeat_every = host->health_default_warn_repeat_every;
623
- rt->crit_repeat_every = host->health_default_crit_repeat_every;
622
+ rt->warn_repeat_every = host->health.health_default_warn_repeat_every;
623
+ rt->crit_repeat_every = host->health.health_default_crit_repeat_every;
624
if (alert_cfg)
625
alert_config_free(alert_cfg);
626
alert_cfg = callocz(1, sizeof(struct alert_config));
@@ -1171,7 +1171,7 @@ void sql_refresh_hashes(void)
1171
}
1172
1173
void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath) {
1174
- if(unlikely((!host->health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)) ||
1174
+ if(unlikely((!host->health.health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)) ||
1175
!service_running(SERVICE_HEALTH)) {
1176
debug(D_HEALTH, "CONFIG health is not enabled for host '%s'", rrdhost_hostname(host));
1177
return;
health/health_json.c
+5
-5
@@ -75,8 +75,8 @@ void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host)
75
, (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false"
76
, (unsigned long)ae->exec_run_timestamp
77
, (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false"
78
- , ae->exec?ae_exec(ae):string2str(host->health_default_exec)
79
- , ae->recipient?ae_recipient(ae):string2str(host->health_default_recipient)
78
+ , ae->exec?ae_exec(ae):string2str(host->health.health_default_exec)
79
+ , ae->recipient?ae_recipient(ae):string2str(host->health.health_default_recipient)
80
, ae->exec_code
81
, ae_source(ae)
82
, edit_command
@@ -219,8 +219,8 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
219
, (rc->rrdset)?"true":"false"
220
, (rc->run_flags & RRDCALC_FLAG_DISABLED)?"true":"false"
221
, (rc->run_flags & RRDCALC_FLAG_SILENCED)?"true":"false"
222
- , rc->exec?rrdcalc_exec(rc):string2str(host->health_default_exec)
223
- , rc->recipient?rrdcalc_recipient(rc):string2str(host->health_default_recipient)
222
+ , rc->exec?rrdcalc_exec(rc):string2str(host->health.health_default_exec)
223
+ , rc->recipient?rrdcalc_recipient(rc):string2str(host->health.health_default_recipient)
224
, rrdcalc_source(rc)
225
, rrdcalc_units(rc)
226
, rrdcalc_info(rc)
@@ -372,7 +372,7 @@ void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
372
"\n\t\"alarms\": {\n",
373
rrdhost_hostname(host),
374
(host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0,
375
- host->health_enabled?"true":"false",
375
+ host->health.health_enabled?"true":"false",
376
(unsigned long)now_realtime_sec());
377
378
health_alarms2json_fill_alarms(host, wb, all, health_rrdcalc2json_nolock);
health/health_log.c
+37
-37
@@ -7,25 +7,25 @@
7
// no need for locking - only one thread is reading / writing the alarms log
8
9
inline int health_alarm_log_open(RRDHOST *host) {
10
- if(host->health_log_fp)
11
- fclose(host->health_log_fp);
10
+ if(host->health.health_log_fp)
11
+ fclose(host->health.health_log_fp);
12
13
- host->health_log_fp = fopen(host->health_log_filename, "a");
13
+ host->health.health_log_fp = fopen(host->health.health_log_filename, "a");
14
15
- if(host->health_log_fp) {
16
- if (setvbuf(host->health_log_fp, NULL, _IOLBF, 0) != 0)
17
- error("HEALTH [%s]: cannot set line buffering on health log file '%s'.", rrdhost_hostname(host), host->health_log_filename);
15
+ if(host->health.health_log_fp) {
16
+ if (setvbuf(host->health.health_log_fp, NULL, _IOLBF, 0) != 0)
17
+ error("HEALTH [%s]: cannot set line buffering on health log file '%s'.", rrdhost_hostname(host), host->health.health_log_filename);
18
return 0;
19
}
20
21
- error("HEALTH [%s]: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", rrdhost_hostname(host), host->health_log_filename);
21
+ error("HEALTH [%s]: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", rrdhost_hostname(host), host->health.health_log_filename);
22
return -1;
23
}
24
25
static inline void health_alarm_log_close(RRDHOST *host) {
26
- if(host->health_log_fp) {
27
- fclose(host->health_log_fp);
28
- host->health_log_fp = NULL;
26
+ if(host->health.health_log_fp) {
27
+ fclose(host->health.health_log_fp);
28
+ host->health.health_log_fp = NULL;
29
}
30
}
31
@@ -37,33 +37,33 @@ static inline void health_log_rotate(RRDHOST *host) {
37
if(rotate_every < 100) rotate_every = 100;
38
}
39
40
- if(unlikely(host->health_log_entries_written > rotate_every)) {
41
- if(unlikely(host->health_log_fp)) {
40
+ if(unlikely(host->health.health_log_entries_written > rotate_every)) {
41
+ if(unlikely(host->health.health_log_fp)) {
42
health_alarm_log_close(host);
43
44
char old_filename[FILENAME_MAX + 1];
45
- snprintfz(old_filename, FILENAME_MAX, "%s.old", host->health_log_filename);
45
+ snprintfz(old_filename, FILENAME_MAX, "%s.old", host->health.health_log_filename);
46
47
if(unlink(old_filename) == -1 && errno != ENOENT)
48
error("HEALTH [%s]: cannot remove old alarms log file '%s'", rrdhost_hostname(host), old_filename);
49
50
- if(link(host->health_log_filename, old_filename) == -1 && errno != ENOENT)
51
- error("HEALTH [%s]: cannot move file '%s' to '%s'.", rrdhost_hostname(host), host->health_log_filename, old_filename);
50
+ if(link(host->health.health_log_filename, old_filename) == -1 && errno != ENOENT)
51
+ error("HEALTH [%s]: cannot move file '%s' to '%s'.", rrdhost_hostname(host), host->health.health_log_filename, old_filename);
52
53
- if(unlink(host->health_log_filename) == -1 && errno != ENOENT)
54
- error("HEALTH [%s]: cannot remove old alarms log file '%s'", rrdhost_hostname(host), host->health_log_filename);
53
+ if(unlink(host->health.health_log_filename) == -1 && errno != ENOENT)
54
+ error("HEALTH [%s]: cannot remove old alarms log file '%s'", rrdhost_hostname(host), host->health.health_log_filename);
55
56
// open it with truncate
57
- host->health_log_fp = fopen(host->health_log_filename, "w");
57
+ host->health.health_log_fp = fopen(host->health.health_log_filename, "w");
58
59
- if(host->health_log_fp)
60
- fclose(host->health_log_fp);
59
+ if(host->health.health_log_fp)
60
+ fclose(host->health.health_log_fp);
61
else
62
- error("HEALTH [%s]: cannot truncate health log '%s'", rrdhost_hostname(host), host->health_log_filename);
62
+ error("HEALTH [%s]: cannot truncate health log '%s'", rrdhost_hostname(host), host->health.health_log_filename);
63
64
- host->health_log_fp = NULL;
64
+ host->health.health_log_fp = NULL;
65
66
- host->health_log_entries_written = 0;
66
+ host->health.health_log_entries_written = 0;
67
health_alarm_log_open(host);
68
}
69
}
@@ -72,17 +72,17 @@ static inline void health_log_rotate(RRDHOST *host) {
72
inline void health_label_log_save(RRDHOST *host) {
73
health_log_rotate(host);
74
75
- if(unlikely(host->health_log_fp)) {
75
+ if(unlikely(host->health.health_log_fp)) {
76
BUFFER *wb = buffer_create(1024);
77
78
rrdlabels_to_buffer(localhost->rrdlabels, wb, "", "=", "", "\t ", NULL, NULL, NULL, NULL);
79
char *write = (char *) buffer_tostring(wb);
80
81
- if (unlikely(fprintf(host->health_log_fp, "L\t%s", write) < 0))
81
+ if (unlikely(fprintf(host->health.health_log_fp, "L\t%s", write) < 0))
82
error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.",
83
- rrdhost_hostname(host), host->health_log_filename);
83
+ rrdhost_hostname(host), host->health.health_log_filename);
84
else
85
- host->health_log_entries_written++;
85
+ host->health.health_log_entries_written++;
86
87
buffer_free(wb);
88
}
@@ -90,8 +90,8 @@ inline void health_label_log_save(RRDHOST *host) {
90
91
inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
92
health_log_rotate(host);
93
- if(unlikely(host->health_log_fp)) {
94
- if(unlikely(fprintf(host->health_log_fp
93
+ if(unlikely(host->health.health_log_fp)) {
94
+ if(unlikely(fprintf(host->health.health_log_fp
95
, "%c\t%s"
96
"\t%08x\t%08x\t%08x\t%08x\t%08x"
97
"\t%08x\t%08x\t%08x"
@@ -139,10 +139,10 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
139
, (ae->component)?ae_component(ae):"Unknown"
140
, (ae->type)?ae_type(ae):"Unknown"
141
) < 0))
142
- error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", rrdhost_hostname(host), host->health_log_filename);
142
+ error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", rrdhost_hostname(host), host->health.health_log_filename);
143
else {
144
ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
145
- host->health_log_entries_written++;
145
+ host->health.health_log_entries_written++;
146
}
147
}else
148
sql_health_alarm_log_save(host, ae);
@@ -193,7 +193,7 @@ static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char
193
netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
194
195
while((s = fgets_trim_len(buf, 65536, fp, &len))) {
196
- host->health_log_entries_written++;
196
+ host->health.health_log_entries_written++;
197
line++;
198
199
int max_entries = 33, entries = 0;
@@ -419,7 +419,7 @@ inline void health_alarm_log_load(RRDHOST *host) {
419
health_alarm_log_close(host);
420
421
char filename[FILENAME_MAX + 1];
422
- snprintfz(filename, FILENAME_MAX, "%s.old", host->health_log_filename);
422
+ snprintfz(filename, FILENAME_MAX, "%s.old", host->health.health_log_filename);
423
FILE *fp = fopen(filename, "r");
424
if(!fp)
425
error("HEALTH [%s]: cannot open health file: %s", rrdhost_hostname(host), filename);
@@ -428,12 +428,12 @@ inline void health_alarm_log_load(RRDHOST *host) {
428
fclose(fp);
429
}
430
431
- host->health_log_entries_written = 0;
432
- fp = fopen(host->health_log_filename, "r");
431
+ host->health.health_log_entries_written = 0;
432
+ fp = fopen(host->health.health_log_filename, "r");
433
if(!fp)
434
- error("HEALTH [%s]: cannot open health file: %s", rrdhost_hostname(host), host->health_log_filename);
434
+ error("HEALTH [%s]: cannot open health file: %s", rrdhost_hostname(host), host->health.health_log_filename);
435
else {
436
- health_alarm_log_read(host, fp, host->health_log_filename);
436
+ health_alarm_log_read(host, fp, host->health.health_log_filename);
437
fclose(fp);
438
}
439
}
streaming/receiver.c
+2
-2
@@ -449,7 +449,7 @@ bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
449
450
if (rpt->config.health_enabled != CONFIG_BOOLEAN_NO) {
451
if (rpt->config.alarms_delay > 0) {
452
- host->health_delay_up_to = now_realtime_sec() + rpt->config.alarms_delay;
452
+ host->health.health_delay_up_to = now_realtime_sec() + rpt->config.alarms_delay;
453
log_health(
454
"[%s]: Postponing health checks for %" PRId64 " seconds, because it was just connected.",
455
rrdhost_hostname(host),
@@ -491,7 +491,7 @@ static void rrdhost_clear_receiver(struct receiver_state *rpt) {
491
host->child_disconnected_time = now_realtime_sec();
492
493
if (rpt->config.health_enabled == CONFIG_BOOLEAN_AUTO)
494
- host->health_enabled = 0;
494
+ host->health.health_enabled = 0;
495
496
rrdpush_sender_thread_stop(host, "RECEIVER LEFT", false);
497