Health thread per host (#13712)
* Rebased * rebased * health_execute_pending_updates -> health_execute_delayed_initializations * fix labels for current host only * missing bracket * misc fixes, reload health for disconnected hosts * remove volatile, add comment
Emmanuel Vasilakis committed
Oct 19, 2022 at 18:30 UTC
42e85b5a092b18881501b0fe76f91e4969dca088
14 files changed
+668
-567
aclk/aclk.c
-2
@@ -47,8 +47,6 @@ time_t last_disconnect_time = 0;
47
time_t next_connection_attempt = 0;
48
float last_backoff_value = 0;
49
50
-int aclk_alert_reloaded = 0; //1 on health log exchange, and again on health_reload
51
-
50
time_t aclk_block_until = 0;
51
52
#ifdef ENABLE_ACLK
aclk/aclk.h
-1
@@ -14,7 +14,6 @@
14
#endif /* ENABLE_ACLK */
15
16
extern int aclk_connected;
17
-extern int aclk_alert_reloaded;
17
extern int use_mqtt_5;
18
extern int aclk_ctx_based;
19
extern int aclk_disable_runtime;
daemon/main.c
+3
@@ -409,6 +409,9 @@ static void log_init(void) {
409
snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir);
410
stdaccess_filename = config_get(CONFIG_SECTION_LOGS, "access", filename);
411
412
+ snprintfz(filename, FILENAME_MAX, "%s/health.log", netdata_configured_log_dir);
413
+ stdhealth_filename = config_get(CONFIG_SECTION_LOGS, "health", filename);
414
+
415
#ifdef ENABLE_ACLK
416
aclklog_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "conversation log", CONFIG_BOOLEAN_NO);
417
if (aclklog_enabled) {
daemon/static_threads.c
-9
@@ -63,15 +63,6 @@ const struct netdata_static_thread static_threads_common[] = {
63
.init_routine = NULL,
64
.start_routine = global_statistics_main
65
},
66
- {
67
- .name = "HEALTH",
68
- .config_section = NULL,
69
- .config_name = NULL,
70
- .enabled = 1,
71
- .thread = NULL,
72
- .init_routine = NULL,
73
- .start_routine = health_main
74
- },
66
{
67
.name = "PLUGINSD",
68
.config_section = NULL,
database/rrd.h
+12
-10
@@ -948,16 +948,18 @@ struct rrdhost {
948
// ------------------------------------------------------------------------
949
// health monitoring options
950
951
- unsigned int health_enabled; // 1 when this host has health enabled
952
- time_t health_delay_up_to; // a timestamp to delay alarms processing up to
953
- STRING *health_default_exec; // the full path of the alarms notifications program
954
- STRING *health_default_recipient; // the default recipient for all alarms
955
- char *health_log_filename; // the alarms event log filename
956
- size_t health_log_entries_written; // the number of alarm events written to the alarms event log
957
- FILE *health_log_fp; // the FILE pointer to the open alarms event log file
958
- uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications
959
- uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications
960
-
951
+ unsigned int health_enabled; // 1 when this host has health enabled
952
+ bool health_spawn; // true when health thread is running
953
+ netdata_thread_t health_thread; // the health thread
954
+ unsigned int aclk_alert_reloaded; // 1 on thread start and health reload, 0 after removed are sent
955
+ time_t health_delay_up_to; // a timestamp to delay alarms processing up to
956
+ STRING *health_default_exec; // the full path of the alarms notifications program
957
+ STRING *health_default_recipient; // the default recipient for all alarms
958
+ char *health_log_filename; // the alarms event log filename
959
+ size_t health_log_entries_written; // the number of alarm events written to the alarms event log
960
+ FILE *health_log_fp; // the FILE pointer to the open alarms event log file
961
+ uint32_t health_default_warn_repeat_every; // the default value for the interval between repeating warning notifications
962
+ uint32_t health_default_crit_repeat_every; // the default value for the interval between repeating critical notifications
963
964
// all RRDCALCs are primarily allocated and linked here
965
DICTIONARY *rrdcalc_root_index;
database/rrdhost.c
+13
-80
@@ -231,82 +231,6 @@ static void rrdhost_initialize_rrdpush_sender(RRDHOST *host,
231
rrdhost_option_clear(host, RRDHOST_OPTION_SENDER_ENABLED);
232
}
233
234
-static void rrdhost_initialize_health(RRDHOST *host,
235
- int is_localhost
236
- ) {
237
- if(!host->health_enabled || rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)) return;
238
- rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH);
239
-
240
- rrdfamily_index_init(host);
241
- rrdcalctemplate_index_init(host);
242
- rrdcalc_rrdhost_index_init(host);
243
-
244
- host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
245
- host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");
246
-
247
- host->health_log.next_log_id = 1;
248
- host->health_log.next_alarm_id = 1;
249
- host->health_log.max = 1000;
250
- host->health_log.next_log_id = (uint32_t)now_realtime_sec();
251
- host->health_log.next_alarm_id = 0;
252
-
253
- long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max);
254
- if(n < 10) {
255
- error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", rrdhost_hostname(host), n, host->health_log.max);
256
- config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max);
257
- }
258
- else
259
- host->health_log.max = (unsigned int)n;
260
-
261
- netdata_rwlock_init(&host->health_log.alarm_log_rwlock);
262
-
263
- char filename[FILENAME_MAX + 1];
264
-
265
- if(!is_localhost) {
266
- int r = mkdir(host->varlib_dir, 0775);
267
- if (r != 0 && errno != EEXIST)
268
- error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir);
269
- }
270
-
271
- {
272
- snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir);
273
- int r = mkdir(filename, 0775);
274
- if(r != 0 && errno != EEXIST)
275
- error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename);
276
- }
277
-
278
- snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir);
279
- host->health_log_filename = strdupz(filename);
280
-
281
- snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir);
282
- host->health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
283
- host->health_default_recipient = string_strdupz("root");
284
-
285
- // ------------------------------------------------------------------------
286
- // load health configuration
287
-
288
- health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
289
-
290
- if (!file_is_migrated(host->health_log_filename)) {
291
- int rc = sql_create_health_log_table(host);
292
- if (unlikely(rc)) {
293
- error_report("Failed to create health log table in the database");
294
- health_alarm_log_load(host);
295
- health_alarm_log_open(host);
296
- }
297
- else {
298
- health_alarm_log_load(host);
299
- add_migrated_file(host->health_log_filename, 0);
300
- }
301
- } else {
302
- // TODO: This needs to go to the metadata thread
303
- // Health should wait before accessing the table (needs to be created by the metadata thread
304
- sql_create_health_log_table(host);
305
- sql_health_alarm_log_load(host);
306
- }
307
-}
308
-
309
-
234
RRDHOST *rrdhost_create(const char *hostname,
235
const char *registry_hostname,
236
const char *guid,
@@ -423,7 +347,12 @@ int is_legacy = 1;
347
else
348
error_report("Host machine GUID %s is not valid", host->machine_guid);
349
426
- rrdhost_initialize_health(host, is_localhost);
350
+ rrdfamily_index_init(host);
351
+ rrdcalctemplate_index_init(host);
352
+ rrdcalc_rrdhost_index_init(host);
353
+
354
+ if (health_enabled)
355
+ health_thread_spawn(host);
356
357
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
358
#ifdef ENABLE_DBENGINE
@@ -640,7 +569,9 @@ void rrdhost_update(RRDHOST *host
569
rrdpush_api_key,
570
rrdpush_send_charts_matching);
571
643
- rrdhost_initialize_health(host, host == localhost);
572
+ rrdfamily_index_init(host);
573
+ rrdcalctemplate_index_init(host);
574
+ rrdcalc_rrdhost_index_init(host);
575
576
rrd_hosts_available++;
577
ml_new_host(host);
@@ -648,6 +579,9 @@ void rrdhost_update(RRDHOST *host
579
info("Host %s is not in archived mode anymore", rrdhost_hostname(host));
580
}
581
582
+ if (health_enabled)
583
+ health_thread_spawn(host);
584
+
585
return;
586
}
587
@@ -916,8 +850,6 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
850
}
851
}
852
919
- health_init();
920
-
853
unittest:
854
metadata_sync_init();
855
debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
@@ -1119,6 +1051,7 @@ void rrdhost_free(RRDHOST *host, bool force) {
1051
1052
freez(host->exporting_flags);
1053
1054
+ health_thread_stop(host);
1055
health_alarm_log_free(host);
1056
1057
#ifdef ENABLE_DBENGINE
database/sqlite/sqlite_aclk_alert.c
-2
@@ -532,8 +532,6 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
532
533
freez(claim_id);
534
buffer_free(sql);
535
-
536
- aclk_alert_reloaded = 1;
535
#endif
536
537
return;
database/sqlite/sqlite_health.c
+1
-1
@@ -806,7 +806,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
806
if (unlikely(!host->health_log.next_alarm_id || host->health_log.next_alarm_id <= host->health_max_alarm_id))
807
host->health_log.next_alarm_id = host->health_max_alarm_id + 1;
808
809
- info("HEALTH [%s]: Table health_log_%s, loaded %zd alarm entries, errors in %zd entries.", rrdhost_hostname(host), uuid_str, loaded, errored);
809
+ log_health("[%s]: Table health_log_%s, loaded %zd alarm entries, errors in %zd entries.", rrdhost_hostname(host), uuid_str, loaded, errored);
810
811
ret = sqlite3_finalize(res);
812
if (unlikely(ret != SQLITE_OK))
health/health.c
+575
-455
@@ -22,7 +22,7 @@ unsigned int default_health_enabled = 1;
22
char *silencers_filename;
23
24
// the queue of executed alarm notifications that haven't been waited for yet
25
-static struct {
25
+static __thread struct {
26
ALARM_ENTRY *head; // oldest
27
ALARM_ENTRY *tail; // latest
28
} alarm_notifications_in_progress = {NULL, NULL};
@@ -135,22 +135,6 @@ static void health_silencers_init(void) {
135
}
136
}
137
138
-/**
139
- * Health Init
140
- *
141
- * Initialize the health thread.
142
- */
143
-void health_init(void) {
144
- debug(D_HEALTH, "Health configuration initializing");
145
-
146
- if(!(default_health_enabled = (unsigned int)config_get_boolean(CONFIG_SECTION_HEALTH, "enabled", default_health_enabled))) {
147
- debug(D_HEALTH, "Health is disabled.");
148
- return;
149
- }
150
-
151
- health_silencers_init();
152
-}
153
-
138
// ----------------------------------------------------------------------------
139
// re-load health configuration
140
@@ -162,9 +146,11 @@ void health_init(void) {
146
* @param host the structure of the host that the function will reload the configuration.
147
*/
148
static void health_reload_host(RRDHOST *host) {
165
- if(unlikely(!host->health_enabled))
149
+ if(unlikely(!host->health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH))
150
return;
151
152
+ log_health("[%s]: Reloading health.", rrdhost_hostname(host));
153
+
154
char *user_path = health_user_config_dir();
155
char *stock_path = health_stock_config_dir();
156
@@ -204,7 +190,7 @@ static void health_reload_host(RRDHOST *host) {
190
rrdcalctemplate_link_matching_templates_to_rrdset(st);
191
}
192
rrdset_foreach_done(st);
207
-
193
+ host->aclk_alert_reloaded = 1;
194
}
195
196
/**
@@ -222,11 +208,6 @@ void health_reload(void) {
208
health_reload_host(host);
209
210
rrd_unlock();
225
-#ifdef ENABLE_ACLK
226
- if (netdata_cloud_setting) {
227
- aclk_alert_reloaded = 1;
228
- }
229
-#endif
211
}
212
213
// ----------------------------------------------------------------------------
@@ -261,6 +242,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
242
if(unlikely(ae->new_status <= RRDCALC_STATUS_CLEAR && (ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) {
243
// do not send notifications for disabled statuses
244
debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)", ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
245
+ log_health("[%s]: Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
246
// mark it as run, so that we will send the same alarm if it happens again
247
goto done;
248
}
@@ -282,6 +264,8 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
264
// don't send the notification for the same status again
265
debug(D_HEALTH, "Health not sending again notification for alarm '%s.%s' status %s", ae_chart_name(ae), ae_name(ae)
266
, rrdcalc_status2string(ae->new_status));
267
+ log_health("[%s]: Health not sending again notification for alarm '%s.%s' status %s", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae)
268
+ , rrdcalc_status2string(ae->new_status));
269
goto done;
270
}
271
}
@@ -300,10 +284,12 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
284
285
// Check if alarm notifications are silenced
286
if (ae->flags & HEALTH_ENTRY_FLAG_SILENCED) {
303
- info("Health not sending notification for alarm '%s.%s' status %s (command API has disabled notifications)", ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
287
+ log_health("[%s]: Health not sending notification for alarm '%s.%s' status %s (command API has disabled notifications)", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
288
goto done;
289
}
290
291
+ log_health("[%s]: Sending notification for alarm '%s.%s' status %s.", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), rrdcalc_status2string(ae->new_status));
292
+
293
static char command_to_run[ALARM_EXEC_COMMAND_LENGTH + 1];
294
295
const char *exec = (ae->exec) ? ae_exec(ae) : string2str(host->health_default_exec);
@@ -577,7 +563,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
563
}
564
565
static inline int check_if_resumed_from_suspension(void) {
580
- static usec_t last_realtime = 0, last_monotonic = 0;
566
+ static __thread usec_t last_realtime = 0, last_monotonic = 0;
567
usec_t realtime = now_realtime_usec(), monotonic = now_monotonic_usec();
568
int ret = 0;
569
@@ -593,15 +579,123 @@ static inline int check_if_resumed_from_suspension(void) {
579
return ret;
580
}
581
596
-static void health_main_cleanup(void *ptr) {
582
+static void health_thread_cleanup(void *ptr) {
583
worker_unregister();
584
599
- struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
600
- static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
585
+ struct health_state *h = ptr;
586
+ h->host->health_spawn = 0;
587
602
- info("cleaning up...");
588
+ netdata_thread_detach(netdata_thread_self());
589
+ log_health("[%s]: Health thread ended.", rrdhost_hostname(h->host));
590
+ debug(D_HEALTH, "HEALTH %s: Health thread ended.", rrdhost_hostname(h->host));
591
+}
592
604
- static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
593
+void health_thread_stop(RRDHOST *host) {
594
+ if(host->health_spawn) {
595
+ log_health("[%s]: Signaling health thread to stop...", rrdhost_hostname(host));
596
+
597
+ // signal it to cancel
598
+ netdata_thread_cancel(host->health_thread);
599
+ }
600
+}
601
+
602
+static void initialize_health(RRDHOST *host, int is_localhost) {
603
+ if(!host->health_enabled || rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)) return;
604
+ rrdhost_flag_set(host, RRDHOST_FLAG_INITIALIZED_HEALTH);
605
+
606
+ log_health("[%s]: Initializing health.", rrdhost_hostname(host));
607
+
608
+ host->health_default_warn_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat warning", "never");
609
+ host->health_default_crit_repeat_every = config_get_duration(CONFIG_SECTION_HEALTH, "default repeat critical", "never");
610
+
611
+ host->health_log.next_log_id = 1;
612
+ host->health_log.next_alarm_id = 1;
613
+ host->health_log.max = 1000;
614
+ host->health_log.next_log_id = (uint32_t)now_realtime_sec();
615
+ host->health_log.next_alarm_id = 0;
616
+
617
+ long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max);
618
+ if(n < 10) {
619
+ error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", rrdhost_hostname(host), n, host->health_log.max);
620
+ config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max);
621
+ }
622
+ else
623
+ host->health_log.max = (unsigned int)n;
624
+
625
+ netdata_rwlock_init(&host->health_log.alarm_log_rwlock);
626
+
627
+ char filename[FILENAME_MAX + 1];
628
+
629
+ if(!is_localhost) {
630
+ int r = mkdir(host->varlib_dir, 0775);
631
+ if (r != 0 && errno != EEXIST)
632
+ error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), host->varlib_dir);
633
+ }
634
+
635
+ {
636
+ snprintfz(filename, FILENAME_MAX, "%s/health", host->varlib_dir);
637
+ int r = mkdir(filename, 0775);
638
+ if(r != 0 && errno != EEXIST)
639
+ error("Host '%s': cannot create directory '%s'", rrdhost_hostname(host), filename);
640
+ }
641
+ snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir);
642
+ host->health_log_filename = strdupz(filename);
643
+
644
+ snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_primary_plugins_dir);
645
+ host->health_default_exec = string_strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename));
646
+ host->health_default_recipient = string_strdupz("root");
647
+
648
+ if (!file_is_migrated(host->health_log_filename)) {
649
+ int rc = sql_create_health_log_table(host);
650
+ if (unlikely(rc)) {
651
+ log_health("[%s]: Failed to create health log table in the database", rrdhost_hostname(host));
652
+ health_alarm_log_load(host);
653
+ health_alarm_log_open(host);
654
+ }
655
+ else {
656
+ health_alarm_log_load(host);
657
+ add_migrated_file(host->health_log_filename, 0);
658
+ }
659
+ } else {
660
+ // TODO: This needs to go to the metadata thread
661
+ // Health should wait before accessing the table (needs to be created by the metadata thread)
662
+ sql_create_health_log_table(host);
663
+ sql_health_alarm_log_load(host);
664
+ }
665
+
666
+ // ------------------------------------------------------------------------
667
+ // load health configuration
668
+
669
+ health_readdir(host, health_user_config_dir(), health_stock_config_dir(), NULL);
670
+
671
+ // link the loaded alarms to their charts
672
+ RRDSET *st;
673
+ rrdset_foreach_write(st, host) {
674
+ if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))
675
+ continue;
676
+
677
+ rrdcalc_link_matching_alerts_to_rrdset(st);
678
+ rrdcalctemplate_link_matching_templates_to_rrdset(st);
679
+ }
680
+ rrdset_foreach_done(st);
681
+
682
+ //Discard alarms with labels that do not apply to host
683
+ rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
684
+
685
+ health_silencers_init();
686
+}
687
+
688
+static void health_sleep(time_t next_run, unsigned int loop) {
689
+ time_t now = now_realtime_sec();
690
+ if(now < next_run) {
691
+ worker_is_idle();
692
+ debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now));
693
+ sleep_usec(USEC_PER_SEC * (usec_t) (next_run - now));
694
+ now = now_realtime_sec();
695
+ }
696
+ else {
697
+ debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
698
+ }
699
}
700
701
static SILENCE_TYPE check_silenced(RRDCALC *rc, const char *host, SILENCERS *silencers) {
@@ -756,7 +850,11 @@ void *health_main(void *ptr) {
850
worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDSET, "rrdset init");
851
worker_register_job_name(WORKER_HEALTH_JOB_DELAYED_INIT_RRDDIM, "rrddim init");
852
759
- netdata_thread_cleanup_push(health_main_cleanup, ptr);
853
+ struct health_state *h = ptr;
854
+ netdata_thread_cleanup_push(health_thread_cleanup, ptr);
855
+
856
+ RRDHOST *host = h->host;
857
+ initialize_health(host, host == localhost);
858
859
int min_run_every = (int)config_get_number(CONFIG_SECTION_HEALTH, "run at least every seconds", 10);
860
if(min_run_every < 1) min_run_every = 1;
@@ -766,7 +864,11 @@ void *health_main(void *ptr) {
864
time_t now = now_realtime_sec();
865
time_t hibernation_delay = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60);
866
769
- rrdcalc_delete_alerts_not_matching_host_labels_from_all_hosts();
867
+ bool health_running_logged = false;
868
+
869
+ rrdhost_rdlock(host); //CHECK
870
+ rrdcalc_delete_alerts_not_matching_host_labels_from_this_host(host);
871
+ rrdhost_unlock(host);
872
873
unsigned int loop = 0;
874
#ifdef ENABLE_ACLK
@@ -776,6 +878,7 @@ void *health_main(void *ptr) {
878
loop++;
879
debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
880
881
+ now = now_realtime_sec();
882
int runnable = 0, apply_hibernation_delay = 0;
883
time_t next_run = now + min_run_every;
884
RRDCALC *rc;
@@ -783,491 +886,503 @@ void *health_main(void *ptr) {
886
if (unlikely(check_if_resumed_from_suspension())) {
887
apply_hibernation_delay = 1;
888
786
- info(
787
- "Postponing alarm checks for %"PRId64" seconds, "
788
- "because it seems that the system was just resumed from suspension.",
789
- (int64_t)hibernation_delay);
889
+ log_health(
890
+ "[%s]: Postponing alarm checks for %"PRId64" seconds, "
891
+ "because it seems that the system was just resumed from suspension.",
892
+ rrdhost_hostname(host),
893
+ (int64_t)hibernation_delay);
894
}
895
896
if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) {
793
- static int logged=0;
897
+ static __thread int logged=0;
898
if (!logged) {
795
- info("Skipping health checks, because all alarms are disabled via a %s command.",
796
- HEALTH_CMDAPI_CMD_DISABLEALL);
899
+ log_health("[%s]: Skipping health checks, because all alarms are disabled via a %s command.",
900
+ rrdhost_hostname(host),
901
+ HEALTH_CMDAPI_CMD_DISABLEALL);
902
logged = 1;
903
}
904
}
905
906
#ifdef ENABLE_ACLK
802
- if (aclk_alert_reloaded && !marked_aclk_reload_loop)
907
+ if (host->aclk_alert_reloaded && !marked_aclk_reload_loop)
908
marked_aclk_reload_loop = loop;
909
#endif
910
806
- worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
807
- rrd_rdlock();
808
-
809
- RRDHOST *host;
810
- rrdhost_foreach_read(host) {
811
- if (unlikely(!host->health_enabled))
812
- continue;
911
+ if (unlikely(apply_hibernation_delay)) {
912
+ log_health(
913
+ "[%s]: Postponing health checks for %"PRId64" seconds.",
914
+ rrdhost_hostname(host),
915
+ (int64_t)hibernation_delay);
916
814
- if (unlikely(apply_hibernation_delay)) {
815
- info(
816
- "Postponing health checks for %"PRId64" seconds, on host '%s'.",
817
- (int64_t)hibernation_delay,
818
- rrdhost_hostname(host));
917
+ host->health_delay_up_to = now + hibernation_delay;
918
+ next_run = now + hibernation_delay;
919
+ health_sleep(next_run, loop);
920
+ }
921
820
- host->health_delay_up_to = now + hibernation_delay;
922
+ if (unlikely(host->health_delay_up_to)) {
923
+ if (unlikely(now < host->health_delay_up_to)) {
924
+ next_run = host->health_delay_up_to;
925
+ health_sleep(next_run, loop);
926
+ continue;
927
}
928
823
- if (unlikely(host->health_delay_up_to)) {
824
- if (unlikely(now < host->health_delay_up_to))
825
- continue;
929
+ log_health("[%s]: Resuming health checks after delay.", rrdhost_hostname(host));
930
+ host->health_delay_up_to = 0;
931
+ }
932
+
933
+ if (unlikely(!host->health_enabled)) {
934
+ health_thread_stop(host);
935
+ }
936
827
- info("Resuming health checks on host '%s'.", rrdhost_hostname(host));
828
- host->health_delay_up_to = 0;
937
+ // wait until cleanup of obsolete charts on children is complete
938
+ if (host != localhost) {
939
+ if (unlikely(host->trigger_chart_obsoletion_check == 1)) {
940
+ log_health("[%s]: Waiting for chart obsoletion check.", rrdhost_hostname(host));
941
+ health_sleep(next_run, loop);
942
+ continue;
943
}
944
+ }
945
831
- // wait until cleanup of obsolete charts on children is complete
832
- if (host != localhost)
833
- if (unlikely(host->trigger_chart_obsoletion_check == 1))
834
- continue;
946
+ if (!health_running_logged) {
947
+ log_health("[%s]: Health is running.", rrdhost_hostname(host));
948
+ health_running_logged = true;
949
+ }
950
836
- if(likely(!host->health_log_fp) && (loop == 1 || loop % cleanup_sql_every_loop == 0))
837
- sql_health_alarm_log_cleanup(host);
951
+ if(likely(!host->health_log_fp) && (loop == 1 || loop % cleanup_sql_every_loop == 0))
952
+ sql_health_alarm_log_cleanup(host);
953
839
- health_execute_delayed_initializations(host);
954
+ health_execute_delayed_initializations(host);
955
841
- worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
956
+ worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
957
843
- // the first loop is to lookup values from the db
844
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
958
+ // the first loop is to lookup values from the db
959
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
960
846
- rrdcalc_update_info_using_rrdset_labels(rc);
961
+ rrdcalc_update_info_using_rrdset_labels(rc);
962
848
- if (update_disabled_silenced(host, rc))
849
- continue;
963
+ if (update_disabled_silenced(host, rc))
964
+ continue;
965
+
966
+ // create an alert removed event if the chart is obsolete and
967
+ // has stopped being collected for 60 seconds
968
+ if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
969
+ rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
970
+ now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
971
+ if (!rrdcalc_isrepeating(rc)) {
972
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
973
+ time_t now = now_realtime_sec();
974
+
975
+ ALARM_ENTRY *ae = health_create_alarm_entry(
976
+ host,
977
+ rc->id,
978
+ rc->next_event_id++,
979
+ rc->config_hash_id,
980
+ now,
981
+ rc->name,
982
+ rc->rrdset->id,
983
+ rc->rrdset->context,
984
+ rc->rrdset->family,
985
+ rc->classification,
986
+ rc->component,
987
+ rc->type,
988
+ rc->exec,
989
+ rc->recipient,
990
+ now - rc->last_status_change,
991
+ rc->value,
992
+ NAN,
993
+ rc->status,
994
+ RRDCALC_STATUS_REMOVED,
995
+ rc->source,
996
+ rc->units,
997
+ rc->info,
998
+ 0,
999
+ rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
1000
+
1001
+ if (ae) {
1002
+ health_alarm_log_add_entry(host, ae);
1003
+ rc->old_status = rc->status;
1004
+ rc->status = RRDCALC_STATUS_REMOVED;
1005
+ rc->last_status_change = now;
1006
+ rc->last_updated = now;
1007
+ rc->value = NAN;
1008
851
- // create an alert removed event if the chart is obsolete and
852
- // has stopped being collected for 60 seconds
853
- if (unlikely(rc->rrdset && rc->status != RRDCALC_STATUS_REMOVED &&
854
- rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
855
- now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
856
- if (!rrdcalc_isrepeating(rc)) {
857
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
858
- time_t now = now_realtime_sec();
859
-
860
- ALARM_ENTRY *ae = health_create_alarm_entry(
861
- host,
862
- rc->id,
863
- rc->next_event_id++,
864
- rc->config_hash_id,
865
- now,
866
- rc->name,
867
- rc->rrdset->id,
868
- rc->rrdset->context,
869
- rc->rrdset->family,
870
- rc->classification,
871
- rc->component,
872
- rc->type,
873
- rc->exec,
874
- rc->recipient,
875
- now - rc->last_status_change,
876
- rc->value,
877
- NAN,
878
- rc->status,
879
- RRDCALC_STATUS_REMOVED,
880
- rc->source,
881
- rc->units,
882
- rc->info,
883
- 0,
884
- rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
885
-
886
- if (ae) {
887
- health_alarm_log_add_entry(host, ae);
888
- rc->old_status = rc->status;
889
- rc->status = RRDCALC_STATUS_REMOVED;
890
- rc->last_status_change = now;
891
- rc->last_updated = now;
892
- rc->value = NAN;
1009
#ifdef ENABLE_ACLK
894
- if (netdata_cloud_setting && likely(!aclk_alert_reloaded))
895
- sql_queue_alarm_to_aclk(host, ae, 1);
1010
+ if (netdata_cloud_setting && likely(!host->aclk_alert_reloaded))
1011
+ sql_queue_alarm_to_aclk(host, ae, 1);
1012
#endif
897
- }
1013
}
1014
}
1015
+ }
1016
901
- if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
902
- if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
903
- rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
904
- continue;
905
- }
906
-
907
- runnable++;
908
- rc->old_value = rc->value;
909
- rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
910
-
911
- // ------------------------------------------------------------
912
- // if there is database lookup, do it
913
-
914
- if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
915
- worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
916
-
917
- /* time_t old_db_timestamp = rc->db_before; */
918
- int value_is_null = 0;
1017
+ if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run))) {
1018
+ if (unlikely(rc->run_flags & RRDCALC_FLAG_RUNNABLE))
1019
+ rc->run_flags &= ~RRDCALC_FLAG_RUNNABLE;
1020
+ continue;
1021
+ }
1022
920
- int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
921
- rc->after, rc->before, rc->group, NULL,
922
- 0, rc->options,
923
- &rc->db_after,&rc->db_before,
924
- NULL, NULL, NULL,
925
- &value_is_null, NULL, 0, 0);
1023
+ runnable++;
1024
+ rc->old_value = rc->value;
1025
+ rc->run_flags |= RRDCALC_FLAG_RUNNABLE;
1026
+
1027
+ // ------------------------------------------------------------
1028
+ // if there is database lookup, do it
1029
+
1030
+ if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
1031
+ worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
1032
+
1033
+ /* time_t old_db_timestamp = rc->db_before; */
1034
+ int value_is_null = 0;
1035
+
1036
+ int ret = rrdset2value_api_v1(rc->rrdset, NULL, &rc->value, rrdcalc_dimensions(rc), 1,
1037
+ rc->after, rc->before, rc->group, NULL,
1038
+ 0, rc->options,
1039
+ &rc->db_after,&rc->db_before,
1040
+ NULL, NULL, NULL,
1041
+ &value_is_null, NULL, 0, 0);
1042
+
1043
+ if (unlikely(ret != 200)) {
1044
+ // database lookup failed
1045
+ rc->value = NAN;
1046
+ rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
1047
+
1048
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
1049
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
1050
+ );
1051
+ } else
1052
+ rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
1053
+
1054
+ /* - RRDCALC_FLAG_DB_STALE not currently used
1055
+ if (unlikely(old_db_timestamp == rc->db_before)) {
1056
+ // database is stale
1057
+
1058
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
1059
+
1060
+ if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
1061
+ rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
1062
+ error("Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
1063
+ }
1064
+ }
1065
+ else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
1066
+ rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
1067
+ */
1068
+
1069
+ if (unlikely(value_is_null)) {
1070
+ // collected value is null
1071
+ rc->value = NAN;
1072
+ rc->run_flags |= RRDCALC_FLAG_DB_NAN;
1073
+
1074
+ debug(D_HEALTH,
1075
+ "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
1076
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
1077
+ );
1078
+ } else
1079
+ rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
1080
+
1081
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
1082
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
1083
+ );
1084
+ }
1085
927
- if (unlikely(ret != 200)) {
928
- // database lookup failed
929
- rc->value = NAN;
930
- rc->run_flags |= RRDCALC_FLAG_DB_ERROR;
1086
+ // ------------------------------------------------------------
1087
+ // if there is calculation expression, run it
1088
932
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup returned error %d",
933
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), ret
934
- );
935
- } else
936
- rc->run_flags &= ~RRDCALC_FLAG_DB_ERROR;
1089
+ if (unlikely(rc->calculation)) {
1090
+ worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);
1091
938
- /* - RRDCALC_FLAG_DB_STALE not currently used
939
- if (unlikely(old_db_timestamp == rc->db_before)) {
940
- // database is stale
1092
+ if (unlikely(!expression_evaluate(rc->calculation))) {
1093
+ // calculation failed
1094
+ rc->value = NAN;
1095
+ rc->run_flags |= RRDCALC_FLAG_CALC_ERROR;
1096
942
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
1097
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",
1098
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1099
+ rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)
1100
+ );
1101
+ } else {
1102
+ rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR;
1103
944
- if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
945
- rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
946
- error("Health on host '%s', alarm '%s.%s': database is stale", host->hostname, rc->chart?rc->chart:"NOCHART", rc->name);
947
- }
948
- }
949
- else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
950
- rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
951
- */
1104
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
1105
+ NETDATA_DOUBLE_FORMAT
1106
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1107
+ rc->calculation->parsed_as, rc->calculation->result,
1108
+ buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc)
1109
+ );
1110
953
- if (unlikely(value_is_null)) {
954
- // collected value is null
955
- rc->value = NAN;
956
- rc->run_flags |= RRDCALC_FLAG_DB_NAN;
1111
+ rc->value = rc->calculation->result;
1112
+ }
1113
+ }
1114
+ }
1115
+ foreach_rrdcalc_in_rrdhost_done(rc);
1116
958
- debug(D_HEALTH,
959
- "Health on host '%s', alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
960
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc)
961
- );
962
- } else
963
- rc->run_flags &= ~RRDCALC_FLAG_DB_NAN;
1117
+ if (unlikely(runnable && !netdata_exit)) {
1118
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
1119
+ if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
1120
+ continue;
1121
965
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': database lookup gave value " NETDATA_DOUBLE_FORMAT,
966
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc), rc->value
967
- );
1122
+ if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
1123
+ continue;
1124
}
1125
+ RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
1126
+ RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
1127
970
- // ------------------------------------------------------------
971
- // if there is calculation expression, run it
1128
+ // --------------------------------------------------------
1129
+ // check the warning expression
1130
973
- if (unlikely(rc->calculation)) {
974
- worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);
1131
+ if (likely(rc->warning)) {
1132
+ worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);
1133
976
- if (unlikely(!expression_evaluate(rc->calculation))) {
1134
+ if (unlikely(!expression_evaluate(rc->warning))) {
1135
// calculation failed
978
- rc->value = NAN;
979
- rc->run_flags |= RRDCALC_FLAG_CALC_ERROR;
1136
+ rc->run_flags |= RRDCALC_FLAG_WARN_ERROR;
1137
981
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' failed: %s",
1138
+ debug(D_HEALTH,
1139
+ "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",
1140
rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
983
- rc->calculation->parsed_as, buffer_tostring(rc->calculation->error_msg)
984
- );
1141
+ buffer_tostring(rc->warning->error_msg)
1142
+ );
1143
} else {
986
- rc->run_flags &= ~RRDCALC_FLAG_CALC_ERROR;
987
-
988
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': expression '%s' gave value "
1144
+ rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR;
1145
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
1146
NETDATA_DOUBLE_FORMAT
990
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
991
- rc->calculation->parsed_as, rc->calculation->result,
992
- buffer_tostring(rc->calculation->error_msg), rrdcalc_source(rc)
993
- );
994
-
995
- rc->value = rc->calculation->result;
1147
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
1148
+ rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc)
1149
+ );
1150
+ warning_status = rrdcalc_value2status(rc->warning->result);
1151
}
1152
}
998
- }
999
- foreach_rrdcalc_in_rrdhost_done(rc);
1153
1001
- if (unlikely(runnable && !netdata_exit)) {
1002
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
1003
- if (unlikely(!(rc->run_flags & RRDCALC_FLAG_RUNNABLE)))
1004
- continue;
1154
+ // --------------------------------------------------------
1155
+ // check the critical expression
1156
1006
- if (rc->run_flags & RRDCALC_FLAG_DISABLED) {
1007
- continue;
1008
- }
1009
- RRDCALC_STATUS warning_status = RRDCALC_STATUS_UNDEFINED;
1010
- RRDCALC_STATUS critical_status = RRDCALC_STATUS_UNDEFINED;
1011
-
1012
- // --------------------------------------------------------
1013
- // check the warning expression
1014
-
1015
- if (likely(rc->warning)) {
1016
- worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);
1017
-
1018
- if (unlikely(!expression_evaluate(rc->warning))) {
1019
- // calculation failed
1020
- rc->run_flags |= RRDCALC_FLAG_WARN_ERROR;
1021
-
1022
- debug(D_HEALTH,
1023
- "Health on host '%s', alarm '%s.%s': warning expression failed with error: %s",
1024
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1025
- buffer_tostring(rc->warning->error_msg)
1026
- );
1027
- } else {
1028
- rc->run_flags &= ~RRDCALC_FLAG_WARN_ERROR;
1029
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': warning expression gave value "
1030
- NETDATA_DOUBLE_FORMAT
1031
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
1032
- rrdcalc_name(rc), rc->warning->result, buffer_tostring(rc->warning->error_msg), rrdcalc_source(rc)
1033
- );
1034
- warning_status = rrdcalc_value2status(rc->warning->result);
1035
- }
1036
- }
1157
+ if (likely(rc->critical)) {
1158
+ worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);
1159
1038
- // --------------------------------------------------------
1039
- // check the critical expression
1040
-
1041
- if (likely(rc->critical)) {
1042
- worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);
1043
-
1044
- if (unlikely(!expression_evaluate(rc->critical))) {
1045
- // calculation failed
1046
- rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR;
1047
-
1048
- debug(D_HEALTH,
1049
- "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",
1050
- rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1051
- buffer_tostring(rc->critical->error_msg)
1052
- );
1053
- } else {
1054
- rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
1055
- debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
1056
- NETDATA_DOUBLE_FORMAT
1057
- ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
1058
- rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg),
1059
- rrdcalc_source(rc)
1060
- );
1061
- critical_status = rrdcalc_value2status(rc->critical->result);
1062
- }
1063
- }
1160
+ if (unlikely(!expression_evaluate(rc->critical))) {
1161
+ // calculation failed
1162
+ rc->run_flags |= RRDCALC_FLAG_CRIT_ERROR;
1163
1065
- // --------------------------------------------------------
1066
- // decide the final alarm status
1164
+ debug(D_HEALTH,
1165
+ "Health on host '%s', alarm '%s.%s': critical expression failed with error: %s",
1166
+ rrdhost_hostname(host), rrdcalc_chart_name(rc), rrdcalc_name(rc),
1167
+ buffer_tostring(rc->critical->error_msg)
1168
+ );
1169
+ } else {
1170
+ rc->run_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
1171
+ debug(D_HEALTH, "Health on host '%s', alarm '%s.%s': critical expression gave value "
1172
+ NETDATA_DOUBLE_FORMAT
1173
+ ": %s (source: %s)", rrdhost_hostname(host), rrdcalc_chart_name(rc),
1174
+ rrdcalc_name(rc), rc->critical->result, buffer_tostring(rc->critical->error_msg),
1175
+ rrdcalc_source(rc)
1176
+ );
1177
+ critical_status = rrdcalc_value2status(rc->critical->result);
1178
+ }
1179
+ }
1180
1068
- RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
1181
+ // --------------------------------------------------------
1182
+ // decide the final alarm status
1183
1070
- switch (warning_status) {
1071
- case RRDCALC_STATUS_CLEAR:
1072
- status = RRDCALC_STATUS_CLEAR;
1073
- break;
1184
+ RRDCALC_STATUS status = RRDCALC_STATUS_UNDEFINED;
1185
1075
- case RRDCALC_STATUS_RAISED:
1076
- status = RRDCALC_STATUS_WARNING;
1077
- break;
1186
+ switch (warning_status) {
1187
+ case RRDCALC_STATUS_CLEAR:
1188
+ status = RRDCALC_STATUS_CLEAR;
1189
+ break;
1190
1079
- default:
1080
- break;
1081
- }
1191
+ case RRDCALC_STATUS_RAISED:
1192
+ status = RRDCALC_STATUS_WARNING;
1193
+ break;
1194
1083
- switch (critical_status) {
1084
- case RRDCALC_STATUS_CLEAR:
1085
- if (status == RRDCALC_STATUS_UNDEFINED)
1086
- status = RRDCALC_STATUS_CLEAR;
1087
- break;
1195
+ default:
1196
+ break;
1197
+ }
1198
1089
- case RRDCALC_STATUS_RAISED:
1090
- status = RRDCALC_STATUS_CRITICAL;
1091
- break;
1199
+ switch (critical_status) {
1200
+ case RRDCALC_STATUS_CLEAR:
1201
+ if (status == RRDCALC_STATUS_UNDEFINED)
1202
+ status = RRDCALC_STATUS_CLEAR;
1203
+ break;
1204
1093
- default:
1094
- break;
1095
- }
1205
+ case RRDCALC_STATUS_RAISED:
1206
+ status = RRDCALC_STATUS_CRITICAL;
1207
+ break;
1208
1097
- // --------------------------------------------------------
1098
- // check if the new status and the old differ
1209
+ default:
1210
+ break;
1211
+ }
1212
1100
- if (status != rc->status) {
1101
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1102
- int delay = 0;
1213
+ // --------------------------------------------------------
1214
+ // check if the new status and the old differ
1215
1104
- // apply trigger hysteresis
1216
+ if (status != rc->status) {
1217
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1218
+ int delay = 0;
1219
1106
- if (now > rc->delay_up_to_timestamp) {
1107
- rc->delay_up_current = rc->delay_up_duration;
1108
- rc->delay_down_current = rc->delay_down_duration;
1109
- rc->delay_last = 0;
1110
- rc->delay_up_to_timestamp = 0;
1111
- } else {
1112
- rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier);
1113
- if (rc->delay_up_current > rc->delay_max_duration)
1114
- rc->delay_up_current = rc->delay_max_duration;
1220
+ // apply trigger hysteresis
1221
1116
- rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier);
1117
- if (rc->delay_down_current > rc->delay_max_duration)
1118
- rc->delay_down_current = rc->delay_max_duration;
1119
- }
1222
+ if (now > rc->delay_up_to_timestamp) {
1223
+ rc->delay_up_current = rc->delay_up_duration;
1224
+ rc->delay_down_current = rc->delay_down_duration;
1225
+ rc->delay_last = 0;
1226
+ rc->delay_up_to_timestamp = 0;
1227
+ } else {
1228
+ rc->delay_up_current = (int) (rc->delay_up_current * rc->delay_multiplier);
1229
+ if (rc->delay_up_current > rc->delay_max_duration)
1230
+ rc->delay_up_current = rc->delay_max_duration;
1231
1121
- if (status > rc->status)
1122
- delay = rc->delay_up_current;
1123
- else
1124
- delay = rc->delay_down_current;
1125
-
1126
- // COMMENTED: because we do need to send raising alarms
1127
- // if(now + delay < rc->delay_up_to_timestamp)
1128
- // delay = (int)(rc->delay_up_to_timestamp - now);
1129
-
1130
- rc->delay_last = delay;
1131
- rc->delay_up_to_timestamp = now + delay;
1132
-
1133
-
1134
- ALARM_ENTRY *ae = health_create_alarm_entry(
1135
- host,
1136
- rc->id,
1137
- rc->next_event_id++,
1138
- rc->config_hash_id,
1139
- now,
1140
- rc->name,
1141
- rc->rrdset->id,
1142
- rc->rrdset->context,
1143
- rc->rrdset->family,
1144
- rc->classification,
1145
- rc->component,
1146
- rc->type,
1147
- rc->exec,
1148
- rc->recipient,
1149
- now - rc->last_status_change,
1150
- rc->old_value,
1151
- rc->value,
1152
- rc->status,
1153
- status,
1154
- rc->source,
1155
- rc->units,
1156
- rc->info,
1157
- rc->delay_last,
1158
- (
1159
- ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1160
- ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
1161
- (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
1162
- )
1163
- );
1232
+ rc->delay_down_current = (int) (rc->delay_down_current * rc->delay_multiplier);
1233
+ if (rc->delay_down_current > rc->delay_max_duration)
1234
+ rc->delay_down_current = rc->delay_max_duration;
1235
+ }
1236
1165
- health_alarm_log_add_entry(host, ae);
1237
+ if (status > rc->status)
1238
+ delay = rc->delay_up_current;
1239
+ else
1240
+ delay = rc->delay_down_current;
1241
+
1242
+ // COMMENTED: because we do need to send raising alarms
1243
+ // if(now + delay < rc->delay_up_to_timestamp)
1244
+ // delay = (int)(rc->delay_up_to_timestamp - now);
1245
+
1246
+ rc->delay_last = delay;
1247
+ rc->delay_up_to_timestamp = now + delay;
1248
+
1249
+ ALARM_ENTRY *ae = health_create_alarm_entry(
1250
+ host,
1251
+ rc->id,
1252
+ rc->next_event_id++,
1253
+ rc->config_hash_id,
1254
+ now,
1255
+ rc->name,
1256
+ rc->rrdset->id,
1257
+ rc->rrdset->context,
1258
+ rc->rrdset->family,
1259
+ rc->classification,
1260
+ rc->component,
1261
+ rc->type,
1262
+ rc->exec,
1263
+ rc->recipient,
1264
+ now - rc->last_status_change,
1265
+ rc->old_value,
1266
+ rc->value,
1267
+ rc->status,
1268
+ status,
1269
+ rc->source,
1270
+ rc->units,
1271
+ rc->info,
1272
+ rc->delay_last,
1273
+ (
1274
+ ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1275
+ ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
1276
+ (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
1277
+ )
1278
+ );
1279
+
1280
+ health_alarm_log_add_entry(host, ae);
1281
+
1282
+ log_health("[%s]: Alert event for [%s.%s], value [%s], status [%s].", rrdhost_hostname(host), ae_chart_name(ae), ae_name(ae), ae_new_value_string(ae), rrdcalc_status2string(ae->new_status));
1283
+
1284
+ rc->last_status_change = now;
1285
+ rc->old_status = rc->status;
1286
+ rc->status = status;
1287
+ }
1288
1167
- rc->last_status_change = now;
1168
- rc->old_status = rc->status;
1169
- rc->status = status;
1170
- }
1289
+ rc->last_updated = now;
1290
+ rc->next_update = now + rc->update_every;
1291
1172
- rc->last_updated = now;
1173
- rc->next_update = now + rc->update_every;
1292
+ if (next_run > rc->next_update)
1293
+ next_run = rc->next_update;
1294
+ }
1295
+ foreach_rrdcalc_in_rrdhost_done(rc);
1296
1175
- if (next_run > rc->next_update)
1176
- next_run = rc->next_update;
1177
- }
1178
- foreach_rrdcalc_in_rrdhost_done(rc);
1179
-
1180
- // process repeating alarms
1181
- foreach_rrdcalc_in_rrdhost_read(host, rc) {
1182
- int repeat_every = 0;
1183
- if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
1184
- if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
1185
- rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
1186
- repeat_every = rc->warn_repeat_every;
1187
- } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
1188
- rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
1189
- repeat_every = rc->crit_repeat_every;
1190
- } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
1191
- if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE)) {
1192
- if(rc->old_status == RRDCALC_STATUS_CRITICAL) {
1193
- repeat_every = 1;
1194
- } else if (rc->old_status == RRDCALC_STATUS_WARNING) {
1195
- repeat_every = 1;
1196
- }
1297
+ // process repeating alarms
1298
+ foreach_rrdcalc_in_rrdhost_read(host, rc) {
1299
+ int repeat_every = 0;
1300
+ if(unlikely(rrdcalc_isrepeating(rc) && rc->delay_up_to_timestamp <= now)) {
1301
+ if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
1302
+ rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
1303
+ repeat_every = rc->warn_repeat_every;
1304
+ } else if(unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
1305
+ rc->run_flags &= ~RRDCALC_FLAG_RUN_ONCE;
1306
+ repeat_every = rc->crit_repeat_every;
1307
+ } else if(unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
1308
+ if(!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE)) {
1309
+ if(rc->old_status == RRDCALC_STATUS_CRITICAL) {
1310
+ repeat_every = 1;
1311
+ } else if (rc->old_status == RRDCALC_STATUS_WARNING) {
1312
+ repeat_every = 1;
1313
}
1314
}
1199
- } else {
1200
- continue;
1315
}
1316
+ } else {
1317
+ continue;
1318
+ }
1319
1203
- if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
1204
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1205
- rc->last_repeat = now;
1206
- if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
1207
-
1208
- ALARM_ENTRY *ae = health_create_alarm_entry(
1209
- host,
1210
- rc->id,
1211
- rc->next_event_id++,
1212
- rc->config_hash_id,
1213
- now,
1214
- rc->name,
1215
- rc->rrdset->id,
1216
- rc->rrdset->context,
1217
- rc->rrdset->family,
1218
- rc->classification,
1219
- rc->component,
1220
- rc->type,
1221
- rc->exec,
1222
- rc->recipient,
1223
- now - rc->last_status_change,
1224
- rc->old_value,
1225
- rc->value,
1226
- rc->old_status,
1227
- rc->status,
1228
- rc->source,
1229
- rc->units,
1230
- rc->info,
1231
- rc->delay_last,
1232
- (
1233
- ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1234
- ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
1235
- (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
1236
- )
1237
- );
1238
-
1239
- ae->last_repeat = rc->last_repeat;
1240
- if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
1241
- ae->flags |= HEALTH_ENTRY_RUN_ONCE;
1242
- }
1243
- rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
1244
- health_process_notifications(host, ae);
1245
- debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
1246
- health_alarm_wait_for_execution(ae);
1247
- health_alarm_log_free_one_nochecks_nounlink(ae);
1320
+ if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
1321
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1322
+ rc->last_repeat = now;
1323
+ if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
1324
+
1325
+ ALARM_ENTRY *ae = health_create_alarm_entry(
1326
+ host,
1327
+ rc->id,
1328
+ rc->next_event_id++,
1329
+ rc->config_hash_id,
1330
+ now,
1331
+ rc->name,
1332
+ rc->rrdset->id,
1333
+ rc->rrdset->context,
1334
+ rc->rrdset->family,
1335
+ rc->classification,
1336
+ rc->component,
1337
+ rc->type,
1338
+ rc->exec,
1339
+ rc->recipient,
1340
+ now - rc->last_status_change,
1341
+ rc->old_value,
1342
+ rc->value,
1343
+ rc->old_status,
1344
+ rc->status,
1345
+ rc->source,
1346
+ rc->units,
1347
+ rc->info,
1348
+ rc->delay_last,
1349
+ (
1350
+ ((rc->options & RRDCALC_OPTION_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1351
+ ((rc->run_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0) |
1352
+ (rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0)
1353
+ )
1354
+ );
1355
+
1356
+ ae->last_repeat = rc->last_repeat;
1357
+ if (!(rc->run_flags & RRDCALC_FLAG_RUN_ONCE) && rc->status == RRDCALC_STATUS_CLEAR) {
1358
+ ae->flags |= HEALTH_ENTRY_RUN_ONCE;
1359
}
1360
+ rc->run_flags |= RRDCALC_FLAG_RUN_ONCE;
1361
+ health_process_notifications(host, ae);
1362
+ debug(D_HEALTH, "Notification sent for the repeating alarm %u.", ae->alarm_id);
1363
+ health_alarm_wait_for_execution(ae);
1364
+ health_alarm_log_free_one_nochecks_nounlink(ae);
1365
}
1250
- foreach_rrdcalc_in_rrdhost_done(rc);
1366
}
1367
+ foreach_rrdcalc_in_rrdhost_done(rc);
1368
+ }
1369
1253
- if (unlikely(netdata_exit))
1254
- break;
1370
+ if (unlikely(netdata_exit))
1371
+ break;
1372
1256
- // execute notifications
1257
- // and cleanup
1258
- worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
1259
- health_alarm_log_process(host);
1373
+ // execute notifications
1374
+ // and cleanup
1375
+ worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
1376
+ health_alarm_log_process(host);
1377
1261
- if (unlikely(netdata_exit)) {
1262
- // wait for all notifications to finish before allowing health to be cleaned up
1263
- ALARM_ENTRY *ae;
1264
- while (NULL != (ae = alarm_notifications_in_progress.head)) {
1265
- health_alarm_wait_for_execution(ae);
1266
- }
1267
- break;
1378
+ if (unlikely(netdata_exit)) {
1379
+ // wait for all notifications to finish before allowing health to be cleaned up
1380
+ ALARM_ENTRY *ae;
1381
+ while (NULL != (ae = alarm_notifications_in_progress.head)) {
1382
+ health_alarm_wait_for_execution(ae);
1383
}
1269
-
1270
- } /* rrdhost_foreach */
1384
+ break;
1385
+ }
1386
1387
// wait for all notifications to finish before allowing health to be cleaned up
1388
ALARM_ENTRY *ae;
@@ -1276,31 +1391,17 @@ void *health_main(void *ptr) {
1391
}
1392
1393
#ifdef ENABLE_ACLK
1279
- if (netdata_cloud_setting && unlikely(aclk_alert_reloaded) && loop > (marked_aclk_reload_loop + 2)) {
1280
- rrdhost_foreach_read(host) {
1281
- if (unlikely(!host->health_enabled))
1282
- continue;
1283
- sql_queue_removed_alerts_to_aclk(host);
1284
- }
1285
- aclk_alert_reloaded = 0;
1286
- marked_aclk_reload_loop = 0;
1287
- }
1394
+ if (netdata_cloud_setting && unlikely(host->aclk_alert_reloaded) && loop > (marked_aclk_reload_loop + 2)) {
1395
+ sql_queue_removed_alerts_to_aclk(host);
1396
+ host->aclk_alert_reloaded = 0;
1397
+ marked_aclk_reload_loop = 0;
1398
+ }
1399
#endif
1400
1290
- rrd_unlock();
1291
-
1401
if(unlikely(netdata_exit))
1402
break;
1403
1295
- now = now_realtime_sec();
1296
- if(now < next_run) {
1297
- worker_is_idle();
1298
- debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now));
1299
- sleep_usec(USEC_PER_SEC * (usec_t) (next_run - now));
1300
- now = now_realtime_sec();
1301
- }
1302
- else
1303
- debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
1404
+ health_sleep(next_run, loop);
1405
1406
} // forever
1407
@@ -1317,3 +1418,22 @@ void health_add_host_labels(void) {
1418
int has_unstable_connection = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_HEALTH, "has unstable connection", CONFIG_BOOLEAN_NO);
1419
rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_CONFIG);
1420
}
1421
+
1422
+void health_thread_spawn(RRDHOST * host) {
1423
+ if(!host->health_spawn) {
1424
+ char tag[NETDATA_THREAD_TAG_MAX + 1];
1425
+ snprintfz(tag, NETDATA_THREAD_TAG_MAX, "HEALTH[%s]", rrdhost_hostname(host));
1426
+ struct health_state *health = callocz(1, sizeof(*health));
1427
+ health->host = host;
1428
+
1429
+ if(netdata_thread_create(&host->health_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, health_main, (void *) health)) {
1430
+ log_health("[%s]: Failed to create new thread for client.", rrdhost_hostname(host));
1431
+ error("HEALTH [%s]: Failed to create new thread for client.", rrdhost_hostname(host));
1432
+ }
1433
+ else {
1434
+ log_health("[%s]: Created new thread for client.", rrdhost_hostname(host));
1435
+ host->health_spawn = 1;
1436
+ host->aclk_alert_reloaded = 1;
1437
+ }
1438
+ }
1439
+}
health/health.h
+8
@@ -48,6 +48,9 @@ int health_alarm_log_open(RRDHOST *host);
48
void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
49
void health_alarm_log_load(RRDHOST *host);
50
51
+void health_thread_spawn(RRDHOST *host);
52
+void health_thread_stop(RRDHOST *host);
53
+
54
ALARM_ENTRY* health_create_alarm_entry(
55
RRDHOST *host,
56
uint32_t alarm_id,
@@ -76,6 +79,11 @@ ALARM_ENTRY* health_create_alarm_entry(
79
80
void health_alarm_log_add_entry(RRDHOST *host, ALARM_ENTRY *ae);
81
82
+struct health_state {
83
+ RRDHOST *host;
84
+ netdata_thread_t thread;
85
+};
86
+
87
void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath);
88
char *health_user_config_dir(void);
89
char *health_stock_config_dir(void);
health/health_config.c
+3
-2
@@ -1163,7 +1163,7 @@ void sql_refresh_hashes(void)
1163
}
1164
1165
void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path, const char *subpath) {
1166
- if(unlikely(!host->health_enabled)) {
1166
+ if(unlikely(!host->health_enabled) && !rrdhost_flag_check(host, RRDHOST_FLAG_INITIALIZED_HEALTH)) {
1167
debug(D_HEALTH, "CONFIG health is not enabled for host '%s'", rrdhost_hostname(host));
1168
return;
1169
}
@@ -1172,10 +1172,11 @@ void health_readdir(RRDHOST *host, const char *user_path, const char *stock_path
1172
CONFIG_BOOLEAN_YES);
1173
1174
if (!stock_enabled) {
1175
- info("Netdata will not load stock alarms.");
1175
+ log_health("[%s]: Netdata will not load stock alarms.", rrdhost_hostname(host));
1176
stock_path = user_path;
1177
}
1178
1179
recursive_config_double_dir_load(user_path, stock_path, subpath, health_readfile, (void *) host, 0);
1180
+ log_health("[%s]: Read health configuration.", rrdhost_hostname(host));
1181
sql_store_hashes = 0;
1182
}
libnetdata/log/log.c
+43
-1
@@ -15,14 +15,19 @@ uint64_t debug_flags = 0;
15
int access_log_syslog = 1;
16
int error_log_syslog = 1;
17
int output_log_syslog = 1; // debug log
18
+int health_log_syslog = 1;
19
20
int stdaccess_fd = -1;
21
FILE *stdaccess = NULL;
22
23
+int stdhealth_fd = -1;
24
+FILE *stdhealth = NULL;
25
+
26
const char *stdaccess_filename = NULL;
27
const char *stderr_filename = NULL;
28
const char *stdout_filename = NULL;
29
const char *facility_log = NULL;
30
+const char *stdhealth_filename = NULL;
31
32
#ifdef ENABLE_ACLK
33
const char *aclklog_filename = NULL;
@@ -580,7 +585,10 @@ void reopen_all_log_files() {
585
#endif
586
587
if(stdaccess_filename)
583
- stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd);
588
+ stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd);
589
+
590
+ if(stdhealth_filename)
591
+ stdhealth = open_log_file(stdhealth_fd, stdhealth, stdhealth_filename, &health_log_syslog, 1, &stdhealth_fd);
592
}
593
594
void open_all_log_files() {
@@ -596,6 +604,8 @@ void open_all_log_files() {
604
#endif
605
606
stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd);
607
+
608
+ stdhealth = open_log_file(stdhealth_fd, stdhealth, stdhealth_filename, &health_log_syslog, 1, &stdhealth_fd);
609
}
610
611
// ----------------------------------------------------------------------------
@@ -963,6 +973,38 @@ void log_access( const char *fmt, ... ) {
973
}
974
}
975
976
+// ----------------------------------------------------------------------------
977
+// health log
978
+
979
+void log_health( const char *fmt, ... ) {
980
+ va_list args;
981
+
982
+ if(health_log_syslog) {
983
+ va_start( args, fmt );
984
+ vsyslog(LOG_INFO, fmt, args );
985
+ va_end( args );
986
+ }
987
+
988
+ if(stdhealth) {
989
+ static netdata_mutex_t health_mutex = NETDATA_MUTEX_INITIALIZER;
990
+
991
+ if(web_server_is_multithreaded)
992
+ netdata_mutex_lock(&health_mutex);
993
+
994
+ char date[LOG_DATE_LENGTH];
995
+ log_date(date, LOG_DATE_LENGTH);
996
+ fprintf(stdhealth, "%s: ", date);
997
+
998
+ va_start( args, fmt );
999
+ vfprintf( stdhealth, fmt, args );
1000
+ va_end( args );
1001
+ fputc('\n', stdhealth);
1002
+
1003
+ if(web_server_is_multithreaded)
1004
+ netdata_mutex_unlock(&health_mutex);
1005
+ }
1006
+}
1007
+
1008
#ifdef ENABLE_ACLK
1009
void log_aclk_message_bin( const char *data, const size_t data_len, int tx, const char *mqtt_topic, const char *message_name) {
1010
if (aclklog) {
libnetdata/log/log.h
+6
@@ -57,9 +57,13 @@ extern const char *program_name;
57
extern int stdaccess_fd;
58
extern FILE *stdaccess;
59
60
+extern int stdhealth_fd;
61
+extern FILE *stdhealth;
62
+
63
extern const char *stdaccess_filename;
64
extern const char *stderr_filename;
65
extern const char *stdout_filename;
66
+extern const char *stdhealth_filename;
67
extern const char *facility_log;
68
69
#ifdef ENABLE_ACLK
@@ -72,6 +76,7 @@ extern int aclklog_enabled;
76
extern int access_log_syslog;
77
extern int error_log_syslog;
78
extern int output_log_syslog;
79
+extern int health_log_syslog;
80
81
extern time_t error_log_throttle_period;
82
extern unsigned long error_log_errors_per_period, error_log_errors_per_period_backup;
@@ -107,6 +112,7 @@ void info_int( const char *file, const char *function, const unsigned long line,
112
void error_int( const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(5, 6);
113
void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) NORETURN PRINTFLIKE(4, 5);
114
void log_access( const char *fmt, ... ) PRINTFLIKE(1, 2);
115
+void log_health( const char *fmt, ... ) PRINTFLIKE(1, 2);
116
117
#ifdef ENABLE_ACLK
118
void log_aclk_message_bin( const char *data, const size_t data_len, int tx, const char *mqtt_topic, const char *message_name);
streaming/receiver.c
+4
-4
@@ -735,10 +735,10 @@ static int rrdpush_receive(struct receiver_state *rpt)
735
if(health_enabled != CONFIG_BOOLEAN_NO) {
736
if(alarms_delay > 0) {
737
rpt->host->health_delay_up_to = now_realtime_sec() + alarms_delay;
738
- info(
739
- "Postponing health checks for %" PRId64 " seconds, on host '%s', because it was just connected.",
740
- (int64_t)alarms_delay,
741
- rrdhost_hostname(rpt->host));
738
+ log_health(
739
+ "[%s]: Postponing health checks for %" PRId64 " seconds, because it was just connected.",
740
+ rrdhost_hostname(rpt->host),
741
+ (int64_t)alarms_delay);
742
}
743
}
744
rpt->host->senders_connect_time = now_realtime_sec();