preparations for saving/loading the alarm log
Costa Tsaousis (ktsaou) committed
Sep 27, 2016 at 01:05 UTC
9b949c37599fc093da3e03ba116658b4e9001439
1 file changed
+107
-14
src/health.c
+107
-14
@@ -2,12 +2,96 @@
2
3
#define RRDVAR_MAX_LENGTH 1024
4
5
-static const char *health_default_exec = PLUGINS_DIR "/alarm-notify.sh";
6
-static const char *health_default_recipient = "root";
5
+struct health_options {
6
+ const char *health_default_exec;
7
+ const char *health_default_recipient;
8
+ const char *log_filename;
9
+ FILE *log_fp;
10
+};
11
+
12
+static struct health_options health = {
13
+ .health_default_exec = PLUGINS_DIR "/alarm-notify.sh",
14
+ .health_default_recipient = "root",
15
+ .log_filename = VARLIB_DIR "/health/alarm_log.db",
16
+ .log_fp = NULL
17
+};
18
+
19
int health_enabled = 1;
20
21
// ----------------------------------------------------------------------------
10
-// Health Alarms Log Management
22
+// health alarm log load/save
23
+// no need for locking - only one thread is reading / writing the alarms log
24
+
25
+static inline int health_alarm_log_open(void) {
26
+ if(health.log_fp)
27
+ fclose(health.log_fp);
28
+
29
+ health.log_fp = fopen(health.log_filename, "a");
30
+
31
+ if(health.log_fp) {
32
+ if (setvbuf(health.log_fp, NULL, _IOLBF, 0) != 0)
33
+ error("Cannot set line buffering on health log file.");
34
+ return 0;
35
+ }
36
+
37
+ error("Cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", health.log_filename);
38
+ return -1;
39
+}
40
+
41
+static inline void health_alarm_log_close(void) {
42
+ if(health.log_fp) {
43
+ fclose(health.log_fp);
44
+ health.log_fp = NULL;
45
+ }
46
+}
47
+
48
+static inline void health_log_recreate(void) {
49
+ if(health.log_fp != NULL) {
50
+ health_alarm_log_close();
51
+
52
+ // open it with truncate
53
+ health.log_fp = fopen(health.log_filename, "w");
54
+ if(health.log_fp) fclose(health.log_fp);
55
+ else error("Cannot truncate health log '%s'", health.log_filename);
56
+
57
+ health.log_fp = NULL;
58
+
59
+ health_alarm_log_open();
60
+ }
61
+}
62
+
63
+static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
64
+ (void)host;
65
+ (void)ae;
66
+
67
+/* if(likely(health.log_fp)) {
68
+ if(unlikely(fprintf(health.log_fp, "A\t%s\t%08x\t%08x\t%08x\t%08x\t%08x\t%08x\t%s\t%s\t%s\t%s\t%s\t%08x\n",
69
+ host->hostname,
70
+ ae->unique_id,
71
+ ae->alarm_id,
72
+ ae->alarm_event_id,
73
+ (uint32_t)ae->when,
74
+ (uint32_t)ae->duration,
75
+ (uint32_t)ae->non_clear_duration,
76
+ (uint32_t)ae->exec_run_timestamp,
77
+ ae->name,
78
+ ae->chart,
79
+ ae->family,
80
+ ae->exec,
81
+ ae->recipient
82
+ ) < 0))
83
+ error("Health: failed to save alarm log entry. Health data may be lost in case of abnormal restart.");
84
+ }
85
+*/
86
+}
87
+
88
+static inline void health_alarm_log_load(RRDHOST *host) {
89
+ (void)host;
90
+
91
+}
92
+
93
+// ----------------------------------------------------------------------------
94
+// health alarm log management
95
96
static inline void health_alarm_log(RRDHOST *host,
97
uint32_t alarm_id, uint32_t alarm_event_id,
@@ -75,6 +159,8 @@ static inline void health_alarm_log(RRDHOST *host,
159
if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
160
(t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
161
ae->non_clear_duration += t->non_clear_duration;
162
+
163
+ health_alarm_log_save(host, t);
164
}
165
else {
166
// no need to continue
@@ -83,6 +169,8 @@ static inline void health_alarm_log(RRDHOST *host,
169
}
170
}
171
pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
172
+
173
+ health_alarm_log_save(host, ae);
174
}
175
176
// ----------------------------------------------------------------------------
@@ -1709,12 +1797,14 @@ void health_init(void) {
1797
return;
1798
}
1799
1800
+ health_alarm_log_load(&localhost);
1801
+
1802
char *path = health_config_dir();
1803
1804
{
1805
char buffer[FILENAME_MAX + 1];
1806
snprintfz(buffer, FILENAME_MAX, "%s/alarm-notify.sh", config_get("global", "plugins directory", PLUGINS_DIR));
1717
- health_default_exec = config_get("health", "script to execute on alarm", buffer);
1807
+ health.health_default_exec = config_get("health", "script to execute on alarm", buffer);
1808
}
1809
1810
long n = config_get_number("health", "in memory max health log entries", (long)localhost.health_log.max);
@@ -1776,8 +1866,8 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R
1866
(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)?"true":"false",
1867
(unsigned long)ae->exec_run_timestamp,
1868
(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED)?"true":"false",
1779
- ae->exec?ae->exec:health_default_exec,
1780
- ae->recipient?ae->recipient:health_default_recipient,
1869
+ ae->exec?ae->exec:health.health_default_exec,
1870
+ ae->recipient?ae->recipient:health.health_default_recipient,
1871
ae->exec_code,
1872
ae->source,
1873
ae->units?ae->units:"",
@@ -1852,8 +1942,8 @@ static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1942
, rc->chart
1943
, (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
1944
, (rc->rrdset)?"true":"false"
1855
- , rc->exec?rc->exec:health_default_exec
1856
- , rc->recipient?rc->recipient:health_default_recipient
1945
+ , rc->exec?rc->exec:health.health_default_exec
1946
+ , rc->recipient?rc->recipient:health.health_default_recipient
1947
, rc->source
1948
, rc->units?rc->units:""
1949
, rc->info?rc->info:""
@@ -2016,7 +2106,6 @@ void health_reload(void) {
2106
}
2107
}
2108
2019
-
2109
// ----------------------------------------------------------------------------
2110
// health main thread and friends
2111
@@ -2039,23 +2128,23 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2128
if(t && t->new_status == ae->new_status) {
2129
// don't send the same notification again
2130
info("Health not sending again notification for alarm '%s.%s' status %s", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
2042
- return;
2131
+ goto done;
2132
}
2133
2134
if((ae->old_status == RRDCALC_STATUS_UNDEFINED && ae->new_status == RRDCALC_STATUS_UNINITIALIZED)
2135
|| (ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)) {
2136
info("Health not sending notification for first initialization of alarm '%s.%s' status %s", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
2048
- return;
2137
+ goto done;
2138
}
2139
2140
char buffer[FILENAME_MAX + 1];
2141
pid_t command_pid;
2142
2143
const char *exec = ae->exec;
2055
- if(!exec) exec = health_default_exec;
2144
+ if(!exec) exec = health.health_default_exec;
2145
2146
const char *recipient = ae->recipient;
2058
- if(!recipient) recipient = health_default_recipient;
2147
+ if(!recipient) recipient = health.health_default_recipient;
2148
2149
snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s'",
2150
exec,
@@ -2086,7 +2175,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2175
FILE *fp = mypopen(buffer, &command_pid);
2176
if(!fp) {
2177
error("HEALTH: Cannot popen(\"%s\", \"r\").", buffer);
2089
- return;
2178
+ goto done;
2179
}
2180
debug(D_HEALTH, "HEALTH reading from command");
2181
char *s = fgets(buffer, FILENAME_MAX, fp);
@@ -2096,6 +2185,10 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2185
2186
if(ae->exec_code != 0)
2187
ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED;
2188
+
2189
+done:
2190
+ health_alarm_log_save(host, ae);
2191
+ return;
2192
}
2193
2194
static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {