health log is saved and loaded back
Costa Tsaousis (ktsaou) committed
Oct 15, 2016 at 23:38 UTC
673e1b1d142b91df08e6ffe8f7fda53995ae022c
3 files changed
+235
-49
src/health.c
+226
-42
@@ -63,31 +63,196 @@ static inline void health_log_recreate(void) {
63
static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
64
(void)host;
65
(void)ae;
66
-
67
-/* if(likely(health.log_fp)) {
68
- if(unlikely(fprintf(health.log_fp, "A\t%s\t%08x\t%08x\t%08x\t%08x\t%08x\t%08x\t%s\t%s\t%s\t%s\t%s\t%08x\n",
69
- host->hostname,
70
- ae->unique_id,
71
- ae->alarm_id,
72
- ae->alarm_event_id,
73
- (uint32_t)ae->when,
74
- (uint32_t)ae->duration,
75
- (uint32_t)ae->non_clear_duration,
76
- (uint32_t)ae->exec_run_timestamp,
77
- ae->name,
78
- ae->chart,
79
- ae->family,
80
- ae->exec,
81
- ae->recipient
82
- ) < 0))
66
+
67
+ if(likely(health.log_fp)) {
68
+ if(unlikely(fprintf(health.log_fp
69
+ , "%c\t%s"
70
+ "\t%08x\t%08x\t%08x\t%08x\t%08x"
71
+ "\t%08x\t%08x\t%08x"
72
+ "\t%08x\t%08x\t%08x"
73
+ "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"
74
+ "\t%d\t%d\t%d\t%d"
75
+ "\t%Lf\t%Lf"
76
+ "\n"
77
+ , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'
78
+ , host->hostname
79
+
80
+ , ae->unique_id
81
+ , ae->alarm_id
82
+ , ae->alarm_event_id
83
+ , ae->updated_by_id
84
+ , ae->updates_id
85
+
86
+ , (uint32_t)ae->when
87
+ , (uint32_t)ae->duration
88
+ , (uint32_t)ae->non_clear_duration
89
+ , (uint32_t)ae->flags
90
+ , (uint32_t)ae->exec_run_timestamp
91
+ , (uint32_t)ae->delay_up_to_timestamp
92
+
93
+ , (ae->name)?ae->name:""
94
+ , (ae->chart)?ae->chart:""
95
+ , (ae->family)?ae->family:""
96
+ , (ae->exec)?ae->exec:""
97
+ , (ae->recipient)?ae->recipient:""
98
+ , (ae->source)?ae->source:""
99
+ , (ae->units)?ae->units:""
100
+ , (ae->info)?ae->info:""
101
+
102
+ , ae->exec_code
103
+ , ae->new_status
104
+ , ae->old_status
105
+ , ae->delay
106
+
107
+ , (long double)ae->new_value
108
+ , (long double)ae->old_value
109
+ ) < 0))
110
error("Health: failed to save alarm log entry. Health data may be lost in case of abnormal restart.");
111
+ else
112
+ ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
113
}
85
-*/
114
}
115
116
static inline void health_alarm_log_load(RRDHOST *host) {
117
(void)host;
118
119
+ health_alarm_log_close();
120
+
121
+ FILE *fp = fopen(health.log_filename, "r");
122
+ if(!fp)
123
+ error("Registry: cannot open health file: %s", health.log_filename);
124
+ else {
125
+ errno = 0;
126
+
127
+ char *s, *buf = mallocz(65536 + 1);
128
+ size_t line = 0;
129
+ size_t len = 0;
130
+
131
+ while((s = fgets_trim_len(buf, 65536, fp, &len))) {
132
+ line++;
133
+ // fprintf(stderr, "line %zu: '%s'\n", line, s);
134
+
135
+ int max_entries = 30, entries = 0;
136
+ char *pointers[max_entries];
137
+
138
+ pointers[entries++] = s;
139
+ while(*s) {
140
+ if(unlikely(*s == '\t')) {
141
+ *s = '\0';
142
+ pointers[entries++] = s;
143
+ if(entries > max_entries) {
144
+ error("Line %zd of file '%s' has more than %d entries. Ignoring excessive entries.", line, health.log_filename, max_entries);
145
+ break;
146
+ }
147
+ }
148
+ s++;
149
+ }
150
+
151
+ if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
152
+ ALARM_ENTRY *ae = NULL;
153
+
154
+ if(entries < 26) {
155
+ error("Line %zd of file '%s' should have at least 26 entries, but it has %d. Ignoring line.", line, health.log_filename, entries);
156
+ continue;
157
+ }
158
+
159
+ // if this is an update, find it
160
+ if(unlikely(*pointers[0] == 'U')) {
161
+ uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
162
+
163
+ // fprintf(stderr, "searching for alarm entry with unique id %u\n", unique_id);
164
+
165
+ // find it
166
+ for(ae = host->health_log.alarms; ae ;ae = ae->next) {
167
+ if(unlikely(ae->unique_id == unique_id)) break;
168
+ }
169
+
170
+ if(!ae) {
171
+ *pointers[0] = 'A';
172
+ error("Line %zd of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, health.log_filename, unique_id);
173
+ }
174
+ }
175
+
176
+ // create a new one
177
+ if(likely(!ae)) {
178
+ ae = callocz(1, sizeof(ALARM_ENTRY));
179
+ }
180
+
181
+ if(strcmp(pointers[1], host->hostname))
182
+ error("Line %zd of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, health.log_filename, pointers[1], host->hostname);
183
+
184
+ ae->unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
185
+ ae->alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
186
+ ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16);
187
+ ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16);
188
+ ae->updates_id = (uint32_t)strtoul(pointers[6], NULL, 16);
189
+
190
+ ae->when = (uint32_t)strtoul(pointers[7], NULL, 16);
191
+ ae->duration = (uint32_t)strtoul(pointers[8], NULL, 16);
192
+ ae->non_clear_duration = (uint32_t)strtoul(pointers[9], NULL, 16);
193
+ ae->flags = (uint32_t)strtoul(pointers[10], NULL, 16);
194
+ ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16);
195
+ ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16);
196
+
197
+ if(unlikely(ae->name)) freez(ae->name);
198
+ ae->name = strdupz(pointers[13]);
199
+
200
+ if(unlikely(ae->chart)) freez(ae->chart);
201
+ ae->chart = strdupz(pointers[14]);
202
+
203
+ if(unlikely(ae->family)) freez(ae->family);
204
+ ae->family = strdupz(pointers[15]);
205
+
206
+ if(unlikely(ae->exec)) freez(ae->exec);
207
+ ae->exec = strdupz(pointers[16]);
208
+ if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
209
+
210
+ if(unlikely(ae->recipient)) freez(ae->recipient);
211
+ ae->recipient = strdupz(pointers[17]);
212
+ if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
213
+
214
+ if(unlikely(ae->source)) freez(ae->source);
215
+ ae->source = strdupz(pointers[18]);
216
+ if(!*ae->source) { freez(ae->source); ae->source = NULL; }
217
+
218
+ if(unlikely(ae->units)) freez(ae->units);
219
+ ae->units = strdupz(pointers[19]);
220
+ if(!*ae->units) { freez(ae->units); ae->units = NULL; }
221
+
222
+ if(unlikely(ae->info)) freez(ae->info);
223
+ ae->info = strdupz(pointers[20]);
224
+ if(!*ae->info) { freez(ae->info); ae->info = NULL; }
225
+
226
+ ae->exec_code = atoi(pointers[21]);
227
+ ae->new_status = atoi(pointers[22]);
228
+ ae->old_status = atoi(pointers[23]);
229
+ ae->delay = atoi(pointers[24]);
230
+
231
+ ae->new_value = strtold(pointers[25], NULL);
232
+ ae->old_value = strtold(pointers[26], NULL);
233
+
234
+ // add it to host if not already there
235
+ if(unlikely(*pointers[0] == 'A')) {
236
+ ae->next = host->health_log.alarms;
237
+ host->health_log.alarms = ae;
238
+ }
239
+
240
+ if(unlikely(ae->unique_id >= host->health_log.next_log_id))
241
+ host->health_log.next_log_id = ae->unique_id + 1;
242
+
243
+ if(unlikely(ae->alarm_id >= host->health_log.next_alarm_id))
244
+ host->health_log.next_alarm_id = ae->alarm_id + 1;
245
+ }
246
+ else {
247
+ error("Line %zd of file '%s' is invalid (unrecognized entry type '%s').", line, health.log_filename, pointers[0]);
248
+ }
249
+ }
250
+
251
+ freez(buf);
252
+ fclose(fp);
253
+ }
254
+
255
+ health_alarm_log_open();
256
}
257
258
// ----------------------------------------------------------------------------
@@ -152,8 +317,8 @@ static inline void health_alarm_log(RRDHOST *host,
317
ALARM_ENTRY *t;
318
for(t = host->health_log.alarms ; t ; t = t->next) {
319
if(t != ae && t->alarm_id == ae->alarm_id) {
155
- if(!(t->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED) && !t->updated_by_id) {
156
- t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
320
+ if(!(t->flags & HEALTH_ENTRY_FLAG_UPDATED) && !t->updated_by_id) {
321
+ t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
322
t->updated_by_id = ae->unique_id;
323
ae->updates_id = t->unique_id;
324
@@ -1353,6 +1518,16 @@ static inline int health_parse_db_lookup(
1518
return 1;
1519
}
1520
1521
+static inline char *tabs2spaces(char *s) {
1522
+ char *t = s;
1523
+ while(*t) {
1524
+ if(unlikely(*t == '\t')) *t = ' ';
1525
+ t++;
1526
+ }
1527
+
1528
+ return s;
1529
+}
1530
+
1531
static inline char *health_source_file(size_t line, const char *path, const char *filename) {
1532
char buffer[FILENAME_MAX + 1];
1533
snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
@@ -1460,7 +1635,7 @@ int health_readfile(const char *path, const char *filename) {
1635
1636
rc = callocz(1, sizeof(RRDCALC));
1637
rc->next_event_id = 1;
1463
- rc->name = strdupz(value);
1638
+ rc->name = tabs2spaces(strdupz(value));
1639
rc->hash = simple_hash(rc->name);
1640
rc->source = health_source_file(line, path, filename);
1641
rc->green = NAN;
@@ -1483,7 +1658,7 @@ int health_readfile(const char *path, const char *filename) {
1658
rrdcalctemplate_free(&localhost, rt);
1659
1660
rt = callocz(1, sizeof(RRDCALCTEMPLATE));
1486
- rt->name = strdupz(value);
1661
+ rt->name = tabs2spaces(strdupz(value));
1662
rt->hash_name = simple_hash(rt->name);
1663
rt->source = health_source_file(line, path, filename);
1664
rt->green = NAN;
@@ -1502,7 +1677,7 @@ int health_readfile(const char *path, const char *filename) {
1677
1678
freez(rc->chart);
1679
}
1505
- rc->chart = strdupz(value);
1680
+ rc->chart = tabs2spaces(strdupz(value));
1681
rc->hash_chart = simple_hash(rc->chart);
1682
}
1683
else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
@@ -1566,7 +1741,7 @@ int health_readfile(const char *path, const char *filename) {
1741
1742
freez(rc->exec);
1743
}
1569
- rc->exec = strdupz(value);
1744
+ rc->exec = tabs2spaces(strdupz(value));
1745
}
1746
else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1747
if(rc->recipient) {
@@ -1576,7 +1751,7 @@ int health_readfile(const char *path, const char *filename) {
1751
1752
freez(rc->recipient);
1753
}
1579
- rc->recipient = strdupz(value);
1754
+ rc->recipient = tabs2spaces(strdupz(value));
1755
}
1756
else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1757
if(rc->units) {
@@ -1586,7 +1761,7 @@ int health_readfile(const char *path, const char *filename) {
1761
1762
freez(rc->units);
1763
}
1589
- rc->units = strdupz(value);
1764
+ rc->units = tabs2spaces(strdupz(value));
1765
strip_quotes(rc->units);
1766
}
1767
else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
@@ -1597,7 +1772,7 @@ int health_readfile(const char *path, const char *filename) {
1772
1773
freez(rc->info);
1774
}
1600
- rc->info = strdupz(value);
1775
+ rc->info = tabs2spaces(strdupz(value));
1776
strip_quotes(rc->info);
1777
}
1778
else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
@@ -1617,7 +1792,7 @@ int health_readfile(const char *path, const char *filename) {
1792
1793
freez(rt->context);
1794
}
1620
- rt->context = strdupz(value);
1795
+ rt->context = tabs2spaces(strdupz(value));
1796
rt->hash_context = simple_hash(rt->context);
1797
}
1798
else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
@@ -1681,7 +1856,7 @@ int health_readfile(const char *path, const char *filename) {
1856
1857
freez(rt->exec);
1858
}
1684
- rt->exec = strdupz(value);
1859
+ rt->exec = tabs2spaces(strdupz(value));
1860
}
1861
else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1862
if(rt->recipient) {
@@ -1691,7 +1866,7 @@ int health_readfile(const char *path, const char *filename) {
1866
1867
freez(rt->recipient);
1868
}
1694
- rt->recipient = strdupz(value);
1869
+ rt->recipient = tabs2spaces(strdupz(value));
1870
}
1871
else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1872
if(rt->units) {
@@ -1701,7 +1876,7 @@ int health_readfile(const char *path, const char *filename) {
1876
1877
freez(rt->units);
1878
}
1704
- rt->units = strdupz(value);
1879
+ rt->units = tabs2spaces(strdupz(value));
1880
strip_quotes(rt->units);
1881
}
1882
else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
@@ -1712,7 +1887,7 @@ int health_readfile(const char *path, const char *filename) {
1887
1888
freez(rt->info);
1889
}
1715
- rt->info = strdupz(value);
1890
+ rt->info = tabs2spaces(strdupz(value));
1891
strip_quotes(rt->info);
1892
}
1893
else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
@@ -1798,7 +1973,16 @@ void health_init(void) {
1973
return;
1974
}
1975
1976
+ char *pathname = config_get("health", "health db directory", VARLIB_DIR "/health");
1977
+ if(mkdir(pathname, 0770) == -1 && errno != EEXIST)
1978
+ fatal("Cannot create directory '%s'.", pathname);
1979
+
1980
+ char filename[FILENAME_MAX + 1];
1981
+ snprintfz(filename, FILENAME_MAX, "%s/health-log.db", pathname);
1982
+ health.log_filename = config_get("health", "health db file", filename);
1983
+
1984
health_alarm_log_load(&localhost);
1985
+ health_alarm_log_open();
1986
1987
char *path = health_config_dir();
1988
@@ -1865,10 +2049,10 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, R
2049
ae->name,
2050
ae->chart,
2051
ae->family,
1868
- (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED)?"true":"false",
1869
- (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)?"true":"false",
2052
+ (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false",
2053
+ (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false",
2054
(unsigned long)ae->exec_run_timestamp,
1871
- (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED)?"true":"false",
2055
+ (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false",
2056
ae->exec?ae->exec:health.health_default_exec,
2057
ae->recipient?ae->recipient:health.health_default_recipient,
2058
ae->exec_code,
@@ -2085,7 +2269,7 @@ void health_reload(void) {
2269
ALARM_ENTRY *t;
2270
for(t = localhost.health_log.alarms ; t ; t = t->next) {
2271
if(t->new_status != RRDCALC_STATUS_REMOVED)
2088
- t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
2272
+ t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
2273
}
2274
2275
// reset all thresholds to all charts
@@ -2121,12 +2305,12 @@ static inline int rrdcalc_value2status(calculated_number n) {
2305
}
2306
2307
static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2124
- ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED;
2308
+ ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED;
2309
2310
// find the previous notification for the same alarm
2311
ALARM_ENTRY *t;
2312
for(t = ae->next; t ;t = t->next) {
2129
- if(t->alarm_id == ae->alarm_id && t->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN)
2313
+ if(t->alarm_id == ae->alarm_id && t->flags & HEALTH_ENTRY_FLAG_EXEC_RUN)
2314
break;
2315
}
2316
@@ -2173,7 +2357,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2357
ae->info?ae->info:""
2358
);
2359
2176
- ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN;
2360
+ ae->flags |= HEALTH_ENTRY_FLAG_EXEC_RUN;
2361
ae->exec_run_timestamp = time(NULL);
2362
2363
debug(D_HEALTH, "executing command '%s'", buffer);
@@ -2189,7 +2373,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2373
debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);
2374
2375
if(ae->exec_code != 0)
2192
- ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED;
2376
+ ae->flags |= HEALTH_ENTRY_FLAG_EXEC_FAILED;
2377
2378
done:
2379
health_alarm_log_save(host, ae);
@@ -2217,8 +2401,8 @@ static inline void health_alarm_log_process(RRDHOST *host) {
2401
ALARM_ENTRY *ae;
2402
for(ae = host->health_log.alarms; ae && ae->unique_id >= stop_at_id ; ae = ae->next) {
2403
if(unlikely(
2220
- !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) &&
2221
- !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)
2404
+ !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) &&
2405
+ !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED)
2406
)) {
2407
2408
if(unlikely(ae->unique_id < first_waiting))
src/health.h
+6
-5
@@ -264,10 +264,11 @@ typedef struct rrdcalctemplate {
264
265
#define RRDCALCTEMPLATE_HAS_CALCULATION(rt) ((rt)->after)
266
267
-#define HEALTH_ENTRY_NOTIFICATIONS_PROCESSED 0x00000001
268
-#define HEALTH_ENTRY_NOTIFICATIONS_UPDATED 0x00000002
269
-#define HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN 0x00000004
270
-#define HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED 0x00000008
267
+#define HEALTH_ENTRY_FLAG_PROCESSED 0x00000001
268
+#define HEALTH_ENTRY_FLAG_UPDATED 0x00000002
269
+#define HEALTH_ENTRY_FLAG_EXEC_RUN 0x00000004
270
+#define HEALTH_ENTRY_FLAG_EXEC_FAILED 0x00000008
271
+#define HEALTH_ENTRY_FLAG_SAVED 0x10000000
272
273
typedef struct alarm_entry {
274
uint32_t unique_id;
@@ -300,7 +301,7 @@ typedef struct alarm_entry {
301
int old_status;
302
int new_status;
303
303
- uint32_t notifications;
304
+ uint32_t flags;
305
306
int delay;
307
time_t delay_up_to_timestamp;
src/registry.c
+3
-2
@@ -743,8 +743,7 @@ static inline void registry_log_recreate_nolock(void) {
743
}
744
745
int registry_log_load(void) {
746
- char *s, buf[4096 + 1];
747
- size_t line = -1;
746
+ ssize_t line = -1;
747
748
// closing the log is required here
749
// otherwise we will append to it the values we read
@@ -755,8 +754,10 @@ int registry_log_load(void) {
754
if(!fp)
755
error("Registry: cannot open registry file: %s", registry.log_filename);
756
else {
757
+ char *s, buf[4096 + 1];
758
line = 0;
759
size_t len = 0;
760
+
761
while ((s = fgets_trim_len(buf, 4096, fp, &len))) {
762
line++;
763