properly load the alarm log; additional checks; alarms are now sequencial even on restarts
Costa Tsaousis (ktsaou) committed
Oct 16, 2016 at 15:17 UTC
2e39d3da9f0be574d68c3a92c6ad38c0672e3d75
1 file changed
+75
-40
src/health.c
+75
-40
@@ -30,11 +30,11 @@ static inline int health_alarm_log_open(void) {
30
31
if(health.log_fp) {
32
if (setvbuf(health.log_fp, NULL, _IOLBF, 0) != 0)
33
- error("Cannot set line buffering on health log file.");
33
+ error("Health: cannot set line buffering on health log file.");
34
return 0;
35
}
36
37
- error("Cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", health.log_filename);
37
+ error("Health: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", health.log_filename);
38
return -1;
39
}
40
@@ -52,7 +52,7 @@ static inline void health_log_recreate(void) {
52
// open it with truncate
53
health.log_fp = fopen(health.log_filename, "w");
54
if(health.log_fp) fclose(health.log_fp);
55
- else error("Cannot truncate health log '%s'", health.log_filename);
55
+ else error("Health: cannot truncate health log '%s'", health.log_filename);
56
57
health.log_fp = NULL;
58
@@ -113,76 +113,95 @@ static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
113
}
114
}
115
116
-static inline void health_alarm_log_load(RRDHOST *host) {
117
- (void)host;
118
-
116
+static inline ssize_t health_alarm_log_load(RRDHOST *host) {
117
+ uint32_t max_unique_id = 0, max_alarm_id = 0;
118
+ ssize_t loaded = -1, updated = -1, errored = -1, duplicate = -1;
119
health_alarm_log_close();
120
121
FILE *fp = fopen(health.log_filename, "r");
122
if(!fp)
123
- error("Registry: cannot open health file: %s", health.log_filename);
123
+ error("Health: cannot open health file: %s", health.log_filename);
124
else {
125
errno = 0;
126
127
char *s, *buf = mallocz(65536 + 1);
128
- size_t line = 0;
129
- size_t len = 0;
128
+ size_t line = 0, len = 0;
129
+ loaded = updated = errored = duplicate = 0;
130
+
131
+ pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
132
133
while((s = fgets_trim_len(buf, 65536, fp, &len))) {
134
line++;
133
- // fprintf(stderr, "line %zu: '%s'\n", line, s);
135
136
int max_entries = 30, entries = 0;
137
char *pointers[max_entries];
138
138
- pointers[entries++] = s;
139
+ pointers[entries++] = s++;
140
while(*s) {
141
if(unlikely(*s == '\t')) {
142
*s = '\0';
142
- pointers[entries++] = s;
143
- if(entries > max_entries) {
144
- error("Line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, health.log_filename, max_entries);
143
+ pointers[entries++] = ++s;
144
+ if(entries >= max_entries) {
145
+ error("Health: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, health.log_filename, max_entries);
146
break;
147
}
148
}
148
- s++;
149
+ else s++;
150
}
151
152
if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
153
ALARM_ENTRY *ae = NULL;
154
155
if(entries < 26) {
155
- error("Line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring line.", line, health.log_filename, entries);
156
+ error("Health: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring line.", line, health.log_filename, entries);
157
+ errored++;
158
continue;
159
}
160
159
- // if this is an update, find it
160
- if(unlikely(*pointers[0] == 'U')) {
161
- uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
161
+ // check that we have valid ids
162
+ uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
163
+ if(!unique_id) {
164
+ error("Health: line %zu of file '%s' states alarm entry with unique id %u (%s). Ignoring line.", line, health.log_filename, unique_id, pointers[2]);
165
+ errored++;
166
+ continue;
167
+ }
168
163
- // fprintf(stderr, "searching for alarm entry with unique id %u\n", unique_id);
169
+ uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
170
+ if(!alarm_id) {
171
+ error("Health: line %zu of file '%s' states alarm entry for alarm id %u (%s). Ignoring line.", line, health.log_filename, alarm_id, pointers[3]);
172
+ errored++;
173
+ continue;
174
+ }
175
165
- // find it
166
- for(ae = host->health_log.alarms; ae ;ae = ae->next) {
167
- if(unlikely(ae->unique_id == unique_id)) break;
176
+ // find a possible overwrite
177
+ for(ae = host->health_log.alarms; ae ;ae = ae->next) {
178
+ if(unlikely(ae->unique_id == unique_id)) {
179
+ if(unlikely(*pointers[0] == 'A')) {
180
+ error("Health: line %zu of file '%s' adds duplicate alarm log entry with unique id %u.", line, health.log_filename, unique_id);
181
+ *pointers[0] = 'U';
182
+ duplicate++;
183
+ }
184
+ break;
185
}
186
+ }
187
170
- if(!ae) {
188
+ // if not found, create a new one
189
+ if(likely(!ae)) {
190
+ // if it is an update, but we haven't found it, make it an addition
191
+ if(unlikely(*pointers[0] == 'U')) {
192
*pointers[0] = 'A';
172
- error("Line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, health.log_filename, unique_id);
193
+ error("Health: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, health.log_filename, unique_id);
194
}
174
- }
195
176
- // create a new one
177
- if(likely(!ae)) {
196
ae = callocz(1, sizeof(ALARM_ENTRY));
197
}
198
199
+ // check for a possible host missmatch
200
if(strcmp(pointers[1], host->hostname))
182
- error("Line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, health.log_filename, pointers[1], host->hostname);
201
+ error("Health: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, health.log_filename, pointers[1], host->hostname);
202
184
- ae->unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
185
- ae->alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
203
+ ae->unique_id = unique_id;
204
+ ae->alarm_id = alarm_id;
205
ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16);
206
ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16);
207
ae->updates_id = (uint32_t)strtoul(pointers[6], NULL, 16);
@@ -190,7 +209,10 @@ static inline void health_alarm_log_load(RRDHOST *host) {
209
ae->when = (uint32_t)strtoul(pointers[7], NULL, 16);
210
ae->duration = (uint32_t)strtoul(pointers[8], NULL, 16);
211
ae->non_clear_duration = (uint32_t)strtoul(pointers[9], NULL, 16);
212
+
213
ae->flags = (uint32_t)strtoul(pointers[10], NULL, 16);
214
+ ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
215
+
216
ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16);
217
ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16);
218
@@ -235,24 +257,38 @@ static inline void health_alarm_log_load(RRDHOST *host) {
257
if(unlikely(*pointers[0] == 'A')) {
258
ae->next = host->health_log.alarms;
259
host->health_log.alarms = ae;
260
+ loaded++;
261
}
262
+ else updated++;
263
240
- if(unlikely(ae->unique_id >= host->health_log.next_log_id))
241
- host->health_log.next_log_id = ae->unique_id + 1;
264
+ if(unlikely(ae->unique_id > max_unique_id))
265
+ max_unique_id = ae->unique_id;
266
243
- if(unlikely(ae->alarm_id >= host->health_log.next_alarm_id))
244
- host->health_log.next_alarm_id = ae->alarm_id + 1;
267
+ if(unlikely(ae->alarm_id >= max_alarm_id))
268
+ max_alarm_id = ae->alarm_id;
269
}
270
else {
247
- error("Line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, health.log_filename, pointers[0]);
271
+ error("Health: line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, health.log_filename, pointers[0]);
272
+ errored++;
273
}
274
}
275
276
+ pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
277
+
278
freez(buf);
279
fclose(fp);
280
}
281
282
+ if(!max_unique_id) max_unique_id = (uint32_t)time(NULL);
283
+ if(!max_alarm_id) max_alarm_id = (uint32_t)time(NULL);
284
+
285
+ host->health_log.next_log_id = max_unique_id + 1;
286
+ host->health_log.next_alarm_id = max_alarm_id + 1;
287
+
288
+ fprintf(stderr, "Health loaded %zd alarms, updated %zd alarms, errors %zd entries, duplicate %zd\n", loaded, updated, errored, duplicate);
289
+
290
health_alarm_log_open();
291
+ return loaded;
292
}
293
294
// ----------------------------------------------------------------------------
@@ -328,10 +364,9 @@ static inline void health_alarm_log(RRDHOST *host,
364
365
health_alarm_log_save(host, t);
366
}
331
- else {
332
- // no need to continue
333
- break;
334
- }
367
+
368
+ // no need to continue
369
+ break;
370
}
371
}
372
pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);