rotate health log file
Costa Tsaousis (ktsaou) committed
Oct 16, 2016 at 19:40 UTC
0b238c44d6549e05641822eba61112bc1b850842
2 files changed
+182
-142
src/health.c
+181
-141
@@ -6,6 +6,7 @@ struct health_options {
6
const char *health_default_exec;
7
const char *health_default_recipient;
8
const char *log_filename;
9
+ size_t log_entries_written;
10
FILE *log_fp;
11
};
12
@@ -13,6 +14,7 @@ static struct health_options health = {
14
.health_default_exec = PLUGINS_DIR "/alarm-notify.sh",
15
.health_default_recipient = "root",
16
.log_filename = VARLIB_DIR "/health/alarm_log.db",
17
+ .log_entries_written = 0,
18
.log_fp = NULL
19
};
20
@@ -45,14 +47,33 @@ static inline void health_alarm_log_close(void) {
47
}
48
}
49
48
-static inline void health_log_recreate(void) {
49
- if(health.log_fp != NULL) {
50
+static inline void health_log_rotate(void) {
51
+ static size_t rotate_every = 0;
52
+
53
+ if(unlikely(rotate_every == 0)) {
54
+ rotate_every = (size_t)config_get_number("health", "rotate log every lines", 100);
55
+ if(rotate_every < 100) rotate_every = 100;
56
+ }
57
+
58
+ if(unlikely(health.log_entries_written > rotate_every)) {
59
health_alarm_log_close();
60
61
+ char old_filename[FILENAME_MAX + 1];
62
+ snprintfz(old_filename, FILENAME_MAX, "%s.old", health.log_filename);
63
+
64
+ if(unlink(old_filename) == -1 && errno != ENOENT)
65
+ error("Health: cannot remove old alarms log file '%s'", old_filename);
66
+
67
+ if(link(health.log_filename, old_filename) == -1 && errno != ENOENT)
68
+ error("Health: cannot move file '%s' to '%s'.", health.log_filename, old_filename);
69
+
70
// open it with truncate
71
health.log_fp = fopen(health.log_filename, "w");
54
- if(health.log_fp) fclose(health.log_fp);
55
- else error("Health: cannot truncate health log '%s'", health.log_filename);
72
+
73
+ if(health.log_fp)
74
+ fclose(health.log_fp);
75
+ else
76
+ error("Health: cannot truncate health log '%s'", health.log_filename);
77
78
health.log_fp = NULL;
79
@@ -61,8 +82,7 @@ static inline void health_log_recreate(void) {
82
}
83
84
static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
64
- (void)host;
65
- (void)ae;
85
+ health_log_rotate();
86
87
if(likely(health.log_fp)) {
88
if(unlikely(fprintf(health.log_fp
@@ -108,197 +128,217 @@ static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
128
, (long double)ae->old_value
129
) < 0))
130
error("Health: failed to save alarm log entry. Health data may be lost in case of abnormal restart.");
111
- else
131
+ else {
132
ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
133
+ health.log_entries_written++;
134
+ }
135
}
136
}
137
116
-static inline ssize_t health_alarm_log_load(RRDHOST *host) {
138
+static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) {
139
uint32_t max_unique_id = 0, max_alarm_id = 0;
140
ssize_t loaded = -1, updated = -1, errored = -1, duplicate = -1;
119
- health_alarm_log_close();
120
-
121
- FILE *fp = fopen(health.log_filename, "r");
122
- if(!fp)
123
- error("Health: cannot open health file: %s", health.log_filename);
124
- else {
125
- errno = 0;
141
127
- char *s, *buf = mallocz(65536 + 1);
128
- size_t line = 0, len = 0;
129
- loaded = updated = errored = duplicate = 0;
142
+ errno = 0;
143
131
- pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
144
+ char *s, *buf = mallocz(65536 + 1);
145
+ size_t line = 0, len = 0;
146
+ loaded = updated = errored = duplicate = 0;
147
133
- while((s = fgets_trim_len(buf, 65536, fp, &len))) {
134
- line++;
148
+ pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
149
136
- int max_entries = 30, entries = 0;
137
- char *pointers[max_entries];
150
+ while((s = fgets_trim_len(buf, 65536, fp, &len))) {
151
+ health.log_entries_written++;
152
+ line++;
153
139
- pointers[entries++] = s++;
140
- while(*s) {
141
- if(unlikely(*s == '\t')) {
142
- *s = '\0';
143
- pointers[entries++] = ++s;
144
- if(entries >= max_entries) {
145
- error("Health: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, health.log_filename, max_entries);
146
- break;
147
- }
154
+ int max_entries = 30, entries = 0;
155
+ char *pointers[max_entries];
156
+
157
+ pointers[entries++] = s++;
158
+ while(*s) {
159
+ if(unlikely(*s == '\t')) {
160
+ *s = '\0';
161
+ pointers[entries++] = ++s;
162
+ if(entries >= max_entries) {
163
+ error("Health: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, filename, max_entries);
164
+ break;
165
}
149
- else s++;
166
}
167
+ else s++;
168
+ }
169
152
- if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
153
- ALARM_ENTRY *ae = NULL;
170
+ if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
171
+ ALARM_ENTRY *ae = NULL;
172
155
- if(entries < 26) {
156
- error("Health: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring line.", line, health.log_filename, entries);
157
- errored++;
158
- continue;
159
- }
173
+ if(entries < 26) {
174
+ error("Health: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring line.", line, filename, entries);
175
+ errored++;
176
+ continue;
177
+ }
178
161
- // check that we have valid ids
162
- uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
163
- if(!unique_id) {
164
- error("Health: line %zu of file '%s' states alarm entry with unique id %u (%s). Ignoring line.", line, health.log_filename, unique_id, pointers[2]);
165
- errored++;
166
- continue;
167
- }
179
+ // check that we have valid ids
180
+ uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
181
+ if(!unique_id) {
182
+ error("Health: line %zu of file '%s' states alarm entry with unique id %u (%s). Ignoring line.", line, filename, unique_id, pointers[2]);
183
+ errored++;
184
+ continue;
185
+ }
186
169
- uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
170
- if(!alarm_id) {
171
- error("Health: line %zu of file '%s' states alarm entry for alarm id %u (%s). Ignoring line.", line, health.log_filename, alarm_id, pointers[3]);
172
- errored++;
173
- continue;
174
- }
187
+ uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
188
+ if(!alarm_id) {
189
+ error("Health: line %zu of file '%s' states alarm entry for alarm id %u (%s). Ignoring line.", line, filename, alarm_id, pointers[3]);
190
+ errored++;
191
+ continue;
192
+ }
193
176
- // find a possible overwrite
177
- for(ae = host->health_log.alarms; ae; ae = ae->next) {
178
- if(unlikely(ae->unique_id == unique_id)) {
179
- if(unlikely(*pointers[0] == 'A')) {
180
- error("Health: line %zu of file '%s' adds duplicate alarm log entry with unique id %u."
181
- , line, health.log_filename, unique_id);
182
- *pointers[0] = 'U';
183
- duplicate++;
184
- }
185
- break;
194
+ // find a possible overwrite
195
+ for(ae = host->health_log.alarms; ae; ae = ae->next) {
196
+ if(unlikely(ae->unique_id == unique_id)) {
197
+ if(unlikely(*pointers[0] == 'A')) {
198
+ error("Health: line %zu of file '%s' adds duplicate alarm log entry with unique id %u."
199
+ , line, filename, unique_id);
200
+ *pointers[0] = 'U';
201
+ duplicate++;
202
}
203
+ break;
204
}
205
+ }
206
189
- // if not found, create a new one
190
- if(likely(!ae)) {
191
-
192
- // if it is an update, but we haven't found it, make it an addition
193
- if(unlikely(*pointers[0] == 'U')) {
194
- *pointers[0] = 'A';
195
- error("Health: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, health.log_filename, unique_id);
196
- }
207
+ // if not found, create a new one
208
+ if(likely(!ae)) {
209
198
- // alarms should be added in the right order
199
- if(unlikely(unique_id < max_unique_id)) {
200
- error("Health: line %zu of file '%s' has alarm log entry with %u in wrong order.", line
201
- , health.log_filename, ae->unique_id);
202
- }
203
-
204
- ae = callocz(1, sizeof(ALARM_ENTRY));
210
+ // if it is an update, but we haven't found it, make it an addition
211
+ if(unlikely(*pointers[0] == 'U')) {
212
+ *pointers[0] = 'A';
213
+ error("Health: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, filename, unique_id);
214
}
215
207
- // check for a possible host missmatch
208
- if(strcmp(pointers[1], host->hostname))
209
- error("Health: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, health.log_filename, pointers[1], host->hostname);
216
+ // alarms should be added in the right order
217
+ if(unlikely(unique_id < max_unique_id)) {
218
+ error("Health: line %zu of file '%s' has alarm log entry with %u in wrong order.", line
219
+ , filename, ae->unique_id);
220
+ }
221
211
- ae->unique_id = unique_id;
212
- ae->alarm_id = alarm_id;
213
- ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16);
214
- ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16);
215
- ae->updates_id = (uint32_t)strtoul(pointers[6], NULL, 16);
222
+ ae = callocz(1, sizeof(ALARM_ENTRY));
223
+ }
224
217
- ae->when = (uint32_t)strtoul(pointers[7], NULL, 16);
218
- ae->duration = (uint32_t)strtoul(pointers[8], NULL, 16);
219
- ae->non_clear_duration = (uint32_t)strtoul(pointers[9], NULL, 16);
225
+ // check for a possible host missmatch
226
+ if(strcmp(pointers[1], host->hostname))
227
+ error("Health: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, filename, pointers[1], host->hostname);
228
221
- ae->flags = (uint32_t)strtoul(pointers[10], NULL, 16);
222
- ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
229
+ ae->unique_id = unique_id;
230
+ ae->alarm_id = alarm_id;
231
+ ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16);
232
+ ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16);
233
+ ae->updates_id = (uint32_t)strtoul(pointers[6], NULL, 16);
234
224
- ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16);
225
- ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16);
235
+ ae->when = (uint32_t)strtoul(pointers[7], NULL, 16);
236
+ ae->duration = (uint32_t)strtoul(pointers[8], NULL, 16);
237
+ ae->non_clear_duration = (uint32_t)strtoul(pointers[9], NULL, 16);
238
227
- if(unlikely(ae->name)) freez(ae->name);
228
- ae->name = strdupz(pointers[13]);
239
+ ae->flags = (uint32_t)strtoul(pointers[10], NULL, 16);
240
+ ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
241
230
- if(unlikely(ae->chart)) freez(ae->chart);
231
- ae->chart = strdupz(pointers[14]);
242
+ ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16);
243
+ ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16);
244
233
- if(unlikely(ae->family)) freez(ae->family);
234
- ae->family = strdupz(pointers[15]);
245
+ if(unlikely(ae->name)) freez(ae->name);
246
+ ae->name = strdupz(pointers[13]);
247
236
- if(unlikely(ae->exec)) freez(ae->exec);
237
- ae->exec = strdupz(pointers[16]);
238
- if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
248
+ if(unlikely(ae->chart)) freez(ae->chart);
249
+ ae->chart = strdupz(pointers[14]);
250
240
- if(unlikely(ae->recipient)) freez(ae->recipient);
241
- ae->recipient = strdupz(pointers[17]);
242
- if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
251
+ if(unlikely(ae->family)) freez(ae->family);
252
+ ae->family = strdupz(pointers[15]);
253
244
- if(unlikely(ae->source)) freez(ae->source);
245
- ae->source = strdupz(pointers[18]);
246
- if(!*ae->source) { freez(ae->source); ae->source = NULL; }
254
+ if(unlikely(ae->exec)) freez(ae->exec);
255
+ ae->exec = strdupz(pointers[16]);
256
+ if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
257
248
- if(unlikely(ae->units)) freez(ae->units);
249
- ae->units = strdupz(pointers[19]);
250
- if(!*ae->units) { freez(ae->units); ae->units = NULL; }
258
+ if(unlikely(ae->recipient)) freez(ae->recipient);
259
+ ae->recipient = strdupz(pointers[17]);
260
+ if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
261
252
- if(unlikely(ae->info)) freez(ae->info);
253
- ae->info = strdupz(pointers[20]);
254
- if(!*ae->info) { freez(ae->info); ae->info = NULL; }
262
+ if(unlikely(ae->source)) freez(ae->source);
263
+ ae->source = strdupz(pointers[18]);
264
+ if(!*ae->source) { freez(ae->source); ae->source = NULL; }
265
256
- ae->exec_code = atoi(pointers[21]);
257
- ae->new_status = atoi(pointers[22]);
258
- ae->old_status = atoi(pointers[23]);
259
- ae->delay = atoi(pointers[24]);
266
+ if(unlikely(ae->units)) freez(ae->units);
267
+ ae->units = strdupz(pointers[19]);
268
+ if(!*ae->units) { freez(ae->units); ae->units = NULL; }
269
261
- ae->new_value = strtold(pointers[25], NULL);
262
- ae->old_value = strtold(pointers[26], NULL);
270
+ if(unlikely(ae->info)) freez(ae->info);
271
+ ae->info = strdupz(pointers[20]);
272
+ if(!*ae->info) { freez(ae->info); ae->info = NULL; }
273
264
- // add it to host if not already there
265
- if(unlikely(*pointers[0] == 'A')) {
266
- ae->next = host->health_log.alarms;
267
- host->health_log.alarms = ae;
268
- loaded++;
269
- }
270
- else updated++;
274
+ ae->exec_code = atoi(pointers[21]);
275
+ ae->new_status = atoi(pointers[22]);
276
+ ae->old_status = atoi(pointers[23]);
277
+ ae->delay = atoi(pointers[24]);
278
272
- if(unlikely(ae->unique_id > max_unique_id))
273
- max_unique_id = ae->unique_id;
279
+ ae->new_value = strtold(pointers[25], NULL);
280
+ ae->old_value = strtold(pointers[26], NULL);
281
275
- if(unlikely(ae->alarm_id >= max_alarm_id))
276
- max_alarm_id = ae->alarm_id;
277
- }
278
- else {
279
- error("Health: line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, health.log_filename, pointers[0]);
280
- errored++;
282
+ // add it to host if not already there
283
+ if(unlikely(*pointers[0] == 'A')) {
284
+ ae->next = host->health_log.alarms;
285
+ host->health_log.alarms = ae;
286
+ loaded++;
287
}
282
- }
288
+ else updated++;
289
284
- pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
290
+ if(unlikely(ae->unique_id > max_unique_id))
291
+ max_unique_id = ae->unique_id;
292
286
- freez(buf);
287
- fclose(fp);
293
+ if(unlikely(ae->alarm_id >= max_alarm_id))
294
+ max_alarm_id = ae->alarm_id;
295
+ }
296
+ else {
297
+ error("Health: line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, filename, pointers[0]);
298
+ errored++;
299
+ }
300
}
301
302
+ pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
303
+
304
+ freez(buf);
305
+
306
if(!max_unique_id) max_unique_id = (uint32_t)time(NULL);
307
if(!max_alarm_id) max_alarm_id = (uint32_t)time(NULL);
308
309
host->health_log.next_log_id = max_unique_id + 1;
310
host->health_log.next_alarm_id = max_alarm_id + 1;
311
296
- info("Health: loaded %zd alarms, updated %zd alarms, errors %zd entries, duplicate %zd", loaded, updated, errored, duplicate);
312
+ info("Health: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", filename, loaded, updated, errored, duplicate);
313
+ return loaded;
314
+}
315
+
316
+static inline void health_alarm_log_load(RRDHOST *host) {
317
+ health_alarm_log_close();
318
+
319
+ char buffer[FILENAME_MAX + 1];
320
+ snprintfz(buffer, FILENAME_MAX, "%s.old", health.log_filename);
321
+ FILE *fp = fopen(buffer, "r");
322
+ if(!fp)
323
+ error("Health: cannot open health file: %s", health.log_filename);
324
+ else {
325
+ health_alarm_log_read(host, fp, health.log_filename);
326
+ fclose(fp);
327
+ }
328
+
329
+ health.log_entries_written = 0;
330
+ fp = fopen(health.log_filename, "r");
331
+ if(!fp)
332
+ error("Health: cannot open health file: %s", health.log_filename);
333
+ else {
334
+ health_alarm_log_read(host, fp, health.log_filename);
335
+ fclose(fp);
336
+ }
337
338
health_alarm_log_open();
299
- return loaded;
339
}
340
341
+
342
// ----------------------------------------------------------------------------
343
// health alarm log management
344
src/registry.c
+1
-1
@@ -1472,7 +1472,7 @@ int registry_save(void) {
1472
// rename the db to .old
1473
debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);
1474
if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT)
1475
- error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, registry.db_filename);
1475
+ error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", registry.db_filename, old_filename);
1476
1477
else {
1478
// remove the database (it is saved in .old)