Store alert log entries even if alert is repeating. (#12226)
* store alarm log entries even if it is repeating * log times repeated for an alert
Emmanuel Vasilakis committed
Apr 20, 2022 at 14:32 UTC
6caab18c2e7c8b142b0fac3e0087cb3b67862410
5 files changed
+71
-75
database/rrdcalc.c
+54
-58
@@ -81,35 +81,32 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
81
82
if(!rc->units) rc->units = strdupz(st->units);
83
84
- if(!rrdcalc_isrepeating(rc)) {
85
- time_t now = now_realtime_sec();
86
- ALARM_ENTRY *ae = health_create_alarm_entry(
87
- host,
88
- rc->id,
89
- rc->next_event_id++,
90
- rc->config_hash_id,
91
- now,
92
- rc->name,
93
- rc->rrdset->id,
94
- rc->rrdset->family,
95
- rc->classification,
96
- rc->component,
97
- rc->type,
98
- rc->exec,
99
- rc->recipient,
100
- now - rc->last_status_change,
101
- rc->old_value,
102
- rc->value,
103
- rc->status,
104
- RRDCALC_STATUS_UNINITIALIZED,
105
- rc->source,
106
- rc->units,
107
- rc->info,
108
- 0,
109
- 0
110
- );
111
- health_alarm_log(host, ae);
112
- }
84
+ time_t now = now_realtime_sec();
85
+ ALARM_ENTRY *ae = health_create_alarm_entry(
86
+ host,
87
+ rc->id,
88
+ rc->next_event_id++,
89
+ rc->config_hash_id,
90
+ now,
91
+ rc->name,
92
+ rc->rrdset->id,
93
+ rc->rrdset->family,
94
+ rc->classification,
95
+ rc->component,
96
+ rc->type,
97
+ rc->exec,
98
+ rc->recipient,
99
+ now - rc->last_status_change,
100
+ rc->old_value,
101
+ rc->value,
102
+ rc->status,
103
+ RRDCALC_STATUS_UNINITIALIZED,
104
+ rc->source,
105
+ rc->units,
106
+ rc->info,
107
+ 0,
108
+ 0);
109
+ health_alarm_log(host, ae);
110
}
111
112
static inline int rrdcalc_test_additional_restriction(RRDCALC *rc, RRDSET *st){
@@ -159,35 +156,32 @@ inline void rrdsetcalc_unlink(RRDCALC *rc) {
156
157
RRDHOST *host = st->rrdhost;
158
162
- if(!rrdcalc_isrepeating(rc)) {
163
- time_t now = now_realtime_sec();
164
- ALARM_ENTRY *ae = health_create_alarm_entry(
165
- host,
166
- rc->id,
167
- rc->next_event_id++,
168
- rc->config_hash_id,
169
- now,
170
- rc->name,
171
- rc->rrdset->id,
172
- rc->rrdset->family,
173
- rc->classification,
174
- rc->component,
175
- rc->type,
176
- rc->exec,
177
- rc->recipient,
178
- now - rc->last_status_change,
179
- rc->old_value,
180
- rc->value,
181
- rc->status,
182
- RRDCALC_STATUS_REMOVED,
183
- rc->source,
184
- rc->units,
185
- rc->info,
186
- 0,
187
- 0
188
- );
189
- health_alarm_log(host, ae);
190
- }
159
+ time_t now = now_realtime_sec();
160
+ ALARM_ENTRY *ae = health_create_alarm_entry(
161
+ host,
162
+ rc->id,
163
+ rc->next_event_id++,
164
+ rc->config_hash_id,
165
+ now,
166
+ rc->name,
167
+ rc->rrdset->id,
168
+ rc->rrdset->family,
169
+ rc->classification,
170
+ rc->component,
171
+ rc->type,
172
+ rc->exec,
173
+ rc->recipient,
174
+ now - rc->last_status_change,
175
+ rc->old_value,
176
+ rc->value,
177
+ rc->status,
178
+ RRDCALC_STATUS_REMOVED,
179
+ rc->source,
180
+ rc->units,
181
+ rc->info,
182
+ 0,
183
+ 0);
184
+ health_alarm_log(host, ae);
185
186
debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);
187
@@ -422,6 +416,7 @@ inline RRDCALC *rrdcalc_create_from_template(RRDHOST *host, RRDCALCTEMPLATE *rt,
416
rc->delay_multiplier = rt->delay_multiplier;
417
418
rc->last_repeat = 0;
419
+ rc->times_repeat = 0;
420
rc->warn_repeat_every = rt->warn_repeat_every;
421
rc->crit_repeat_every = rt->crit_repeat_every;
422
@@ -534,6 +529,7 @@ inline RRDCALC *rrdcalc_create_from_rrdcalc(RRDCALC *rc, RRDHOST *host, const ch
529
newrc->delay_multiplier = rc->delay_multiplier;
530
531
newrc->last_repeat = 0;
532
+ newrc->times_repeat = 0;
533
newrc->warn_repeat_every = rc->warn_repeat_every;
534
newrc->crit_repeat_every = rc->crit_repeat_every;
535
database/rrdcalc.h
+1
@@ -121,6 +121,7 @@ struct rrdcalc {
121
time_t next_update; // the next update timestamp of the alarm
122
time_t last_status_change; // the timestamp of the last time this alarm changed status
123
time_t last_repeat; // the last time the alarm got repeated
124
+ uint32_t times_repeat; // number of times the alarm got repeated
125
126
time_t db_after; // the first timestamp evaluated by the db lookup
127
time_t db_before; // the last timestamp evaluated by the db lookup
health/health.c
+14
-13
@@ -1041,19 +1041,19 @@ void *health_main(void *ptr) {
1041
rc->delay_last = delay;
1042
rc->delay_up_to_timestamp = now + delay;
1043
1044
- if(likely(!rrdcalc_isrepeating(rc))) {
1045
- ALARM_ENTRY *ae = health_create_alarm_entry(
1046
- host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id,
1047
- rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
1048
- rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info,
1049
- rc->delay_last,
1050
- (
1051
- ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1052
- ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0)
1053
- )
1054
- );
1055
- health_alarm_log(host, ae);
1056
- }
1044
+
1045
+ ALARM_ENTRY *ae = health_create_alarm_entry(
1046
+ host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id,
1047
+ rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
1048
+ rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info,
1049
+ rc->delay_last,
1050
+ (
1051
+ ((rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)? HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION : 0) |
1052
+ ((rc->rrdcalc_flags & RRDCALC_FLAG_SILENCED)? HEALTH_ENTRY_FLAG_SILENCED : 0)
1053
+ )
1054
+ );
1055
+ health_alarm_log(host, ae);
1056
+
1057
rc->last_status_change = now;
1058
rc->old_status = rc->status;
1059
rc->status = status;
@@ -1092,6 +1092,7 @@ void *health_main(void *ptr) {
1092
1093
if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
1094
rc->last_repeat = now;
1095
+ if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
1096
ALARM_ENTRY *ae = health_create_alarm_entry(
1097
host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id,
1098
rc->rrdset->family, rc->classification, rc->component, rc->type, rc->exec, rc->recipient, now - rc->last_status_change,
health/health_json.c
+2
@@ -227,6 +227,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
227
"\t\t\t\"crit_repeat_every\": \"%u\",\n"
228
"\t\t\t\"value_string\": \"%s\",\n"
229
"\t\t\t\"last_repeat\": \"%lu\",\n"
230
+ "\t\t\t\"times_repeat\": %lu,\n"
231
, rc->chart, rc->name
232
, (unsigned long)rc->id
233
, hash_id
@@ -259,6 +260,7 @@ static inline void health_rrdcalc2json_nolock(RRDHOST *host, BUFFER *wb, RRDCALC
260
, rc->crit_repeat_every
261
, value_string
262
, (unsigned long)rc->last_repeat
263
+ , (unsigned long)rc->times_repeat
264
);
265
266
if(unlikely(rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)) {
health/health_log.c
-4
@@ -560,10 +560,6 @@ inline void health_alarm_log(
560
) {
561
debug(D_HEALTH, "Health adding alarm log entry with id: %u", ae->unique_id);
562
563
- if(unlikely(alarm_entry_isrepeating(host, ae))) {
564
- error("Repeating alarms cannot be added to host's alarm log entries. It seems somewhere in the logic, API is being misused. Alarm id: %u", ae->alarm_id);
565
- return;
566
- }
563
// link it
564
netdata_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
565
ae->next = host->health_log.alarms;