Show last 15 alerts in notification (#13434)
* show last 15 alerts in notification * change to alerts * fix shellcheck
Emmanuel Vasilakis committed
Aug 1, 2022 at 10:58 UTC
7b6d6ea9bdef2ba496cc6c52c224b17411baf784
2 files changed
+55
-11
health/health.c
+47
-11
@@ -11,6 +11,12 @@ static struct {
11
ALARM_ENTRY *tail; // latest
12
} alarm_notifications_in_progress = {NULL, NULL};
13
14
+typedef struct active_alerts {
15
+ char *name;
16
+ time_t last_status_change;
17
+ RRDCALC_STATUS status;
18
+} active_alerts_t;
19
+
20
static inline void enqueue_alarm_notify_in_progress(ALARM_ENTRY *ae)
21
{
22
ae->prev_in_progress = NULL;
@@ -245,6 +251,15 @@ static inline RRDCALC_STATUS rrdcalc_value2status(NETDATA_DOUBLE n) {
251
}
252
253
#define ALARM_EXEC_COMMAND_LENGTH 8192
254
+#define ACTIVE_ALARMS_LIST_EXAMINE 500
255
+#define ACTIVE_ALARMS_LIST 15
256
+
257
+static inline int compare_active_alerts(const void * a, const void * b) {
258
+ active_alerts_t *active_alerts_a = (active_alerts_t *)a;
259
+ active_alerts_t *active_alerts_b = (active_alerts_t *)b;
260
+
261
+ return ( active_alerts_b->last_status_change - active_alerts_a->last_status_change );
262
+}
263
264
static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
265
ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED;
@@ -310,31 +325,28 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
325
RRDCALC *rc;
326
EVAL_EXPRESSION *expr=NULL;
327
BUFFER *warn_alarms, *crit_alarms;
328
+ active_alerts_t *active_alerts = callocz(ACTIVE_ALARMS_LIST_EXAMINE, sizeof(active_alerts_t));
329
330
warn_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
331
crit_alarms = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
332
317
- for(rc = host->alarms; rc ; rc = rc->next) {
333
+ for(rc = host->alarms; rc && (n_warn + n_crit) < ACTIVE_ALARMS_LIST_EXAMINE ; rc = rc->next) {
334
if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
335
continue;
336
337
if (unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
338
if (likely(ae->alarm_id != rc->id) || likely(ae->alarm_event_id != rc->next_event_id - 1)) {
323
- if (n_warn)
324
- buffer_strcat(warn_alarms, ",");
325
- buffer_strcat(warn_alarms, rc->name);
326
- buffer_strcat(warn_alarms, "=");
327
- buffer_snprintf(warn_alarms, 11, "%"PRId64"", (int64_t)rc->last_status_change);
339
+ active_alerts[n_warn+n_crit].name = rc->name;
340
+ active_alerts[n_warn+n_crit].last_status_change = rc->last_status_change;
341
+ active_alerts[n_warn+n_crit].status = rc->status;
342
n_warn++;
343
} else if (ae->alarm_id == rc->id)
344
expr = rc->warning;
345
} else if (unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
346
if (likely(ae->alarm_id != rc->id) || likely(ae->alarm_event_id != rc->next_event_id - 1)) {
333
- if (n_crit)
334
- buffer_strcat(crit_alarms, ",");
335
- buffer_strcat(crit_alarms, rc->name);
336
- buffer_strcat(crit_alarms, "=");
337
- buffer_snprintf(crit_alarms, 11, "%"PRId64"", (int64_t)rc->last_status_change);
347
+ active_alerts[n_warn+n_crit].name = rc->name;
348
+ active_alerts[n_warn+n_crit].last_status_change = rc->last_status_change;
349
+ active_alerts[n_warn+n_crit].status = rc->status;
350
n_crit++;
351
} else if (ae->alarm_id == rc->id)
352
expr = rc->critical;
@@ -344,6 +356,29 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
356
}
357
}
358
359
+ if (n_warn+n_crit>1)
360
+ qsort (active_alerts, n_warn+n_crit, sizeof(active_alerts_t), compare_active_alerts);
361
+
362
+ int count_w = 0, count_c = 0;
363
+ while (count_w + count_c < n_warn + n_crit && count_w + count_c < ACTIVE_ALARMS_LIST) {
364
+ if (active_alerts[count_w+count_c].status == RRDCALC_STATUS_WARNING) {
365
+ if (count_w)
366
+ buffer_strcat(warn_alarms, ",");
367
+ buffer_strcat(warn_alarms, active_alerts[count_w+count_c].name);
368
+ buffer_strcat(warn_alarms, "=");
369
+ buffer_snprintf(warn_alarms, 11, "%"PRId64"", (int64_t)active_alerts[count_w+count_c].last_status_change);
370
+ count_w++;
371
+ }
372
+ else if (active_alerts[count_w+count_c].status == RRDCALC_STATUS_CRITICAL) {
373
+ if (count_c)
374
+ buffer_strcat(crit_alarms, ",");
375
+ buffer_strcat(crit_alarms, active_alerts[count_w+count_c].name);
376
+ buffer_strcat(crit_alarms, "=");
377
+ buffer_snprintf(crit_alarms, 11, "%"PRId64"", (int64_t)active_alerts[count_w+count_c].last_status_change);
378
+ count_c++;
379
+ }
380
+ }
381
+
382
char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0=UNKNOWN");
383
384
snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" NETDATA_DOUBLE_FORMAT_ZERO
@@ -392,6 +427,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
427
freez(edit_command);
428
buffer_free(warn_alarms);
429
buffer_free(crit_alarms);
430
+ freez(active_alerts);
431
432
return; //health_alarm_wait_for_execution
433
done:
health/notifications/alarm-notify.sh.in
+8
@@ -2898,6 +2898,10 @@ if [ -n "$total_crit_alarms" ]; then
2898
done <<<"$total_crit_alarms,"
2899
fi
2900
2901
+if (( total_warnings + total_critical > 15 )); then
2902
+ EXTRA_ALARMS_LIST_TEXT="(Showing latest 15 alerts)"
2903
+fi
2904
+
2905
if [ -n "$edit_command_line" ]; then
2906
IFS='=' read -r edit_command line s_host <<<"$edit_command_line"
2907
fi
@@ -3423,6 +3427,10 @@ Content-Transfer-Encoding: 8bit
3427
<span style="font-weight:600">${total_critical} critical</span>
3428
additional active alert(s)</div>
3429
</td>
3430
+ </tr>
3431
+ <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
3432
+ <div style="font-family:Open Sans, sans-serif;font-size:12px;line-height:1;text-align:center;color:#35414A;">${EXTRA_ALARMS_LIST_TEXT}</div>
3433
+ </td>
3434
</tr>
3435
</tbody>
3436
</table>