Add variables to alarm-notify.sh to show number of warning and critical alarms, evaluated expression and expression variable values. Use them in the email notifications (#5096)
Chris Akritidis committed
Jan 3, 2019 at 15:29 UTC
c9666eb4880f03a5b0a6ecb97b2c9ffc7fe30611
2 files changed
+58
-4
health/health.c
+29
-3
@@ -153,7 +153,29 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
153
const char *exec = (ae->exec) ? ae->exec : host->health_default_exec;
154
const char *recipient = (ae->recipient) ? ae->recipient : host->health_default_recipient;
155
156
- snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" CALCULATED_NUMBER_FORMAT_ZERO "' '" CALCULATED_NUMBER_FORMAT_ZERO "' '%s' '%u' '%u' '%s' '%s' '%s' '%s'",
156
+ int n_warn=0, n_crit=0;
157
+ RRDCALC *rc;
158
+ EVAL_EXPRESSION *expr=NULL;
159
+
160
+ for(rc = host->alarms; rc ; rc = rc->next) {
161
+ if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
162
+ continue;
163
+
164
+ if(unlikely(rc->status == RRDCALC_STATUS_WARNING)) {
165
+ n_warn++;
166
+ if (ae->alarm_id == rc->id)
167
+ expr=rc->warning;
168
+ } else if (unlikely(rc->status == RRDCALC_STATUS_CRITICAL)) {
169
+ n_crit++;
170
+ if (ae->alarm_id == rc->id)
171
+ expr=rc->critical;
172
+ } else if (unlikely(rc->status == RRDCALC_STATUS_CLEAR)) {
173
+ if (ae->alarm_id == rc->id)
174
+ expr=rc->warning;
175
+ }
176
+ }
177
+
178
+ snprintfz(command_to_run, ALARM_EXEC_COMMAND_LENGTH, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '" CALCULATED_NUMBER_FORMAT_ZERO "' '" CALCULATED_NUMBER_FORMAT_ZERO "' '%s' '%u' '%u' '%s' '%s' '%s' '%s' '%s' '%s' '%d' '%d'",
179
exec,
180
recipient,
181
host->registry_hostname,
@@ -162,7 +184,7 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
184
ae->alarm_event_id,
185
(unsigned long)ae->when,
186
ae->name,
165
- ae->chart?ae->chart:"NOCAHRT",
187
+ ae->chart?ae->chart:"NOCHART",
188
ae->family?ae->family:"NOFAMILY",
189
rrdcalc_status2string(ae->new_status),
190
rrdcalc_status2string(ae->old_status),
@@ -174,7 +196,11 @@ static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
196
ae->units?ae->units:"",
197
ae->info?ae->info:"",
198
ae->new_value_string,
177
- ae->old_value_string
199
+ ae->old_value_string,
200
+ (expr && expr->source)?expr->source:"NOSOURCE",
201
+ (expr && expr->error_msg)?buffer_tostring(expr->error_msg):"NOERRMSG",
202
+ n_warn,
203
+ n_crit
204
);
205
206
ae->flags |= HEALTH_ENTRY_FLAG_EXEC_RUN;
health/notifications/alarm-notify.sh.in
+29
-1
@@ -54,7 +54,7 @@ then
54
echo >&2
55
echo >&2 "# SENDING TEST ${x} ALARM TO ROLE: ${recipient}"
56
57
- "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value"
57
+ "${0}" "${recipient}" "$(hostname)" 1 1 "${id}" "$(date +%s)" "test_alarm" "test.chart" "test.family" "${x}" "${last}" 100 90 "${0}" 1 $((0 + id)) "units" "this is a test alarm to verify notifications work" "new value" "old value" "evaluated expression" "expression variable values" 0 0
58
if [ $? -ne 0 ]
59
then
60
echo >&2 "# FAILED"
@@ -218,8 +218,13 @@ else
218
info="${18}" # a short description of the alarm
219
value_string="${19}" # friendly value (with units)
220
old_value_string="${20}" # friendly old value (with units)
221
+ calc_expression="${21}" # contains the expression that was evaluated to trigger the alarm
222
+ calc_param_values="${22}" # the values of the parameters in the expression, at the time of the evaluation
223
+ total_warnings="${23}" # Total number of alarms in WARNING state
224
+ total_critical="${24}" # Total number of alarms in CRITICAL state
225
fi
226
227
+
228
# -----------------------------------------------------------------------------
229
# find a suitable hostname to use, if netdata did not supply a hostname
230
@@ -2074,6 +2079,11 @@ Source : ${src}
2079
Date : ${date}
2080
Notification generated on ${host}
2081
2082
+Evaluated Expression : ${calc_expression}
2083
+Expression Variables : ${calc_param_values}
2084
+
2085
+The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
2086
+
2087
--multipart-boundary
2088
Content-Type: text/html; encoding=${EMAIL_CHARSET}
2089
Content-Disposition: inline
@@ -2134,6 +2144,24 @@ Content-Transfer-Encoding: 8bit
2144
<span>${raised_for_html}</span> <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Time</span>
2145
</td>
2146
</tr>
2147
+ <tr style="margin: 0; padding: 0;">
2148
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
2149
+ <span>${calc_expression}</span>
2150
+ <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Evaluated Expression</span>
2151
+ </td>
2152
+ </tr>
2153
+ <tr style="margin: 0; padding: 0;">
2154
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
2155
+ <span>${calc_param_values}</span>
2156
+ <span style="display: block; color: #666666; font-size: 12px; font-weight: 300; line-height: 1; text-transform: uppercase;">Expression Variables</span>
2157
+ </td>
2158
+ </tr>
2159
+ <tr style="margin: 0; padding: 0;">
2160
+ <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;" align="left" valign="top">
2161
+ The host has ${total_warnings} WARNING and ${total_critical} CRITICAL alarm(s) raised.
2162
+ </td>
2163
+ </tr>
2164
+
2165
<tr style="margin: 0; padding: 0;">
2166
<td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; font-size: 18px; vertical-align: top; margin: 0; padding: 0 0 20px;">
2167
<a href="${goto_url}" style="font-size: 14px; color: #ffffff; text-decoration: none; line-height: 1.5; font-weight: bold; text-align: center; display: inline-block; text-transform: capitalize; background: #35568d; border-width: 1px; border-style: solid; border-color: #2b4c86; margin: 0; padding: 10px 15px;" target="_blank">View Netdata</a>