added roles to receive email alarms
Costa Tsaousis (ktsaou) committed
Sep 3, 2016 at 04:20 UTC
e2f00e7104a33eed680cb42aeec48695635115c9
22 files changed
+223
-55
conf.d/health.d/apache.conf
+1
-1
@@ -9,5 +9,5 @@ template: apache_last_collected_secs
9
crit: $this > (10 * $update_every)
10
units: seconds ago
11
info: number of seconds since the last successful data collection
12
-
12
+ to: webmaster
13
conf.d/health.d/cpu.conf
+3
@@ -6,6 +6,7 @@ template: 5min_cpu_pcent
6
warn: $this > 90
7
units: %
8
info: average cpu utilization for the last 5 minutes
9
+ to: sysadmin
10
11
template: 5min_iowait_cpu_pcent
12
on: system.cpu
@@ -14,6 +15,7 @@ template: 5min_iowait_cpu_pcent
15
warn: $this > 10
16
units: %
17
info: average wait I/O for the last 5 minutes
18
+ to: sysadmin
19
20
template: 20min_steal_cpu_pcent
21
on: system.cpu
@@ -22,3 +24,4 @@ template: 20min_steal_cpu_pcent
24
warn: $this > 10
25
units: %
26
info: average stolen CPU time for the last 20 minutes
27
+ to: sysadmin
conf.d/health.d/disks.conf
+4
@@ -13,6 +13,7 @@ template: disk_full_percent
13
crit: $this > 95
14
units: %
15
info: current disk space usage
16
+ to: sysadmin
17
18
19
# -----------------------------------------------------------------------------
@@ -47,6 +48,7 @@ template: disk_full_after_hours
48
crit: $this > 0 and $this < 24
49
units: hours
50
info: estimated time the disk will run out of space, if the system continues to add data with the rate of the last 30 minutes
51
+ to: sysadmin
52
53
54
# -----------------------------------------------------------------------------
@@ -66,6 +68,7 @@ template: 10min_disk_utilization
68
crit: $this > $red
69
units: %
70
info: the percentage of time the disk was busy, during the last 10 minutes
71
+ to: sysadmin
72
73
74
# raise an alarm if the disk backlog
@@ -83,3 +86,4 @@ template: 10min_disk_backlog
86
crit: $this > $red
87
units: ms
88
info: average of the kernel estimated disk backlog, for the last 10 minutes
89
+ to: sysadmin
conf.d/health.d/entropy.conf
+1
@@ -11,3 +11,4 @@
11
crit: $this < 100
12
units: entries
13
info: minimum entries in the random numbers pool (entropy), for the last 30 minutes
14
+ to: sysadmin
conf.d/health.d/memcached.conf
+3
@@ -9,6 +9,7 @@ template: memcached_last_collected_secs
9
crit: $this > (10 * $update_every)
10
units: seconds ago
11
info: number of seconds since the last successful data collection
12
+ to: dba
13
14
15
# detect if memcached cache is full
@@ -21,6 +22,7 @@ template: cache_full_pcent
22
crit: $this > 90
23
units: %
24
info: current cache memory usage
25
+ to: dba
26
27
28
# find the rate memcached cache is filling
@@ -44,3 +46,4 @@ template: cache_full_after_hours
46
crit: $this > 0 and $this < 24
47
units: hours
48
info: estimated time the cache will run out of space, if the system continues to add data with the rate of the last 30 minutes
49
+ to: dba
conf.d/health.d/named.conf
+2
-1
@@ -6,7 +6,8 @@ template: named_last_collected_secs
6
calc: $now - $last_collected_t
7
every: 10s
8
warn: $this > ( 5 * $update_every)
9
- crit: $this > (10 * $update_every)
9
+ crit: $this > (60 * $update_every)
10
units: seconds ago
11
info: number of seconds since the last successful data collection
12
+ to: domainadmin
13
conf.d/health.d/net.conf
+2
@@ -10,6 +10,7 @@ template: 30min_packet_drops
10
crit: $this > 0
11
units: packets
12
info: dropped packets in the last 30 minutes
13
+ to: sysadmin
14
15
16
# check if an interface is having FIFO
@@ -24,4 +25,5 @@ template: 30min_fifo_errors
25
crit: $this > 0
26
units: errors
27
info: network interface fifo errors in the last 30 minutes
28
+ to: sysadmin
29
conf.d/health.d/nginx.conf
+1
@@ -9,4 +9,5 @@ template: nginx_last_collected_secs
9
crit: $this > (10 * $update_every)
10
units: seconds ago
11
info: number of seconds since the last successful data collection
12
+ to: webmaster
13
conf.d/health.d/qos.conf
+1
@@ -10,3 +10,4 @@
10
# warn: $this > 0
11
# units: packets
12
# info: dropped packets in the last 30 minutes
13
+# to: sysadmin
conf.d/health.d/ram.conf
+1
@@ -7,3 +7,4 @@
7
crit: $this > 90
8
units: %
9
info: system RAM usage
10
+ to: sysadmin
conf.d/health.d/redis.conf
+2
-1
@@ -6,7 +6,8 @@ template: redis_last_collected_secs
6
calc: $now - $last_collected_t
7
every: 10s
8
warn: $this > ( 5 * $update_every)
9
- crit: $this > (10 * $update_every)
9
+ crit: $this > (60 * $update_every)
10
units: seconds ago
11
info: number of seconds since the last successful data collection
12
+ to: dba
13
conf.d/health.d/squid.conf
+2
-1
@@ -6,7 +6,8 @@ template: squid_last_collected_secs
6
calc: $now - $last_collected_t
7
every: 10s
8
warn: $this > ( 5 * $update_every)
9
- crit: $this > (10 * $update_every)
9
+ crit: $this > (60 * $update_every)
10
units: seconds ago
11
info: number of seconds since the last successful data collection
12
+ to: proxyadmin
13
conf.d/health.d/swap.conf
+2
@@ -9,6 +9,7 @@
9
crit: $this > 20
10
units: % of RAM
11
info: the amount of memory swapped in the last 30 minutes, as a percentage of the system RAM
12
+ to: sysadmin
13
14
alarm: pcent_of_ram_in_swap
15
on: system.swap
@@ -18,3 +19,4 @@
19
crit: $this > 50
20
units: % of RAM
21
info: the swap memory used, as a percentage of the system RAM
22
+ to: sysadmin
conf.d/health_email_recipients.conf
new
+53
@@ -0,0 +1,53 @@
1
+# Configuration for alarms recipients
2
+
3
+# netdata alarms have been categorized to allow different roles to receive
4
+# alarms related to their work.
5
+
6
+# this file defines the email addresses for each role. if a role is not
7
+# defined, the email will be sent to root.
8
+
9
+# you can set multiple addresses for each role, like this:
10
+#
11
+# recipients[sysadmin]="admin1@example.com, admin2@example.com"
12
+#
13
+# it is important to add the comma between email addresses.
14
+
15
+# This configuration file is a BASH script itself. The 'recipients' variable is
16
+# an associative array. So you can use other variables too, like this one:
17
+
18
+default_recipient_for_all_roles="root"
19
+
20
+
21
+# -----------------------------------------------------------------------------
22
+# generic system alarms
23
+# CPU, disks, entropy, etc
24
+
25
+recipients[sysadmin]="${default_recipient_for_all_roles}"
26
+
27
+
28
+# -----------------------------------------------------------------------------
29
+# DNS related alarms
30
+
31
+recipients[domainadmin]="${default_recipient_for_all_roles}"
32
+
33
+
34
+# -----------------------------------------------------------------------------
35
+# database servers alarms
36
+# mysql, redis, memcached, etc
37
+
38
+recipients[dba]="${default_recipient_for_all_roles}"
39
+
40
+
41
+# -----------------------------------------------------------------------------
42
+# web servers alarms
43
+# apache, nginx, etc
44
+
45
+recipients[webmaster]="${default_recipient_for_all_roles}"
46
+
47
+
48
+# -----------------------------------------------------------------------------
49
+# proxy servers alarms
50
+# apache, nginx, etc
51
+
52
+recipients[proxyadmin]="${default_recipient_for_all_roles}"
53
+
plugins.d/alarm-email.sh
+40
-21
@@ -8,6 +8,13 @@ then
8
echo >&2 "I cannot send emails - there is no sendmail command available."
9
fi
10
11
+default_recipient_for_all_roles="root"
12
+declare -A recipients=()
13
+if [ -f "${NETDATA_CONFIG_DIR}/health_email_recipients.conf" ]
14
+ then
15
+ source "${NETDATA_CONFIG_DIR}/health_email_recipients.conf"
16
+fi
17
+
18
sendmail_from_pipe() {
19
"${sendmail}" -t
20
@@ -21,31 +28,43 @@ sendmail_from_pipe() {
28
fi
29
}
30
24
-name="${1}" # the name of the alarm, as given in netdata health.d entries
25
-chart="${2}" # the name of the chart (type.id)
26
-family="${3}" # the family of the chart
27
-status="${4}" # the current status : UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
28
-old_status="${5}" # the previous status: UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
29
-value="${6}" # the current value
30
-old_value="${7}" # the previous value
31
-src="${8}" # the line number and file the alarm has been configured
32
-duration="${9}" # the duration in seconds the previous state took
33
-non_clear_duration="${10}" # the total duration in seconds this is non-clear
34
-units="${11}" # the units of the value
35
-info="${12}" # a short description of the alarm
31
+recipient="${1}" # the recepient of the email
32
+hostname="${2}" # the hostname this event refers to
33
+unique_id="${3}" # the unique id of this event
34
+alarm_id="${4}" # the unique id of the alarm that generated this event
35
+event_id="${5}" # the incremental id of the event, for this alarm
36
+when="${6}" # the timestamp this event occured
37
+name="${7}" # the name of the alarm, as given in netdata health.d entries
38
+chart="${8}" # the name of the chart (type.id)
39
+family="${9}" # the family of the chart
40
+status="${10}" # the current status : UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
41
+old_status="${11}" # the previous status: UNITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
42
+value="${12}" # the current value
43
+old_value="${13}" # the previous value
44
+src="${14}" # the line number and file the alarm has been configured
45
+duration="${15}" # the duration in seconds the previous state took
46
+non_clear_duration="${16}" # the total duration in seconds this is non-clear
47
+units="${17}" # the units of the value
48
+info="${18}" # a short description of the alarm
49
+
50
+to="${recipients[${recipient}]}"
51
+[ -z "${to}" ] && to="${default_recipient_for_all_roles}"
52
+[ -z "${to}" ] && to="root"
53
54
[ ! -z "${info}" ] && info=" <small><br/>${info}</small>"
55
56
# get the system hostname
40
-hostname="${NETDATA_HOSTNAME}"
57
+[ -z "${hostname}" ] && hostname="${NETDATA_HOSTNAME}"
58
[ -z "${hostname}" ] && hostname="${NETDATA_REGISTRY_HOSTNAME}"
42
-[ -z "${hostname}" ] && hostname="$(hostname)"
59
+[ -z "${hostname}" ] && hostname="$(hostname 2>/dev/null)"
60
61
goto_url="${NETDATA_REGISTRY_URL}/goto-host-from-alarm.html?machine_guid=${NETDATA_REGISTRY_UNIQUE_ID}&chart=${chart}&family=${family}"
62
46
-# get the current date
47
-date="$(date)"
63
+date="$(date --date=@${when} 2>/dev/null)"
64
+[ -z "${date}" ] && date="$(date 2>/dev/null)"
65
66
+# convert a duration in seconds, to a human readable duration
67
+# using DAYS, MINUTES, SECONDS
68
duration4human() {
69
local s="${1}" d=0 h=0 m=0 ds="day" hs="hour" ms="minute" ss="second"
70
d=$(( s / 86400 ))
@@ -94,7 +113,7 @@ duration4human() {
113
}
114
115
severity="${status}"
97
-raised_for="<br/>(was ${old_status,,} for $(duration4human ${duration}))"
116
+raised_for="<br/><small>(was ${old_status,,} for $(duration4human ${duration}))</small>"
117
status_message="status unknown"
118
color="grey"
119
alarm="${name} = ${value} ${units}"
@@ -136,7 +155,7 @@ then
155
severity="Recovered from ${old_status}"
156
if [ $non_clear_duration -gt $duration ]
157
then
139
- raised_for="<br/>(had issues for $(duration4human ${non_clear_duration}))"
158
+ raised_for="<br/><small>(had issues for $(duration4human ${non_clear_duration}))</small>"
159
fi
160
161
elif [ "${old_status}" = "WARNING" -a "${status}" = "CRITICAL" ]
@@ -144,7 +163,7 @@ then
163
severity="Escalated to ${status}"
164
if [ $non_clear_duration -gt $duration ]
165
then
147
- raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
166
+ raised_for="<br/><small>(has issues for $(duration4human ${non_clear_duration}))</small>"
167
fi
168
169
elif [ "${old_status}" = "CRITICAL" -a "${status}" = "WARNING" ]
@@ -152,7 +171,7 @@ then
171
severity="Demoted to ${status}"
172
if [ $non_clear_duration -gt $duration ]
173
then
155
- raised_for="<br/>(has issues for $(duration4human ${non_clear_duration}))"
174
+ raised_for="<br/><small>(has issues for $(duration4human ${non_clear_duration}))</small>"
175
fi
176
177
else
@@ -161,7 +180,7 @@ fi
180
181
# send the email
182
cat <<EOF | sendmail_from_pipe
164
-To: root
183
+To: ${to}
184
Subject: ${hostname} ${status_message} - ${chart}.${name}
185
Content-Type: text/html
186
src/health.c
+76
-21
@@ -3,6 +3,7 @@
3
#define RRDVAR_MAX_LENGTH 1024
4
5
static const char *health_default_exec = PLUGINS_DIR "/alarm-email.sh";
6
+static const char *health_default_recipient = "root";
7
int health_enabled = 1;
8
9
// ----------------------------------------------------------------------------
@@ -618,7 +619,7 @@ static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const cha
619
const char *units, const char *info,
620
int group_method, int after, int before, int update_every, uint32_t options,
621
calculated_number green, calculated_number red,
621
- const char *exec, const char *source,
622
+ const char *exec, const char *recipient, const char *source,
623
const char *calc, const char *warn, const char *crit) {
624
625
char fullname[RRDVAR_MAX_LENGTH + 1];
@@ -628,10 +629,10 @@ static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const cha
629
return NULL;
630
631
RRDCALC *rc = callocz(1, sizeof(RRDCALC));
631
-
632
+ rc->id = host->health_log.next_alarm_id++;
633
+ rc->next_event_id = 1;
634
rc->name = strdupz(name);
635
rc->hash = simple_hash(rc->name);
634
-
636
rc->chart = strdupz(chart);
637
rc->hash_chart = simple_hash(rc->chart);
638
@@ -649,6 +650,7 @@ static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const cha
650
rc->options = options;
651
652
if(exec) rc->exec = strdupz(exec);
653
+ if(recipient) rc->recipient = strdupz(recipient);
654
if(source) rc->source = strdupz(source);
655
if(units) rc->units = strdupz(units);
656
if(info) rc->info = strdupz(info);
@@ -669,10 +671,11 @@ static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const cha
671
error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, name, crit);
672
}
673
672
- debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
674
+ debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
675
(rc->chart)?rc->chart:"NOCHART",
676
rc->name,
677
(rc->exec)?rc->exec:"DEFAULT",
678
+ (rc->recipient)?rc->recipient:"DEFAULT",
679
rc->green,
680
rc->red,
681
rc->group,
@@ -724,6 +727,7 @@ void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
727
freez(rc->family);
728
freez(rc->dimensions);
729
freez(rc->exec);
730
+ freez(rc->recipient);
731
freez(rc->source);
732
freez(rc->units);
733
freez(rc->info);
@@ -740,7 +744,8 @@ void rrdcalctemplate_link_matching(RRDSET *st) {
744
if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)) {
745
RRDCALC *rc = rrdcalc_create(st->rrdhost, rt->name, st->id,
746
rt->dimensions, rt->units, rt->info, rt->group, rt->after, rt->before, rt->update_every, rt->options,
743
- rt->green, rt->red, rt->exec, rt->source,
747
+ rt->green, rt->red,
748
+ rt->exec, rt->recipient, rt->source,
749
(rt->calculation)?rt->calculation->source:NULL,
750
(rt->warning)?rt->warning->source:NULL,
751
(rt->critical)?rt->critical->source:NULL);
@@ -781,6 +786,7 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
786
787
freez(rt->name);
788
freez(rt->exec);
789
+ freez(rt->recipient);
790
freez(rt->context);
791
freez(rt->source);
792
freez(rt->units);
@@ -805,6 +811,7 @@ static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
811
#define HEALTH_WARN_KEY "warn"
812
#define HEALTH_CRIT_KEY "crit"
813
#define HEALTH_EXEC_KEY "exec"
814
+#define HEALTH_RECIPIENT_KEY "to"
815
#define HEALTH_UNITS_KEY "units"
816
#define HEALTH_INFO_KEY "info"
817
@@ -832,10 +839,11 @@ static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
839
return 0;
840
}
841
835
- debug(D_HEALTH, "Health configuration adding alarm '%s.%s': exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
842
+ debug(D_HEALTH, "Health configuration adding alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
843
rc->chart?rc->chart:"NOCHART",
844
rc->name,
845
(rc->exec)?rc->exec:"DEFAULT",
846
+ (rc->recipient)?rc->recipient:"DEFAULT",
847
rc->green,
848
rc->red,
849
rc->group,
@@ -878,10 +886,11 @@ static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCAL
886
}
887
}
888
881
- debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s'",
889
+ debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s'",
890
rt->name,
891
(rt->context)?rt->context:"NONE",
892
(rt->exec)?rt->exec:"DEFAULT",
893
+ (rt->recipient)?rt->recipient:"DEFAULT",
894
rt->green,
895
rt->red,
896
rt->group,
@@ -1067,7 +1076,7 @@ static inline void strip_quotes(char *s) {
1076
int health_readfile(const char *path, const char *filename) {
1077
debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
1078
1070
- static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0;
1079
+ static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0;
1080
char buffer[HEALTH_CONF_MAX_LINE + 1];
1081
1082
if(unlikely(!hash_alarm)) {
@@ -1084,6 +1093,7 @@ int health_readfile(const char *path, const char *filename) {
1093
hash_every = simple_uhash(HEALTH_EVERY_KEY);
1094
hash_units = simple_hash(HEALTH_UNITS_KEY);
1095
hash_info = simple_hash(HEALTH_INFO_KEY);
1096
+ hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
1097
}
1098
1099
snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
@@ -1155,6 +1165,8 @@ int health_readfile(const char *path, const char *filename) {
1165
}
1166
1167
rc = callocz(1, sizeof(RRDCALC));
1168
+ rc->id = localhost.health_log.next_alarm_id++;
1169
+ rc->next_event_id = 1;
1170
rc->name = strdupz(value);
1171
rc->hash = simple_hash(rc->name);
1172
rc->source = health_source_file(line, path, filename);
@@ -1261,6 +1273,16 @@ int health_readfile(const char *path, const char *filename) {
1273
}
1274
rc->exec = strdupz(value);
1275
}
1276
+ else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1277
+ if(rc->recipient) {
1278
+ if(strcmp(rc->recipient, value))
1279
+ info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1280
+ line, path, filename, rc->name, key, rc->recipient, value, value);
1281
+
1282
+ freez(rc->recipient);
1283
+ }
1284
+ rc->recipient = strdupz(value);
1285
+ }
1286
else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1287
if(rc->units) {
1288
if(strcmp(rc->units, value))
@@ -1363,6 +1385,16 @@ int health_readfile(const char *path, const char *filename) {
1385
}
1386
rt->exec = strdupz(value);
1387
}
1388
+ else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1389
+ if(rt->recipient) {
1390
+ if(strcmp(rt->recipient, value))
1391
+ info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1392
+ line, path, filename, rt->name, key, rt->recipient, value, value);
1393
+
1394
+ freez(rt->recipient);
1395
+ }
1396
+ rt->recipient = strdupz(value);
1397
+ }
1398
else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1399
if(rt->units) {
1400
if(strcmp(rt->units, value))
@@ -1493,7 +1525,9 @@ static inline void health_string2json(BUFFER *wb, const char *prefix, const char
1525
1526
static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae) {
1527
buffer_sprintf(wb, "\n\t{\n"
1496
- "\t\t\"id\":%u,\n"
1528
+ "\t\t\"unique_id\":%u,\n"
1529
+ "\t\t\"alarm_id\":%u,\n"
1530
+ "\t\t\"alarm_event_id\":%u,\n"
1531
"\t\t\"name\":\"%s\",\n"
1532
"\t\t\"chart\":\"%s\",\n"
1533
"\t\t\"family\":\"%s\",\n"
@@ -1502,6 +1536,7 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae) {
1536
"\t\t\"exec_run\":%s,\n"
1537
"\t\t\"exec_failed\":%s,\n"
1538
"\t\t\"exec\":\"%s\",\n"
1539
+ "\t\t\"recipient\":\"%s\",\n"
1540
"\t\t\"exec_code\":%d,\n"
1541
"\t\t\"source\":\"%s\",\n"
1542
"\t\t\"units\":\"%s\",\n"
@@ -1511,7 +1546,9 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae) {
1546
"\t\t\"non_clear_duration\":%lu,\n"
1547
"\t\t\"status\":\"%s\",\n"
1548
"\t\t\"old_status\":\"%s\",\n",
1514
- ae->id,
1549
+ ae->unique_id,
1550
+ ae->alarm_id,
1551
+ ae->alarm_event_id,
1552
ae->name,
1553
ae->chart,
1554
ae->family,
@@ -1520,6 +1557,7 @@ static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae) {
1557
(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN)?"true":"false",
1558
(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED)?"true":"false",
1559
ae->exec?ae->exec:health_default_exec,
1560
+ ae->recipient?ae->recipient:health_default_recipient,
1561
ae->exec_code,
1562
ae->source,
1563
ae->units?ae->units:"",
@@ -1568,6 +1606,7 @@ static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1606
"\t\t\t\"family\": \"%s\",\n"
1607
"\t\t\t\"active\": %s,\n"
1608
"\t\t\t\"exec\": \"%s\",\n"
1609
+ "\t\t\t\"recipient\": \"%s\",\n"
1610
"\t\t\t\"source\": \"%s\",\n"
1611
"\t\t\t\"units\": \"%s\",\n"
1612
"\t\t\t\"info\": \"%s\",\n"
@@ -1582,6 +1621,7 @@ static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1621
, (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
1622
, (rc->rrdset)?"true":"false"
1623
, rc->exec?rc->exec:health_default_exec
1624
+ , rc->recipient?rc->recipient:health_default_recipient
1625
, rc->source
1626
, rc->units?rc->units:""
1627
, rc->info?rc->info:""
@@ -1729,7 +1769,7 @@ static inline int rrdcalc_value2status(calculated_number n) {
1769
return RRDCALC_STATUS_CLEAR;
1770
}
1771
1732
-static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1772
+static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
1773
if(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)
1774
return;
1775
@@ -1739,8 +1779,17 @@ static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1779
const char *exec = ae->exec;
1780
if(!exec) exec = health_default_exec;
1781
1742
- snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s'",
1782
+ const char *recipient = ae->recipient;
1783
+ if(!recipient) recipient = health_default_recipient;
1784
+
1785
+ snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%u' '%u' '%u' '%zu' '%s' '%s' '%s' '%s' '%s' '%0.1Lf' '%0.1Lf' '%s' '%u' '%u' '%s' '%s'",
1786
exec,
1787
+ recipient,
1788
+ host->hostname,
1789
+ ae->unique_id,
1790
+ ae->alarm_id,
1791
+ ae->alarm_event_id,
1792
+ ae->when,
1793
ae->name,
1794
ae->chart?ae->chart:"NOCAHRT",
1795
ae->family?ae->family:"NOFAMILY",
@@ -1774,7 +1823,7 @@ static inline void health_alarm_execute(ALARM_ENTRY *ae) {
1823
ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED;
1824
}
1825
1777
-static inline void health_process_notifications(ALARM_ENTRY *ae) {
1826
+static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
1827
info("Health alarm '%s.%s' = %0.2Lf - changed status from %s to %s",
1828
ae->chart?ae->chart:"NOCHART", ae->name,
1829
ae->new_value,
@@ -1782,12 +1831,14 @@ static inline void health_process_notifications(ALARM_ENTRY *ae) {
1831
rrdcalc_status2string(ae->new_status)
1832
);
1833
1785
- health_alarm_execute(ae);
1834
+ health_alarm_execute(host, ae);
1835
}
1836
1788
-static inline void health_alarm_log(RRDHOST *host, time_t when,
1837
+static inline void health_alarm_log(RRDHOST *host,
1838
+ uint32_t alarm_id, uint32_t alarm_event_id,
1839
+ time_t when,
1840
const char *name, const char *chart, const char *family,
1790
- const char *exec, time_t duration,
1841
+ const char *exec, const char *recipient, time_t duration,
1842
calculated_number old_value, calculated_number new_value,
1843
int old_status, int new_status,
1844
const char *source,
@@ -1807,11 +1858,14 @@ static inline void health_alarm_log(RRDHOST *host, time_t when,
1858
ae->family = strdupz(family);
1859
1860
if(exec) ae->exec = strdupz(exec);
1861
+ if(recipient) ae->recipient = strdupz(recipient);
1862
if(source) ae->source = strdupz(source);
1863
if(units) ae->units = strdupz(units);
1864
if(info) ae->info = strdupz(info);
1865
1814
- ae->id = host->health_log.nextid++;
1866
+ ae->unique_id = host->health_log.next_log_id++;
1867
+ ae->alarm_id = alarm_id;
1868
+ ae->alarm_event_id = alarm_event_id;
1869
ae->when = when;
1870
ae->old_value = old_value;
1871
ae->new_value = new_value;
@@ -1863,17 +1917,17 @@ static inline void health_alarm_log_process(RRDHOST *host) {
1917
pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1918
1919
for(ae = host->health_log.alarms; ae ;ae = ae->next) {
1866
- if(last_processed >= ae->id) break;
1920
+ if(last_processed >= ae->unique_id) break;
1921
1922
if(!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) &&
1923
!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)) {
1924
ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED;
1871
- health_process_notifications(ae);
1925
+ health_process_notifications(host, ae);
1926
}
1927
}
1928
1929
if(host->health_log.alarms)
1876
- last_processed = host->health_log.alarms->id;
1930
+ last_processed = host->health_log.alarms->unique_id;
1931
1932
pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1933
@@ -1899,6 +1953,7 @@ static inline void health_alarm_log_process(RRDHOST *host) {
1953
freez(ae->chart);
1954
freez(ae->family);
1955
freez(ae->exec);
1956
+ freez(ae->recipient);
1957
freez(ae->source);
1958
freez(ae->units);
1959
freez(ae->info);
@@ -2175,7 +2230,7 @@ void *health_main(void *ptr) {
2230
}
2231
2232
if(status != rc->status) {
2178
- health_alarm_log(&localhost, time(NULL), rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info);
2233
+ health_alarm_log(&localhost, rc->id, rc->next_event_id++, time(NULL), rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info);
2234
rc->last_status_change = now;
2235
rc->status = status;
2236
}
src/health.h
+11
-2
@@ -120,10 +120,14 @@ typedef struct rrddimvar {
120
#define RRDCALC_FLAG_CRIT_ERROR 0x00000020
121
122
typedef struct rrdcalc {
123
+ uint32_t id;
124
+ uint32_t next_event_id;
125
+
126
char *name;
127
uint32_t hash;
128
129
char *exec;
130
+ char *recipient;
131
132
char *chart; // the chart id this should be linked to
133
uint32_t hash_chart;
@@ -182,6 +186,7 @@ typedef struct rrdcalctemplate {
186
uint32_t hash_name;
187
188
char *exec;
189
+ char *recipient;
190
191
char *context;
192
uint32_t hash_context;
@@ -216,7 +221,9 @@ typedef struct rrdcalctemplate {
221
#define HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED 0x00000008
222
223
typedef struct alarm_entry {
219
- uint32_t id;
224
+ uint32_t unique_id;
225
+ uint32_t alarm_id;
226
+ uint32_t alarm_event_id;
227
228
time_t when;
229
time_t duration;
@@ -231,6 +238,7 @@ typedef struct alarm_entry {
238
char *family;
239
240
char *exec;
241
+ char *recipient;
242
int exec_code;
243
244
char *source;
@@ -249,7 +257,8 @@ typedef struct alarm_entry {
257
} ALARM_ENTRY;
258
259
typedef struct alarm_log {
252
- uint32_t nextid;
260
+ uint32_t next_log_id;
261
+ uint32_t next_alarm_id;
262
unsigned int count;
263
unsigned int max;
264
ALARM_ENTRY *alarms;
src/main.c
+6
@@ -276,6 +276,7 @@ static const char *verify_required_directory(const char *dir) {
276
277
int main(int argc, char **argv)
278
{
279
+ char *hostname = "localhost";
280
int i, check_config = 0;
281
int config_loaded = 0;
282
int dont_fork = 0;
@@ -646,6 +647,11 @@ int main(int argc, char **argv)
647
info("Successfully set pthread stacksize to %zu bytes", wanted_stacksize);
648
}
649
650
+ // ------------------------------------------------------------------------
651
+ // initialize rrd host
652
+
653
+ rrdhost_init(hostname);
654
+
655
// ------------------------------------------------------------------------
656
// initialize the registry
657
src/registry.c
+1
-1
@@ -1652,7 +1652,7 @@ int registry_init(void) {
1652
registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400;
1653
registry.registry_domain = config_get("registry", "registry domain", "");
1654
registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.my-netdata.io");
1655
- registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", hostname));
1655
+ registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", localhost.hostname));
1656
registry.verify_cookies_redirects = config_get_boolean("registry", "verify browser cookies support", 1);
1657
1658
setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1);
src/rrd.c
+8
-1
@@ -42,7 +42,8 @@ RRDHOST localhost = {
42
AVL_LOCK_INITIALIZER
43
},
44
.health_log = {
45
- .nextid = 1,
45
+ .next_log_id = 1,
46
+ .next_alarm_id = 1,
47
.count = 0,
48
.max = 1000,
49
.alarms = NULL,
@@ -50,6 +51,12 @@ RRDHOST localhost = {
51
}
52
};
53
54
+void rrdhost_init(char *hostname) {
55
+ localhost.hostname = hostname;
56
+ localhost.health_log.next_log_id =
57
+ localhost.health_log.next_alarm_id = time(NULL);
58
+}
59
+
60
void rrdhost_rwlock(RRDHOST *host) {
61
pthread_rwlock_wrlock(&host->rrdset_root_rwlock);
62
}
src/rrd.h
+1
@@ -310,6 +310,7 @@ struct rrdhost {
310
};
311
typedef struct rrdhost RRDHOST;
312
extern RRDHOST localhost;
313
+extern void rrdhost_init(char *hostname);
314
315
#ifdef NETDATA_INTERNAL_CHECKS
316
#define rrdhost_check_wrlock(host) rrdhost_check_wrlock_int(host, __FILE__, __FUNCTION__, __LINE__)
src/rrd2json.c
+2
-5
@@ -1,8 +1,5 @@
1
#include "common.h"
2
3
-#define HOSTNAME_MAX 1024
4
-char *hostname = "unknown";
5
-
3
void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
4
{
5
pthread_rwlock_rdlock(&st->rwlock);
@@ -84,7 +81,7 @@ void rrd_stats_api_v1_charts(BUFFER *wb)
81
",\n\t\"update_every\": %d"
82
",\n\t\"history\": %d"
83
",\n\t\"charts\": {"
87
- , hostname
84
+ , localhost.hostname
85
, rrd_update_every
86
, rrd_default_history_entries
87
);
@@ -246,7 +243,7 @@ void rrd_stats_all_json(BUFFER *wb)
243
"\t\"history\": %d,\n"
244
"\t\"memory\": %lu\n"
245
"}\n"
249
- , hostname
246
+ , localhost.hostname
247
, rrd_update_every
248
, rrd_default_history_entries
249
, memory