New alarms (exporting and Backend) (#9075)
New alarms for exporting and backend.
thiagoftsm committed
May 26, 2020 at 11:57 UTC
ba47e197056601c7db3ce8db0db277d155e8118a
4 files changed
+51
-5
exporting/send_internal_metrics.c
+5
-5
@@ -15,7 +15,7 @@ void create_main_rusage_chart(RRDSET **st_rusage, RRDDIM **rd_user, RRDDIM **rd_
15
return;
16
17
*st_rusage = rrdset_create_localhost(
18
- "netdata", "exporting_main_thread_cpu", NULL, "exporting", NULL, "Netdata Main Exporting Thread CPU Usage",
18
+ "netdata", "exporting_main_thread_cpu", NULL, "exporting", "exporting_cpu_usage", "Netdata Main Exporting Thread CPU Usage",
19
"milliseconds/s", "exporting", NULL, 130600, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
20
21
*rd_user = rrddim_add(*st_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
@@ -67,7 +67,7 @@ void send_internal_metrics(struct instance *instance)
67
netdata_fix_chart_id(id);
68
69
stats->st_metrics = rrdset_create_localhost(
70
- "netdata", id, NULL, buffer_tostring(family), NULL, "Netdata Buffered Metrics", "metrics", "exporting", NULL,
70
+ "netdata", id, NULL, buffer_tostring(family), "exporting_buffer", "Netdata Buffered Metrics", "metrics", "exporting", NULL,
71
130610, instance->config.update_every, RRDSET_TYPE_LINE);
72
73
stats->rd_buffered_metrics = rrddim_add(stats->st_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -80,7 +80,7 @@ void send_internal_metrics(struct instance *instance)
80
netdata_fix_chart_id(id);
81
82
stats->st_bytes = rrdset_create_localhost(
83
- "netdata", id, NULL, buffer_tostring(family), NULL, "Netdata Exporting Data Size", "KiB", "exporting", NULL,
83
+ "netdata", id, NULL, buffer_tostring(family), "exporting_data_size", "Netdata Exporting Data Size", "KiB", "exporting", NULL,
84
130620, instance->config.update_every, RRDSET_TYPE_AREA);
85
86
stats->rd_buffered_bytes = rrddim_add(stats->st_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -94,7 +94,7 @@ void send_internal_metrics(struct instance *instance)
94
netdata_fix_chart_id(id);
95
96
stats->st_ops = rrdset_create_localhost(
97
- "netdata", id, NULL, buffer_tostring(family), NULL, "Netdata Exporting Operations", "operations", "exporting",
97
+ "netdata", id, NULL, buffer_tostring(family), "exporting_operations", "Netdata Exporting Operations", "operations", "exporting",
98
NULL, 130630, instance->config.update_every, RRDSET_TYPE_LINE);
99
100
stats->rd_transmission_successes = rrddim_add(stats->st_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -109,7 +109,7 @@ void send_internal_metrics(struct instance *instance)
109
netdata_fix_chart_id(id);
110
111
stats->st_rusage = rrdset_create_localhost(
112
- "netdata", id, NULL, buffer_tostring(family), NULL, "Netdata Exporting Instance Thread CPU Usage",
112
+ "netdata", id, NULL, buffer_tostring(family), "exporting_instance", "Netdata Exporting Instance Thread CPU Usage",
113
"milliseconds/s", "exporting", NULL, 130640, instance->config.update_every, RRDSET_TYPE_STACKED);
114
115
stats->rd_user = rrddim_add(stats->st_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
health/Makefile.am
+1
@@ -45,6 +45,7 @@ dist_healthconfig_DATA = \
45
health.d/dockerd.conf \
46
health.d/elasticsearch.conf \
47
health.d/entropy.conf \
48
+ health.d/exporting.conf \
49
health.d/fping.conf \
50
health.d/ioping.conf \
51
health.d/fronius.conf \
health/health.d/backend.conf
+11
@@ -1,3 +1,13 @@
1
+# Alert that backends subsystem will be disabled soon
2
+ alarm: backend_metrics_eol
3
+ on: netdata.backend_metrics
4
+ units: boolean
5
+ calc: $now - $last_collected_t
6
+ every: 1m
7
+ warn: $this > 0
8
+ delay: down 5m multiplier 1.5 max 1h
9
+ info: The backends subsystem is deprecated and will be removed soon. Migrate your configuration to exporting.conf.
10
+ to: sysadmin
11
12
# make sure we are sending data to backend
13
@@ -32,6 +42,7 @@
42
info: number of metrics lost due to repeating failures to contact the backend server
43
to: dba
44
45
+
46
# this chart has been removed from netdata
47
# alarm: backend_slow
48
# on: netdata.backend_latency
health/health.d/exporting.conf
new
+34
@@ -0,0 +1,34 @@
1
+
2
+template: exporting_last_buffering
3
+families: *
4
+ on: exporting_data_size
5
+ calc: $now - $last_collected_t
6
+ units: seconds ago
7
+ every: 10s
8
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
9
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
10
+ delay: down 5m multiplier 1.5 max 1h
11
+ info: number of seconds since the last successful buffering of exporting data
12
+ to: dba
13
+
14
+template: exporting_metrics_sent
15
+families: *
16
+ on: exporting_data_size
17
+ units: %
18
+ calc: abs($sent) * 100 / abs($buffered)
19
+ every: 10s
20
+ warn: $this != 100
21
+ delay: down 5m multiplier 1.5 max 1h
22
+ info: percentage of metrics sent to the external database server
23
+ to: dba
24
+
25
+template: exporting_metrics_lost
26
+families: *
27
+ on: exporting_data_size
28
+ units: metrics
29
+ calc: abs($lost)
30
+ every: 10s
31
+ crit: ($this != 0) || ($status == $CRITICAL && abs($sent) == 0)
32
+ delay: down 5m multiplier 1.5 max 1h
33
+ info: number of metrics lost due to repeating failures to contact the external database server
34
+ to: dba