zookeeper and hdfs: alarms and dashboard_info (#6927)
* add zookeeper alarms * add zookeeper to dashboard_info * zookeeper alarm fix * add hdfs alarms * add hfds to dashboard_info * minor * fix hdfs zk links: use latest version * hdfs dashboard_info: change semicolon to comma
Ilya Mashchenko committed
Sep 25, 2019 at 12:50 UTC
0aae17767ff03da1d89767420b40501c7ced1b01
4 files changed
+115
health/Makefile.am
+2
@@ -44,6 +44,7 @@ dist_healthconfig_DATA = \
44
health.d/fronius.conf \
45
health.d/gearman.conf \
46
health.d/haproxy.conf \
47
+ health.d/hdfs.conf \
48
health.d/httpcheck.conf \
49
health.d/ipc.conf \
50
health.d/ipfs.conf \
@@ -91,5 +92,6 @@ dist_healthconfig_DATA = \
92
health.d/wmi.conf \
93
health.d/x509check.conf \
94
health.d/zfs.conf \
95
+ health.d/zookeeper.conf \
96
health.d/dbengine.conf \
97
$(NULL)
health/health.d/hdfs.conf
new
+75
@@ -0,0 +1,75 @@
1
+
2
+# make sure hdfs is running
3
+
4
+template: hdfs_last_collected_secs
5
+ on: hdfs.heap_memory
6
+ calc: $now - $last_collected_t
7
+ units: seconds ago
8
+ every: 10s
9
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
+ delay: down 5m multiplier 1.5 max 1h
12
+ info: number of seconds since the last successful data collection
13
+ to: webmaster
14
+
15
+
16
+# Common
17
+
18
+template: hdfs_capacity_usage
19
+ on: hdfs.capacity
20
+ calc: ($used) * 100 / ($used + $remaining)
21
+ units: %
22
+ every: 10s
23
+ warn: $this > (($status >= $WARNING) ? (70) : (80))
24
+ crit: $this > (($status == $CRITICAL) ? (80) : (98))
25
+ delay: down 15m multiplier 1.5 max 1h
26
+ info: used capacity
27
+ to: sysadmin
28
+
29
+
30
+# NameNode
31
+
32
+template: hdfs_missing_blocks
33
+ on: hdfs.blocks
34
+ calc: $missing
35
+ units: missing blocks
36
+ every: 10s
37
+ warn: $this > 0
38
+ delay: down 15m multiplier 1.5 max 1h
39
+ info: missing blocks
40
+ to: sysadmin
41
+
42
+
43
+template: hdfs_stale_nodes
44
+ on: hdfs.data_nodes
45
+ calc: $stale
46
+ units: dead nodes
47
+ every: 10s
48
+ warn: $this > 0
49
+ delay: down 15m multiplier 1.5 max 1h
50
+ info: stale data nodes
51
+ to: sysadmin
52
+
53
+
54
+template: hdfs_dead_nodes
55
+ on: hdfs.data_nodes
56
+ calc: $dead
57
+ units: dead nodes
58
+ every: 10s
59
+ crit: $this > 0
60
+ delay: down 15m multiplier 1.5 max 1h
61
+ info: dead data nodes
62
+ to: sysadmin
63
+
64
+
65
+# DataNode
66
+
67
+template: hdfs_num_failed_volumes
68
+ on: hdfs.num_failed_volumes
69
+ calc: $fsds_num_failed_volumes
70
+ units: failed volumes
71
+ every: 10s
72
+ warn: $this > 0
73
+ delay: down 15m multiplier 1.5 max 1h
74
+ info: failed volumes
75
+ to: sysadmin
health/health.d/zookeeper.conf
new
+14
@@ -0,0 +1,14 @@
1
+
2
+# make sure zookeeper is running
3
+
4
+template: zookeeper_last_collected_secs
5
+ on: zookeeper.requests
6
+ calc: $now - $last_collected_t
7
+ units: seconds ago
8
+ every: 10s
9
+ warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))
10
+ crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))
11
+ delay: down 5m multiplier 1.5 max 1h
12
+ info: number of seconds since the last successful data collection
13
+ to: webmaster
14
+
web/gui/dashboard_info.js
+24
@@ -504,6 +504,18 @@ netdataDashboard.menu = {
504
title: 'VCSA',
505
icon: '<i class="fas fa-server"></i>',
506
info: 'vCenter Server Appliance health statistics. Data collected from <a href="https://vmware.github.io/vsphere-automation-sdk-rest/vsphere/index.html#SVC_com.vmware.appliance.health">Health API</a>.'
507
+ },
508
+
509
+ 'zookeeper': {
510
+ title: 'Zookeeper',
511
+ icon: '<i class="fas fa-database"></i>',
512
+ info: 'Provides health statistics for <b><a href="https://zookeeper.apache.org/">Zookeeper</a></b> server. Data collected through the command port using <code><a href="https://zookeeper.apache.org/doc/r3.5.5/zookeeperAdmin.html#sc_zkCommands">mntr</a></code> command.'
513
+ },
514
+
515
+ 'hdfs': {
516
+ title: 'HDFS',
517
+ icon: '<i class="fas fa-folder-open"></i>',
518
+ info: 'Provides <b><a href="https://hadoop.apache.org/docs/r3.2.0/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html">Hadoop Distributed File System</a></b> performance statistics. Module collects metrics over <code>Java Management Extensions</code> through the web interface of an <code>HDFS</code> daemon.'
519
}
520
};
521
@@ -2625,5 +2637,17 @@ netdataDashboard.context = {
2637
'<code>2</code>: non-security updates are available; ' +
2638
'<code>3</code>: security updates are available; ' +
2639
'<code>4</code>: an error retrieving information on software updates.'
2640
+ },
2641
+
2642
+ // ------------------------------------------------------------------------
2643
+ // Zookeeper
2644
+
2645
+ 'zookeeper.server_state': {
2646
+ info:
2647
+ '<code>0</code>: unknown, ' +
2648
+ '<code>1</code>: leader, ' +
2649
+ '<code>2</code>: follower, ' +
2650
+ '<code>3</code>: observer, ' +
2651
+ '<code>4</code>: standalone.'
2652
}
2653
};