@cryptotaxi247 / netdata-1 / commits / 622f19235

go.d zfspool: collect vdev health state (#18383)

Ilya Mashchenko committed Aug 21, 2024 at 12:52 UTC 622f19235bd0a286f9efe20a5f1ee6fdd491bc7a
11 files changed +770 -188
src/go/plugin/go.d/modules/zfspool/charts.go
+77 -17
@@ -10,22 +10,44 @@ import (
10 )
11
12 const (
13 - prioZpoolSpaceUtilization = 2820 + iota
13 + prioZpoolHealthState = 2820 + iota
14 + prioVdevHealthState
15 +
16 + prioZpoolSpaceUtilization
17 prioZpoolSpaceUsage
18 +
19 prioZpoolFragmentation
16 - prioZpoolHealthState
20 )
21
22 var zpoolChartsTmpl = module.Charts{
23 + zpoolHealthStateChartTmpl.Copy(),
24 +
25 zpoolSpaceUtilizationChartTmpl.Copy(),
26 zpoolSpaceUsageChartTmpl.Copy(),
27
28 zpoolFragmentationChartTmpl.Copy(),
24 -
25 - zpoolHealthStateChartTmpl.Copy(),
29 }
30
31 var (
32 + zpoolHealthStateChartTmpl = module.Chart{
33 + ID: "zfspool_%s_health_state",
34 + Title: "Zpool health state",
35 + Units: "state",
36 + Fam: "health",
37 + Ctx: "zfspool.pool_health_state",
38 + Type: module.Line,
39 + Priority: prioZpoolHealthState,
40 + Dims: module.Dims{
41 + {ID: "zpool_%s_health_state_online", Name: "online"},
42 + {ID: "zpool_%s_health_state_degraded", Name: "degraded"},
43 + {ID: "zpool_%s_health_state_faulted", Name: "faulted"},
44 + {ID: "zpool_%s_health_state_offline", Name: "offline"},
45 + {ID: "zpool_%s_health_state_unavail", Name: "unavail"},
46 + {ID: "zpool_%s_health_state_removed", Name: "removed"},
47 + {ID: "zpool_%s_health_state_suspended", Name: "suspended"},
48 + },
49 + }
50 +
51 zpoolSpaceUtilizationChartTmpl = module.Chart{
52 ID: "zfspool_%s_space_utilization",
53 Title: "Zpool space utilization",
@@ -64,23 +86,29 @@ var (
86 {ID: "zpool_%s_frag", Name: "fragmentation"},
87 },
88 }
89 +)
90
68 - zpoolHealthStateChartTmpl = module.Chart{
69 - ID: "zfspool_%s_health_state",
70 - Title: "Zpool health state",
91 +var vdevChartsTmpl = module.Charts{
92 + vdevHealthStateChartTmpl.Copy(),
93 +}
94 +
95 +var (
96 + vdevHealthStateChartTmpl = module.Chart{
97 + ID: "vdev_%s_health_state",
98 + Title: "Zpool Vdev health state",
99 Units: "state",
100 Fam: "health",
73 - Ctx: "zfspool.pool_health_state",
101 + Ctx: "zfspool.vdev_health_state",
102 Type: module.Line,
75 - Priority: prioZpoolHealthState,
103 + Priority: prioVdevHealthState,
104 Dims: module.Dims{
77 - {ID: "zpool_%s_health_state_online", Name: "online"},
78 - {ID: "zpool_%s_health_state_degraded", Name: "degraded"},
79 - {ID: "zpool_%s_health_state_faulted", Name: "faulted"},
80 - {ID: "zpool_%s_health_state_offline", Name: "offline"},
81 - {ID: "zpool_%s_health_state_unavail", Name: "unavail"},
82 - {ID: "zpool_%s_health_state_removed", Name: "removed"},
83 - {ID: "zpool_%s_health_state_suspended", Name: "suspended"},
105 + {ID: "vdev_%s_health_state_online", Name: "online"},
106 + {ID: "vdev_%s_health_state_degraded", Name: "degraded"},
107 + {ID: "vdev_%s_health_state_faulted", Name: "faulted"},
108 + {ID: "vdev_%s_health_state_offline", Name: "offline"},
109 + {ID: "vdev_%s_health_state_unavail", Name: "unavail"},
110 + {ID: "vdev_%s_health_state_removed", Name: "removed"},
111 + {ID: "vdev_%s_health_state_suspended", Name: "suspended"},
112 },
113 }
114 )
@@ -104,8 +132,35 @@ func (z *ZFSPool) addZpoolCharts(name string) {
132 }
133
134 func (z *ZFSPool) removeZpoolCharts(name string) {
107 - px := fmt.Sprintf("zpool_%s_", name)
135 + px := fmt.Sprintf("zfspool_%s_", name)
136 + z.removeCharts(px)
137 +}
138
139 +func (z *ZFSPool) addVdevCharts(pool, vdev string) {
140 + charts := vdevChartsTmpl.Copy()
141 +
142 + for _, chart := range *charts {
143 + chart.ID = fmt.Sprintf(chart.ID, cleanVdev(vdev))
144 + chart.Labels = []module.Label{
145 + {Key: "pool", Value: pool},
146 + {Key: "vdev", Value: vdev},
147 + }
148 + for _, dim := range chart.Dims {
149 + dim.ID = fmt.Sprintf(dim.ID, vdev)
150 + }
151 + }
152 +
153 + if err := z.Charts().Add(*charts...); err != nil {
154 + z.Warning(err)
155 + }
156 +}
157 +
158 +func (z *ZFSPool) removeVdevCharts(vdev string) {
159 + px := fmt.Sprintf("vdev_%s_", cleanVdev(vdev))
160 + z.removeCharts(px)
161 +}
162 +
163 +func (z *ZFSPool) removeCharts(px string) {
164 for _, chart := range *z.Charts() {
165 if strings.HasPrefix(chart.ID, px) {
166 chart.MarkRemove()
@@ -113,3 +168,8 @@ func (z *ZFSPool) removeZpoolCharts(name string) {
168 }
169 }
170 }
171 +
172 +func cleanVdev(vdev string) string {
173 + r := strings.NewReplacer(".", "_")
174 + return r.Replace(vdev)
175 +}
src/go/plugin/go.d/modules/zfspool/collect.go
+5 -155
@@ -2,14 +2,6 @@
2
3 package zfspool
4
5 -import (
6 - "bufio"
7 - "bytes"
8 - "fmt"
9 - "strconv"
10 - "strings"
11 -)
12 -
5 var zpoolHealthStates = []string{
6 "online",
7 "degraded",
@@ -20,158 +12,16 @@ var zpoolHealthStates = []string{
12 "suspended",
13 }
14
23 -type zpoolStats struct {
24 - name string
25 - sizeBytes string
26 - allocBytes string
27 - freeBytes string
28 - fragPerc string
29 - capPerc string
30 - dedupRatio string
31 - health string
32 -}
33 -
15 func (z *ZFSPool) collect() (map[string]int64, error) {
35 - bs, err := z.exec.list()
36 - if err != nil {
37 - return nil, err
38 - }
39 -
40 - zpools, err := parseZpoolListOutput(bs)
41 - if err != nil {
42 - return nil, err
43 - }
16
17 mx := make(map[string]int64)
18
47 - z.collectZpoolListStats(mx, zpools)
48 -
49 - return mx, nil
50 -}
51 -
52 -func (z *ZFSPool) collectZpoolListStats(mx map[string]int64, zpools []zpoolStats) {
53 - seen := make(map[string]bool)
54 -
55 - for _, zpool := range zpools {
56 - seen[zpool.name] = true
57 -
58 - if !z.zpools[zpool.name] {
59 - z.addZpoolCharts(zpool.name)
60 - z.zpools[zpool.name] = true
61 - }
62 -
63 - px := "zpool_" + zpool.name + "_"
64 -
65 - if v, ok := parseInt(zpool.sizeBytes); ok {
66 - mx[px+"size"] = v
67 - }
68 - if v, ok := parseInt(zpool.freeBytes); ok {
69 - mx[px+"free"] = v
70 - }
71 - if v, ok := parseInt(zpool.allocBytes); ok {
72 - mx[px+"alloc"] = v
73 - }
74 - if v, ok := parseFloat(zpool.capPerc); ok {
75 - mx[px+"cap"] = int64(v)
76 - }
77 - if v, ok := parseFloat(zpool.fragPerc); ok {
78 - mx[px+"frag"] = int64(v)
79 - }
80 - for _, s := range zpoolHealthStates {
81 - mx[px+"health_state_"+s] = 0
82 - }
83 - mx[px+"health_state_"+zpool.health] = 1
84 - }
85 -
86 - for name := range z.zpools {
87 - if !seen[name] {
88 - z.removeZpoolCharts(name)
89 - delete(z.zpools, name)
90 - }
91 - }
92 -}
93 -
94 -func parseZpoolListOutput(bs []byte) ([]zpoolStats, error) {
95 - var lines []string
96 - sc := bufio.NewScanner(bytes.NewReader(bs))
97 - for sc.Scan() {
98 - if text := strings.TrimSpace(sc.Text()); text != "" {
99 - lines = append(lines, text)
100 - }
101 -
102 - }
103 - if len(lines) < 2 {
104 - return nil, fmt.Errorf("unexpected data: wanted >= 2 lines, got %d", len(lines))
105 - }
106 -
107 - headers := strings.Fields(lines[0])
108 - if len(headers) == 0 {
109 - return nil, fmt.Errorf("unexpected data: missing headers")
110 - }
111 -
112 - var zpools []zpoolStats
113 -
114 - /*
115 - # zpool list -p
116 - NAME SIZE ALLOC FREE EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
117 - rpool 21367462298 9051643576 12240656794 - 33 42 1.00 ONLINE -
118 - zion - - - - - - - FAULTED -
119 - */
120 -
121 - for _, line := range lines[1:] {
122 - values := strings.Fields(line)
123 - if len(values) != len(headers) {
124 - return nil, fmt.Errorf("unequal columns: headers(%d) != values(%d)", len(headers), len(values))
125 - }
126 -
127 - var zpool zpoolStats
128 -
129 - for i, v := range values {
130 - v = strings.TrimSpace(v)
131 - switch strings.ToLower(headers[i]) {
132 - case "name":
133 - zpool.name = v
134 - case "size":
135 - zpool.sizeBytes = v
136 - case "alloc":
137 - zpool.allocBytes = v
138 - case "free":
139 - zpool.freeBytes = v
140 - case "frag":
141 - zpool.fragPerc = v
142 - case "cap":
143 - zpool.capPerc = v
144 - case "dedup":
145 - zpool.dedupRatio = v
146 - case "health":
147 - zpool.health = strings.ToLower(v)
148 - }
149 -
150 - if last := i+1 == len(headers); last && zpool.name != "" && zpool.health != "" {
151 - zpools = append(zpools, zpool)
152 - }
153 - }
154 - }
155 -
156 - if len(zpools) == 0 {
157 - return nil, fmt.Errorf("unexpected data: missing pools")
19 + if err := z.collectZpoolList(mx); err != nil {
20 + return nil, err
21 }
159 -
160 - return zpools, nil
161 -}
162 -
163 -func parseInt(s string) (int64, bool) {
164 - if s == "-" {
165 - return 0, false
22 + if err := z.collectZpoolListVdev(mx); err != nil {
23 + return mx, err
24 }
167 - v, err := strconv.ParseInt(s, 10, 64)
168 - return v, err == nil
169 -}
25
171 -func parseFloat(s string) (float64, bool) {
172 - if s == "-" {
173 - return 0, false
174 - }
175 - v, err := strconv.ParseFloat(s, 64)
176 - return v, err == nil
26 + return mx, nil
27 }
src/go/plugin/go.d/modules/zfspool/collect_zpool_list.go new
+160
@@ -0,0 +1,160 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package zfspool
4 +
5 +import (
6 + "bufio"
7 + "bytes"
8 + "errors"
9 + "fmt"
10 + "strconv"
11 + "strings"
12 +)
13 +
14 +type zpoolEntry struct {
15 + name string
16 + sizeBytes string
17 + allocBytes string
18 + freeBytes string
19 + fragPerc string
20 + capPerc string
21 + dedupRatio string
22 + health string
23 +}
24 +
25 +func (z *ZFSPool) collectZpoolList(mx map[string]int64) error {
26 + bs, err := z.exec.list()
27 + if err != nil {
28 + return err
29 + }
30 +
31 + zpools, err := parseZpoolListOutput(bs)
32 + if err != nil {
33 + return fmt.Errorf("bad zpool list output: %v", err)
34 + }
35 +
36 + seen := make(map[string]bool)
37 +
38 + for _, zpool := range zpools {
39 + seen[zpool.name] = true
40 +
41 + if !z.seenZpools[zpool.name] {
42 + z.addZpoolCharts(zpool.name)
43 + z.seenZpools[zpool.name] = true
44 + }
45 +
46 + px := "zpool_" + zpool.name + "_"
47 +
48 + if v, ok := parseInt(zpool.sizeBytes); ok {
49 + mx[px+"size"] = v
50 + }
51 + if v, ok := parseInt(zpool.freeBytes); ok {
52 + mx[px+"free"] = v
53 + }
54 + if v, ok := parseInt(zpool.allocBytes); ok {
55 + mx[px+"alloc"] = v
56 + }
57 + if v, ok := parseFloat(zpool.capPerc); ok {
58 + mx[px+"cap"] = int64(v)
59 + }
60 + if v, ok := parseFloat(zpool.fragPerc); ok {
61 + mx[px+"frag"] = int64(v)
62 + }
63 + for _, s := range zpoolHealthStates {
64 + mx[px+"health_state_"+s] = 0
65 + }
66 + mx[px+"health_state_"+zpool.health] = 1
67 + }
68 +
69 + for name := range z.seenZpools {
70 + if !seen[name] {
71 + z.removeZpoolCharts(name)
72 + delete(z.seenZpools, name)
73 + }
74 + }
75 +
76 + return nil
77 +}
78 +
79 +func parseZpoolListOutput(bs []byte) ([]zpoolEntry, error) {
80 + /*
81 + # zpool list -p
82 + NAME SIZE ALLOC FREE EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
83 + rpool 21367462298 9051643576 12240656794 - 33 42 1.00 ONLINE -
84 + zion - - - - - - - FAULTED -
85 + */
86 +
87 + var headers []string
88 + var zpools []zpoolEntry
89 + sc := bufio.NewScanner(bytes.NewReader(bs))
90 +
91 + for sc.Scan() {
92 + line := strings.TrimSpace(sc.Text())
93 + if line == "" {
94 + continue
95 + }
96 +
97 + if len(headers) == 0 {
98 + if !strings.HasPrefix(line, "NAME") {
99 + return nil, fmt.Errorf("missing headers (line '%s')", line)
100 + }
101 + headers = strings.Fields(line)
102 + continue
103 + }
104 +
105 + values := strings.Fields(line)
106 + if len(values) != len(headers) {
107 + return nil, fmt.Errorf("unequal columns: headers(%d) != values(%d)", len(headers), len(values))
108 + }
109 +
110 + var zpool zpoolEntry
111 +
112 + for i, v := range values {
113 + v = strings.TrimSpace(v)
114 + switch strings.ToLower(headers[i]) {
115 + case "name":
116 + zpool.name = v
117 + case "size":
118 + zpool.sizeBytes = v
119 + case "alloc":
120 + zpool.allocBytes = v
121 + case "free":
122 + zpool.freeBytes = v
123 + case "frag":
124 + zpool.fragPerc = v
125 + case "cap":
126 + zpool.capPerc = v
127 + case "dedup":
128 + zpool.dedupRatio = v
129 + case "health":
130 + zpool.health = strings.ToLower(v)
131 + }
132 + }
133 +
134 + if zpool.name != "" && zpool.health != "" {
135 + zpools = append(zpools, zpool)
136 + }
137 + }
138 +
139 + if len(zpools) == 0 {
140 + return nil, errors.New("no pools found")
141 + }
142 +
143 + return zpools, nil
144 +}
145 +
146 +func parseInt(s string) (int64, bool) {
147 + if s == "-" {
148 + return 0, false
149 + }
150 + v, err := strconv.ParseInt(s, 10, 64)
151 + return v, err == nil
152 +}
153 +
154 +func parseFloat(s string) (float64, bool) {
155 + if s == "-" {
156 + return 0, false
157 + }
158 + v, err := strconv.ParseFloat(s, 64)
159 + return v, err == nil
160 +}
src/go/plugin/go.d/modules/zfspool/collect_zpool_list_vdev.go new
+138
@@ -0,0 +1,138 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package zfspool
4 +
5 +import (
6 + "bufio"
7 + "bytes"
8 + "fmt"
9 + "strings"
10 +)
11 +
12 +type vdevEntry struct {
13 + name string
14 + vdev string // The full path of the vdev within the zpool hierarchy.
15 + health string
16 +
17 + // Represents the nesting level of the vdev within the zpool hierarchy, based on indentation.
18 + // A level of -1 indicates the root vdev (the pool itself).
19 + level int
20 +}
21 +
22 +func (z *ZFSPool) collectZpoolListVdev(mx map[string]int64) error {
23 + seen := make(map[string]bool)
24 +
25 + for pool := range z.seenZpools {
26 + bs, err := z.exec.listWithVdev(pool)
27 + if err != nil {
28 + return err
29 + }
30 +
31 + vdevs, err := parseZpoolListVdevOutput(bs)
32 + if err != nil {
33 + return fmt.Errorf("bad zpool list vdev output (pool '%s'): %v", pool, err)
34 + }
35 +
36 + for _, vdev := range vdevs {
37 + if vdev.health == "" || vdev.health == "-" {
38 + continue
39 + }
40 +
41 + seen[vdev.vdev] = true
42 + if !z.seenVdevs[vdev.vdev] {
43 + z.seenVdevs[vdev.vdev] = true
44 + z.addVdevCharts(pool, vdev.vdev)
45 + }
46 +
47 + px := fmt.Sprintf("vdev_%s_", vdev.vdev)
48 +
49 + for _, s := range zpoolHealthStates {
50 + mx[px+"health_state_"+s] = 0
51 + }
52 + mx[px+"health_state_"+vdev.health] = 1
53 + }
54 + }
55 +
56 + for name := range z.seenVdevs {
57 + if !seen[name] {
58 + z.removeVdevCharts(name)
59 + delete(z.seenVdevs, name)
60 + }
61 + }
62 +
63 + return nil
64 +}
65 +
66 +func parseZpoolListVdevOutput(bs []byte) ([]vdevEntry, error) {
67 + var headers []string
68 + var vdevs []vdevEntry
69 + sc := bufio.NewScanner(bytes.NewReader(bs))
70 +
71 + for sc.Scan() {
72 + line := sc.Text()
73 + if line == "" {
74 + continue
75 + }
76 +
77 + if len(headers) == 0 {
78 + if !strings.HasPrefix(line, "NAME") {
79 + return nil, fmt.Errorf("missing headers (line '%s')", line)
80 + }
81 + headers = strings.Fields(line)
82 + continue
83 + }
84 +
85 + values := strings.Fields(line)
86 + if len(values) == 0 || len(values) > len(headers) {
87 + return nil, fmt.Errorf("unexpected columns: headers(%d) values(%d) (line '%s')", len(headers), len(values), line)
88 + }
89 +
90 + vdev := vdevEntry{
91 + level: len(line) - len(strings.TrimLeft(line, " ")),
92 + }
93 +
94 + for i, v := range values {
95 + switch strings.ToLower(headers[i]) {
96 + case "name":
97 + vdev.name = v
98 + case "health":
99 + vdev.health = strings.ToLower(v)
100 + }
101 + }
102 +
103 + if vdev.name != "" {
104 + if len(vdevs) == 0 {
105 + vdev.level = -1 // Pool
106 + }
107 + vdevs = append(vdevs, vdev)
108 + }
109 + }
110 +
111 + // set parent/child relationships
112 + for i := range vdevs {
113 + v := &vdevs[i]
114 +
115 + switch i {
116 + case 0:
117 + v.vdev = v.name
118 + default:
119 + // find parent with a lower level
120 + for j := i - 1; j >= 0; j-- {
121 + if vdevs[j].level < v.level {
122 + v.vdev = fmt.Sprintf("%s/%s", vdevs[j].vdev, v.name)
123 + break
124 + }
125 + }
126 + if v.vdev == "" {
127 + return nil, fmt.Errorf("no parent for vdev '%s'", v.name)
128 + }
129 + }
130 + }
131 +
132 + // first is Pool
133 + if len(vdevs) < 2 {
134 + return nil, fmt.Errorf("no vdevs found")
135 + }
136 +
137 + return vdevs[1:], nil
138 +}
src/go/plugin/go.d/modules/zfspool/exec.go
+15
@@ -39,3 +39,18 @@ func (e *zpoolCLIExec) list() ([]byte, error) {
39
40 return bs, nil
41 }
42 +
43 +func (e *zpoolCLIExec) listWithVdev(pool string) ([]byte, error) {
44 + ctx, cancel := context.WithTimeout(context.Background(), e.timeout)
45 + defer cancel()
46 +
47 + cmd := exec.CommandContext(ctx, e.binPath, "list", "-p", "-v", "-L", pool)
48 + e.Debugf("executing '%s'", cmd)
49 +
50 + bs, err := cmd.Output()
51 + if err != nil {
52 + return nil, fmt.Errorf("error on '%s': %v", cmd, err)
53 + }
54 +
55 + return bs, nil
56 +}
src/go/plugin/go.d/modules/zfspool/metadata.yaml
+24
@@ -92,6 +92,10 @@ modules:
92 metric: zfspool.pool_health_state
93 info: "ZFS pool ${label:pool} state is faulted or unavail"
94 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/zfs.conf
95 + - name: zfs_vdev_health_state
96 + metric: zfspool.vdev_health_state
97 + info: "ZFS vdev ${label:vdev} state is faulted or degraded"
98 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/zfs.conf
99 metrics:
100 folding:
101 title: Metrics
@@ -136,3 +140,23 @@ modules:
140 - name: unavail
141 - name: removed
142 - name: suspended
143 + - name: zfs pool vdev
144 + description: These metrics refer to the ZFS pool virtual device.
145 + labels:
146 + - name: pool
147 + description: Zpool name
148 + - name: vdev
149 + description: Unique identifier for a virtual device (vdev) within a ZFS pool.
150 + metrics:
151 + - name: zfspool.vdev_health_state
152 + description: Zpool Vdev health state
153 + unit: 'state'
154 + chart_type: line
155 + dimensions:
156 + - name: online
157 + - name: degraded
158 + - name: faulted
159 + - name: offline
160 + - name: unavail
161 + - name: removed
162 + - name: suspended
src/go/plugin/go.d/modules/zfspool/testdata/zpool-list-vdev-logs-cache.txt new
+12
@@ -0,0 +1,12 @@
1 +NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
2 +rpool 9981503995904 3046188658688 6935315337216 - - 9 30 1.00 DEGRADED -
3 + mirror-0 9981503995904 3046188658688 6935315337216 - - 9 30 - ONLINE
4 + sdc2 9998683602944 - - - - - - - ONLINE
5 + sdd2 9998683602944 - - - - - - - ONLINE
6 +logs - - - - - - - - -
7 + mirror-1 17716740096 393216 17716346880 - - 0 0 - DEGRADED
8 + sdb1 17951621120 - - - - - - - ONLINE
9 + 14807975228228307538 - - - - - - - - UNAVAIL
10 +cache - - - - - - - - -
11 + sdb2 99000254464 98755866624 239665152 - - 0 99 - ONLINE
12 + wwn-0x500151795954c095-part2 - - - - - - - - UNAVAIL
src/go/plugin/go.d/modules/zfspool/testdata/zpool-list-vdev.txt new
+5
@@ -0,0 +1,5 @@
1 +NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
2 +rpool 3985729650688 1647130456064 2338599194624 - - 55 41 1.00 ONLINE -
3 + mirror-0 3985729650688 1647130456064 2338599194624 - - 55 41 - ONLINE
4 + nvme2n1p3 4000249020416 - - - - - - - ONLINE
5 + nvme0n1p3 4000249020416 - - - - - - - ONLINE
src/go/plugin/go.d/modules/zfspool/zfspool.go
+6 -3
@@ -31,8 +31,9 @@ func New() *ZFSPool {
31 BinaryPath: "/usr/bin/zpool",
32 Timeout: web.Duration(time.Second * 2),
33 },
34 - charts: &module.Charts{},
35 - zpools: make(map[string]bool),
34 + charts: &module.Charts{},
35 + seenZpools: make(map[string]bool),
36 + seenVdevs: make(map[string]bool),
37 }
38 }
39
@@ -51,10 +52,12 @@ type (
52
53 exec zpoolCLI
54
54 - zpools map[string]bool
55 + seenZpools map[string]bool
56 + seenVdevs map[string]bool
57 }
58 zpoolCLI interface {
59 list() ([]byte, error)
60 + listWithVdev(pool string) ([]byte, error)
61 }
62 )
63
src/go/plugin/go.d/modules/zfspool/zfspool_test.go
+311 -11
@@ -5,6 +5,7 @@ package zfspool
5 import (
6 "errors"
7 "os"
8 + "strings"
9 "testing"
10
11 "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
@@ -17,7 +18,9 @@ var (
18 dataConfigJSON, _ = os.ReadFile("testdata/config.json")
19 dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")
20
20 - dataZpoolList, _ = os.ReadFile("testdata/zpool-list.txt")
21 + dataZpoolList, _ = os.ReadFile("testdata/zpool-list.txt")
22 + dataZpoolListWithVdev, _ = os.ReadFile("testdata/zpool-list-vdev.txt")
23 + dataZpoolListWithVdevLogsCache, _ = os.ReadFile("testdata/zpool-list-vdev-logs-cache.txt")
24 )
25
26 func Test_testDataIsValid(t *testing.T) {
@@ -25,7 +28,9 @@ func Test_testDataIsValid(t *testing.T) {
28 "dataConfigJSON": dataConfigJSON,
29 "dataConfigYAML": dataConfigYAML,
30
28 - "dataZpoolList": dataZpoolList,
31 + "dataZpoolList": dataZpoolList,
32 + "dataZpoolListWithVdev": dataZpoolListWithVdev,
33 + "dataZpoolListWithVdevLogsCache": dataZpoolListWithVdevLogsCache,
34 } {
35 require.NotNil(t, data, name)
36
@@ -81,7 +86,7 @@ func TestZFSPool_Cleanup(t *testing.T) {
86 "after check": {
87 prepare: func() *ZFSPool {
88 zp := New()
84 - zp.exec = prepareMockOK()
89 + zp.exec = prepareMockOk()
90 _ = zp.Check()
91 return zp
92 },
@@ -89,7 +94,7 @@ func TestZFSPool_Cleanup(t *testing.T) {
94 "after collect": {
95 prepare: func() *ZFSPool {
96 zp := New()
92 - zp.exec = prepareMockOK()
97 + zp.exec = prepareMockOk()
98 _ = zp.Collect()
99 return zp
100 },
@@ -115,7 +120,7 @@ func TestZFSPool_Check(t *testing.T) {
120 wantFail bool
121 }{
122 "success case": {
118 - prepareMock: prepareMockOK,
123 + prepareMock: prepareMockOk,
124 wantFail: false,
125 },
126 "error on list call": {
@@ -153,8 +158,186 @@ func TestZFSPool_Collect(t *testing.T) {
158 wantMetrics map[string]int64
159 }{
160 "success case": {
156 - prepareMock: prepareMockOK,
161 + prepareMock: prepareMockOk,
162 wantMetrics: map[string]int64{
163 + "vdev_rpool/mirror-0/nvme0n1p3_health_state_degraded": 0,
164 + "vdev_rpool/mirror-0/nvme0n1p3_health_state_faulted": 0,
165 + "vdev_rpool/mirror-0/nvme0n1p3_health_state_offline": 0,
166 + "vdev_rpool/mirror-0/nvme0n1p3_health_state_online": 1,
167 + "vdev_rpool/mirror-0/nvme0n1p3_health_state_removed": 0,
168 + "vdev_rpool/mirror-0/nvme0n1p3_health_state_suspended": 0,
169 + "vdev_rpool/mirror-0/nvme0n1p3_health_state_unavail": 0,
170 + "vdev_rpool/mirror-0/nvme2n1p3_health_state_degraded": 0,
171 + "vdev_rpool/mirror-0/nvme2n1p3_health_state_faulted": 0,
172 + "vdev_rpool/mirror-0/nvme2n1p3_health_state_offline": 0,
173 + "vdev_rpool/mirror-0/nvme2n1p3_health_state_online": 1,
174 + "vdev_rpool/mirror-0/nvme2n1p3_health_state_removed": 0,
175 + "vdev_rpool/mirror-0/nvme2n1p3_health_state_suspended": 0,
176 + "vdev_rpool/mirror-0/nvme2n1p3_health_state_unavail": 0,
177 + "vdev_rpool/mirror-0_health_state_degraded": 0,
178 + "vdev_rpool/mirror-0_health_state_faulted": 0,
179 + "vdev_rpool/mirror-0_health_state_offline": 0,
180 + "vdev_rpool/mirror-0_health_state_online": 1,
181 + "vdev_rpool/mirror-0_health_state_removed": 0,
182 + "vdev_rpool/mirror-0_health_state_suspended": 0,
183 + "vdev_rpool/mirror-0_health_state_unavail": 0,
184 + "vdev_zion/mirror-0/nvme0n1p3_health_state_degraded": 0,
185 + "vdev_zion/mirror-0/nvme0n1p3_health_state_faulted": 0,
186 + "vdev_zion/mirror-0/nvme0n1p3_health_state_offline": 0,
187 + "vdev_zion/mirror-0/nvme0n1p3_health_state_online": 1,
188 + "vdev_zion/mirror-0/nvme0n1p3_health_state_removed": 0,
189 + "vdev_zion/mirror-0/nvme0n1p3_health_state_suspended": 0,
190 + "vdev_zion/mirror-0/nvme0n1p3_health_state_unavail": 0,
191 + "vdev_zion/mirror-0/nvme2n1p3_health_state_degraded": 0,
192 + "vdev_zion/mirror-0/nvme2n1p3_health_state_faulted": 0,
193 + "vdev_zion/mirror-0/nvme2n1p3_health_state_offline": 0,
194 + "vdev_zion/mirror-0/nvme2n1p3_health_state_online": 1,
195 + "vdev_zion/mirror-0/nvme2n1p3_health_state_removed": 0,
196 + "vdev_zion/mirror-0/nvme2n1p3_health_state_suspended": 0,
197 + "vdev_zion/mirror-0/nvme2n1p3_health_state_unavail": 0,
198 + "vdev_zion/mirror-0_health_state_degraded": 0,
199 + "vdev_zion/mirror-0_health_state_faulted": 0,
200 + "vdev_zion/mirror-0_health_state_offline": 0,
201 + "vdev_zion/mirror-0_health_state_online": 1,
202 + "vdev_zion/mirror-0_health_state_removed": 0,
203 + "vdev_zion/mirror-0_health_state_suspended": 0,
204 + "vdev_zion/mirror-0_health_state_unavail": 0,
205 + "zpool_rpool_alloc": 9051643576,
206 + "zpool_rpool_cap": 42,
207 + "zpool_rpool_frag": 33,
208 + "zpool_rpool_free": 12240656794,
209 + "zpool_rpool_health_state_degraded": 0,
210 + "zpool_rpool_health_state_faulted": 0,
211 + "zpool_rpool_health_state_offline": 0,
212 + "zpool_rpool_health_state_online": 1,
213 + "zpool_rpool_health_state_removed": 0,
214 + "zpool_rpool_health_state_suspended": 0,
215 + "zpool_rpool_health_state_unavail": 0,
216 + "zpool_rpool_size": 21367462298,
217 + "zpool_zion_health_state_degraded": 0,
218 + "zpool_zion_health_state_faulted": 1,
219 + "zpool_zion_health_state_offline": 0,
220 + "zpool_zion_health_state_online": 0,
221 + "zpool_zion_health_state_removed": 0,
222 + "zpool_zion_health_state_suspended": 0,
223 + "zpool_zion_health_state_unavail": 0,
224 + },
225 + },
226 + "success case vdev logs and cache": {
227 + prepareMock: prepareMockOkVdevLogsCache,
228 + wantMetrics: map[string]int64{
229 + "vdev_rpool/cache/sdb2_health_state_degraded": 0,
230 + "vdev_rpool/cache/sdb2_health_state_faulted": 0,
231 + "vdev_rpool/cache/sdb2_health_state_offline": 0,
232 + "vdev_rpool/cache/sdb2_health_state_online": 1,
233 + "vdev_rpool/cache/sdb2_health_state_removed": 0,
234 + "vdev_rpool/cache/sdb2_health_state_suspended": 0,
235 + "vdev_rpool/cache/sdb2_health_state_unavail": 0,
236 + "vdev_rpool/cache/wwn-0x500151795954c095-part2_health_state_degraded": 0,
237 + "vdev_rpool/cache/wwn-0x500151795954c095-part2_health_state_faulted": 0,
238 + "vdev_rpool/cache/wwn-0x500151795954c095-part2_health_state_offline": 0,
239 + "vdev_rpool/cache/wwn-0x500151795954c095-part2_health_state_online": 0,
240 + "vdev_rpool/cache/wwn-0x500151795954c095-part2_health_state_removed": 0,
241 + "vdev_rpool/cache/wwn-0x500151795954c095-part2_health_state_suspended": 0,
242 + "vdev_rpool/cache/wwn-0x500151795954c095-part2_health_state_unavail": 1,
243 + "vdev_rpool/logs/mirror-1/14807975228228307538_health_state_degraded": 0,
244 + "vdev_rpool/logs/mirror-1/14807975228228307538_health_state_faulted": 0,
245 + "vdev_rpool/logs/mirror-1/14807975228228307538_health_state_offline": 0,
246 + "vdev_rpool/logs/mirror-1/14807975228228307538_health_state_online": 0,
247 + "vdev_rpool/logs/mirror-1/14807975228228307538_health_state_removed": 0,
248 + "vdev_rpool/logs/mirror-1/14807975228228307538_health_state_suspended": 0,
249 + "vdev_rpool/logs/mirror-1/14807975228228307538_health_state_unavail": 1,
250 + "vdev_rpool/logs/mirror-1/sdb1_health_state_degraded": 0,
251 + "vdev_rpool/logs/mirror-1/sdb1_health_state_faulted": 0,
252 + "vdev_rpool/logs/mirror-1/sdb1_health_state_offline": 0,
253 + "vdev_rpool/logs/mirror-1/sdb1_health_state_online": 1,
254 + "vdev_rpool/logs/mirror-1/sdb1_health_state_removed": 0,
255 + "vdev_rpool/logs/mirror-1/sdb1_health_state_suspended": 0,
256 + "vdev_rpool/logs/mirror-1/sdb1_health_state_unavail": 0,
257 + "vdev_rpool/logs/mirror-1_health_state_degraded": 1,
258 + "vdev_rpool/logs/mirror-1_health_state_faulted": 0,
259 + "vdev_rpool/logs/mirror-1_health_state_offline": 0,
260 + "vdev_rpool/logs/mirror-1_health_state_online": 0,
261 + "vdev_rpool/logs/mirror-1_health_state_removed": 0,
262 + "vdev_rpool/logs/mirror-1_health_state_suspended": 0,
263 + "vdev_rpool/logs/mirror-1_health_state_unavail": 0,
264 + "vdev_rpool/mirror-0/sdc2_health_state_degraded": 0,
265 + "vdev_rpool/mirror-0/sdc2_health_state_faulted": 0,
266 + "vdev_rpool/mirror-0/sdc2_health_state_offline": 0,
267 + "vdev_rpool/mirror-0/sdc2_health_state_online": 1,
268 + "vdev_rpool/mirror-0/sdc2_health_state_removed": 0,
269 + "vdev_rpool/mirror-0/sdc2_health_state_suspended": 0,
270 + "vdev_rpool/mirror-0/sdc2_health_state_unavail": 0,
271 + "vdev_rpool/mirror-0/sdd2_health_state_degraded": 0,
272 + "vdev_rpool/mirror-0/sdd2_health_state_faulted": 0,
273 + "vdev_rpool/mirror-0/sdd2_health_state_offline": 0,
274 + "vdev_rpool/mirror-0/sdd2_health_state_online": 1,
275 + "vdev_rpool/mirror-0/sdd2_health_state_removed": 0,
276 + "vdev_rpool/mirror-0/sdd2_health_state_suspended": 0,
277 + "vdev_rpool/mirror-0/sdd2_health_state_unavail": 0,
278 + "vdev_rpool/mirror-0_health_state_degraded": 0,
279 + "vdev_rpool/mirror-0_health_state_faulted": 0,
280 + "vdev_rpool/mirror-0_health_state_offline": 0,
281 + "vdev_rpool/mirror-0_health_state_online": 1,
282 + "vdev_rpool/mirror-0_health_state_removed": 0,
283 + "vdev_rpool/mirror-0_health_state_suspended": 0,
284 + "vdev_rpool/mirror-0_health_state_unavail": 0,
285 + "vdev_zion/cache/sdb2_health_state_degraded": 0,
286 + "vdev_zion/cache/sdb2_health_state_faulted": 0,
287 + "vdev_zion/cache/sdb2_health_state_offline": 0,
288 + "vdev_zion/cache/sdb2_health_state_online": 1,
289 + "vdev_zion/cache/sdb2_health_state_removed": 0,
290 + "vdev_zion/cache/sdb2_health_state_suspended": 0,
291 + "vdev_zion/cache/sdb2_health_state_unavail": 0,
292 + "vdev_zion/cache/wwn-0x500151795954c095-part2_health_state_degraded": 0,
293 + "vdev_zion/cache/wwn-0x500151795954c095-part2_health_state_faulted": 0,
294 + "vdev_zion/cache/wwn-0x500151795954c095-part2_health_state_offline": 0,
295 + "vdev_zion/cache/wwn-0x500151795954c095-part2_health_state_online": 0,
296 + "vdev_zion/cache/wwn-0x500151795954c095-part2_health_state_removed": 0,
297 + "vdev_zion/cache/wwn-0x500151795954c095-part2_health_state_suspended": 0,
298 + "vdev_zion/cache/wwn-0x500151795954c095-part2_health_state_unavail": 1,
299 + "vdev_zion/logs/mirror-1/14807975228228307538_health_state_degraded": 0,
300 + "vdev_zion/logs/mirror-1/14807975228228307538_health_state_faulted": 0,
301 + "vdev_zion/logs/mirror-1/14807975228228307538_health_state_offline": 0,
302 + "vdev_zion/logs/mirror-1/14807975228228307538_health_state_online": 0,
303 + "vdev_zion/logs/mirror-1/14807975228228307538_health_state_removed": 0,
304 + "vdev_zion/logs/mirror-1/14807975228228307538_health_state_suspended": 0,
305 + "vdev_zion/logs/mirror-1/14807975228228307538_health_state_unavail": 1,
306 + "vdev_zion/logs/mirror-1/sdb1_health_state_degraded": 0,
307 + "vdev_zion/logs/mirror-1/sdb1_health_state_faulted": 0,
308 + "vdev_zion/logs/mirror-1/sdb1_health_state_offline": 0,
309 + "vdev_zion/logs/mirror-1/sdb1_health_state_online": 1,
310 + "vdev_zion/logs/mirror-1/sdb1_health_state_removed": 0,
311 + "vdev_zion/logs/mirror-1/sdb1_health_state_suspended": 0,
312 + "vdev_zion/logs/mirror-1/sdb1_health_state_unavail": 0,
313 + "vdev_zion/logs/mirror-1_health_state_degraded": 1,
314 + "vdev_zion/logs/mirror-1_health_state_faulted": 0,
315 + "vdev_zion/logs/mirror-1_health_state_offline": 0,
316 + "vdev_zion/logs/mirror-1_health_state_online": 0,
317 + "vdev_zion/logs/mirror-1_health_state_removed": 0,
318 + "vdev_zion/logs/mirror-1_health_state_suspended": 0,
319 + "vdev_zion/logs/mirror-1_health_state_unavail": 0,
320 + "vdev_zion/mirror-0/sdc2_health_state_degraded": 0,
321 + "vdev_zion/mirror-0/sdc2_health_state_faulted": 0,
322 + "vdev_zion/mirror-0/sdc2_health_state_offline": 0,
323 + "vdev_zion/mirror-0/sdc2_health_state_online": 1,
324 + "vdev_zion/mirror-0/sdc2_health_state_removed": 0,
325 + "vdev_zion/mirror-0/sdc2_health_state_suspended": 0,
326 + "vdev_zion/mirror-0/sdc2_health_state_unavail": 0,
327 + "vdev_zion/mirror-0/sdd2_health_state_degraded": 0,
328 + "vdev_zion/mirror-0/sdd2_health_state_faulted": 0,
329 + "vdev_zion/mirror-0/sdd2_health_state_offline": 0,
330 + "vdev_zion/mirror-0/sdd2_health_state_online": 1,
331 + "vdev_zion/mirror-0/sdd2_health_state_removed": 0,
332 + "vdev_zion/mirror-0/sdd2_health_state_suspended": 0,
333 + "vdev_zion/mirror-0/sdd2_health_state_unavail": 0,
334 + "vdev_zion/mirror-0_health_state_degraded": 0,
335 + "vdev_zion/mirror-0_health_state_faulted": 0,
336 + "vdev_zion/mirror-0_health_state_offline": 0,
337 + "vdev_zion/mirror-0_health_state_online": 1,
338 + "vdev_zion/mirror-0_health_state_removed": 0,
339 + "vdev_zion/mirror-0_health_state_suspended": 0,
340 + "vdev_zion/mirror-0_health_state_unavail": 0,
341 "zpool_rpool_alloc": 9051643576,
342 "zpool_rpool_cap": 42,
343 "zpool_rpool_frag": 33,
@@ -199,16 +382,125 @@ func TestZFSPool_Collect(t *testing.T) {
382 mx := zp.Collect()
383
384 assert.Equal(t, test.wantMetrics, mx)
385 +
386 if len(test.wantMetrics) > 0 {
203 - assert.Len(t, *zp.Charts(), len(zpoolChartsTmpl)*len(zp.zpools))
387 + want := len(zpoolChartsTmpl)*len(zp.seenZpools) + len(vdevChartsTmpl)*len(zp.seenVdevs)
388 +
389 + assert.Len(t, *zp.Charts(), want, "want charts")
390 +
391 + module.TestMetricsHasAllChartsDimsSkip(t, zp.Charts(), mx, func(chart *module.Chart) bool {
392 + return strings.HasPrefix(chart.ID, "zfspool_zion") && !strings.HasSuffix(chart.ID, "health_state")
393 + })
394 }
395 })
396 }
397 }
398
209 -func prepareMockOK() *mockZpoolCLIExec {
399 +func TestZFSPool_parseZpoolListDevOutput(t *testing.T) {
400 + tests := map[string]struct {
401 + input string
402 + want []vdevEntry
403 + }{
404 + "": {
405 + input: `
406 +NAME SIZE ALLOC FREE CKPOINT EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
407 +store 9981503995904 3046188658688 6935315337216 - - 9 30 1.00 DEGRADED -
408 + mirror-0 9981503995904 3046188658688 6935315337216 - - 9 30 - ONLINE
409 + sdc2 9998683602944 - - - - - - - ONLINE
410 + sdd2 9998683602944 - - - - - - - ONLINE
411 +logs - - - - - - - - -
412 + mirror-1 17716740096 393216 17716346880 - - 0 0 - DEGRADED
413 + sdb1 17951621120 - - - - - - - ONLINE
414 + 14807975228228307538 - - - - - - - - UNAVAIL
415 +cache - - - - - - - - -
416 + sdb2 99000254464 98755866624 239665152 - - 0 99 - ONLINE
417 + wwn-0x500151795954c095-part2 - - - - - - - - UNAVAIL
418 +`,
419 + want: []vdevEntry{
420 + {
421 + name: "mirror-0",
422 + health: "online",
423 + vdev: "store/mirror-0",
424 + level: 2,
425 + },
426 + {
427 + name: "sdc2",
428 + health: "online",
429 + vdev: "store/mirror-0/sdc2",
430 + level: 4,
431 + },
432 + {
433 + name: "sdd2",
434 + health: "online",
435 + vdev: "store/mirror-0/sdd2",
436 + level: 4,
437 + },
438 + {
439 + name: "logs",
440 + health: "-",
441 + vdev: "store/logs",
442 + level: 0,
443 + },
444 + {
445 + name: "mirror-1",
446 + health: "degraded",
447 + vdev: "store/logs/mirror-1",
448 + level: 2,
449 + },
450 + {
451 + name: "sdb1",
452 + health: "online",
453 + vdev: "store/logs/mirror-1/sdb1",
454 + level: 4,
455 + },
456 + {
457 + name: "14807975228228307538",
458 + health: "unavail",
459 + vdev: "store/logs/mirror-1/14807975228228307538",
460 + level: 4,
461 + },
462 + {
463 + name: "cache",
464 + health: "-",
465 + vdev: "store/cache",
466 + level: 0,
467 + },
468 + {
469 + name: "sdb2",
470 + health: "online",
471 + vdev: "store/cache/sdb2",
472 + level: 2,
473 + },
474 + {
475 + name: "wwn-0x500151795954c095-part2",
476 + health: "unavail",
477 + vdev: "store/cache/wwn-0x500151795954c095-part2",
478 + level: 2,
479 + },
480 + },
481 + },
482 + }
483 +
484 + for name, test := range tests {
485 + t.Run(name, func(t *testing.T) {
486 + v, err := parseZpoolListVdevOutput([]byte(test.input))
487 + require.NoError(t, err)
488 + assert.Equal(t, test.want, v)
489 + })
490 + }
491 +}
492 +
493 +func prepareMockOk() *mockZpoolCLIExec {
494 return &mockZpoolCLIExec{
211 - listData: dataZpoolList,
495 + listData: dataZpoolList,
496 + listWithVdevData: dataZpoolListWithVdev,
497 + }
498 +}
499 +
500 +func prepareMockOkVdevLogsCache() *mockZpoolCLIExec {
501 + return &mockZpoolCLIExec{
502 + listData: dataZpoolList,
503 + listWithVdevData: dataZpoolListWithVdevLogsCache,
504 }
505 }
506
@@ -233,8 +525,9 @@ Fusce et felis pulvinar, posuere sem non, porttitor eros.
525 }
526
527 type mockZpoolCLIExec struct {
236 - errOnList bool
237 - listData []byte
528 + errOnList bool
529 + listData []byte
530 + listWithVdevData []byte
531 }
532
533 func (m *mockZpoolCLIExec) list() ([]byte, error) {
@@ -244,3 +537,10 @@ func (m *mockZpoolCLIExec) list() ([]byte, error) {
537
538 return m.listData, nil
539 }
540 +
541 +func (m *mockZpoolCLIExec) listWithVdev(pool string) ([]byte, error) {
542 + s := string(m.listWithVdevData)
543 + s = strings.Replace(s, "rpool", pool, 1)
544 +
545 + return []byte(s), nil
546 +}
src/health/health.d/zfs.conf
+17 -2
@@ -67,7 +67,7 @@ component: File system
67 type: System
68 component: File system
69 calc: $degraded
70 - units: boolean
70 + units: status
71 every: 10s
72 warn: $this > 0
73 delay: down 1m multiplier 1.5 max 1h
@@ -81,10 +81,25 @@ component: File system
81 type: System
82 component: File system
83 calc: $faulted + $unavail
84 - units: boolean
84 + units: status
85 every: 10s
86 crit: $this > 0
87 delay: down 1m multiplier 1.5 max 1h
88 summary: Critical ZFS pool ${label:pool} state
89 info: ZFS pool ${label:pool} state is faulted or unavail
90 to: sysadmin
91 +
92 +
93 + template: zfs_vdev_health_state
94 + on: zfspool.vdev_health_state
95 + class: Errors
96 + type: System
97 +component: File system
98 + calc: $degraded + $faulted
99 + units: status
100 + every: 10s
101 + warn: $this > 0
102 + delay: down 1m multiplier 1.5 max 1h
103 + summary: ZFS vdev ${label:vdev} pool ${label:pool} state
104 + info: ZFS vdev ${label:vdev} state is faulted or degraded
105 + to: sysadmin