@cryptotaxi247 / netdata-1 / commits / 3da8b8773

improvement(go.d/zookeeper): add more metrics (#19584)

Ilya Mashchenko committed Feb 6, 2025 at 15:07 UTC 3da8b87733469000551119c5b69fd9db79619d4b
7 files changed +650 -276
src/go/plugin/go.d/collector/zookeeper/charts.go
+234 -81
@@ -4,108 +4,261 @@ package zookeeper
4
5 import "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
6
7 -type (
8 - Charts = module.Charts
9 - Dims = module.Dims
10 - Vars = module.Vars
7 +const (
8 + prioRequests = module.Priority + iota
9 + prioRequestsLatency
10 + prioStaleRequests
11 + prioStaleRequestsDropped
12 +
13 + prioAliveConnections
14 + prioConnectionsDropped
15 + prioConnectionsRejected
16 + prioAuthFails
17 + prioGlobalSessions
18 +
19 + prioServerState
20 +
21 + prioThrottledOps
22 +
23 + prioPackets
24 +
25 + prioFileDescriptors
26 +
27 + prioNodesCount
28 + prioWatchesCount
29 + prioApproxDataSize
30 +
31 + prioUptime
32 )
33
13 -var charts = Charts{
14 - {
15 - ID: "requests",
16 - Title: "Outstanding Requests",
17 - Units: "requests",
18 - Fam: "requests",
19 - Ctx: "zookeeper.requests",
20 - Dims: Dims{
34 +var charts = module.Charts{
35 + chartRequests.Copy(),
36 + chartRequestsLatency.Copy(),
37 + chartStaleRequests.Copy(),
38 + chartStaleRequestsDropped.Copy(),
39 +
40 + chartAliveConnections.Copy(),
41 + chartConnectionsDropped.Copy(),
42 + chartConnectionsRejected.Copy(),
43 + chartAuthFailed.Copy(),
44 + chartGlobalSessions.Copy(),
45 +
46 + chartServerState.Copy(),
47 + chartThrottledOps.Copy(),
48 +
49 + chartPackets.Copy(),
50 +
51 + chartFileDescriptors.Copy(),
52 +
53 + chartNodesCount.Copy(),
54 + chartWatchesCount.Copy(),
55 + chartApproxDataSize.Copy(),
56 +
57 + chartUptime.Copy(),
58 +}
59 +
60 +var (
61 + chartRequests = module.Chart{
62 + ID: "requests",
63 + Title: "Outstanding Requests",
64 + Units: "requests",
65 + Fam: "requests",
66 + Ctx: "zookeeper.requests",
67 + Priority: prioRequests,
68 + Dims: module.Dims{
69 {ID: "outstanding_requests", Name: "outstanding"},
70 },
23 - },
24 - {
25 - ID: "requests_latency",
26 - Title: "Requests Latency",
27 - Units: "ms",
28 - Fam: "requests",
29 - Ctx: "zookeeper.requests_latency",
30 - Dims: Dims{
71 + }
72 + chartRequestsLatency = module.Chart{
73 + ID: "requests_latency",
74 + Title: "Requests Latency",
75 + Units: "ms",
76 + Fam: "requests",
77 + Ctx: "zookeeper.requests_latency",
78 + Priority: prioRequestsLatency,
79 + Dims: module.Dims{
80 {ID: "min_latency", Name: "min", Div: 1000},
81 {ID: "avg_latency", Name: "avg", Div: 1000},
82 {ID: "max_latency", Name: "max", Div: 1000},
83 },
35 - },
36 - {
37 - ID: "connections",
38 - Title: "Alive Connections",
39 - Units: "connections",
40 - Fam: "connections",
41 - Ctx: "zookeeper.connections",
42 - Dims: Dims{
84 + }
85 + chartStaleRequests = module.Chart{
86 + ID: "stale_requests",
87 + Title: "Stale Requests",
88 + Units: "requests/s",
89 + Fam: "requests",
90 + Ctx: "zookeeper.stale_requests",
91 + Priority: prioStaleRequests,
92 + Dims: module.Dims{
93 + {ID: "stale_requests", Name: "stale", Algo: module.Incremental},
94 + },
95 + }
96 + chartStaleRequestsDropped = module.Chart{
97 + ID: "stale_requests_dropped",
98 + Title: "Stale Requests Dropped",
99 + Units: "requests/s",
100 + Fam: "requests",
101 + Ctx: "zookeeper.stale_requests_dropped",
102 + Priority: prioStaleRequestsDropped,
103 + Dims: module.Dims{
104 + {ID: "stale_requests_dropped", Name: "dropped", Algo: module.Incremental},
105 + },
106 + }
107 +
108 + chartAliveConnections = module.Chart{
109 + ID: "connections",
110 + Title: "Alive Connections",
111 + Units: "connections",
112 + Fam: "connections",
113 + Ctx: "zookeeper.connections",
114 + Priority: prioAliveConnections,
115 + Dims: module.Dims{
116 {ID: "num_alive_connections", Name: "alive"},
117 },
45 - },
46 - {
47 - ID: "packets",
48 - Title: "Packets",
49 - Units: "pps",
50 - Fam: "net",
51 - Ctx: "zookeeper.packets",
52 - Dims: Dims{
118 + }
119 + chartConnectionsDropped = module.Chart{
120 + ID: "connections_dropped",
121 + Title: "Dropped Connections",
122 + Units: "connections/s",
123 + Fam: "connections",
124 + Ctx: "zookeeper.connections_dropped",
125 + Priority: prioConnectionsDropped,
126 + Dims: module.Dims{
127 + {ID: "connection_drop_count", Name: "dropped", Algo: module.Incremental},
128 + },
129 + }
130 + chartConnectionsRejected = module.Chart{
131 + ID: "connections_rejected",
132 + Title: "Rejected Connections",
133 + Units: "connections/s",
134 + Fam: "connections",
135 + Ctx: "zookeeper.connections_rejected",
136 + Priority: prioConnectionsRejected,
137 + Dims: module.Dims{
138 + {ID: "connection_rejected", Name: "rejected", Algo: module.Incremental},
139 + },
140 + }
141 + chartAuthFailed = module.Chart{
142 + ID: "auth_failed_count",
143 + Title: "Auth Fails",
144 + Units: "fails/s",
145 + Fam: "connections",
146 + Ctx: "zookeeper.auth_fails",
147 + Priority: prioAuthFails,
148 + Dims: module.Dims{
149 + {ID: "auth_failed_count", Name: "auth", Algo: module.Incremental},
150 + },
151 + }
152 + chartGlobalSessions = module.Chart{
153 + ID: "global_sessions",
154 + Title: "Global Sessions",
155 + Units: "sessions",
156 + Fam: "connections",
157 + Ctx: "zookeeper.global_sessions",
158 + Priority: prioGlobalSessions,
159 + Dims: module.Dims{
160 + {ID: "global_sessions", Name: "global"},
161 + },
162 + }
163 +
164 + chartServerState = module.Chart{
165 + ID: "server_state",
166 + Title: "Server State",
167 + Units: "state",
168 + Fam: "server state",
169 + Ctx: "zookeeper.server_state",
170 + Priority: prioServerState,
171 + Dims: module.Dims{
172 + {ID: "server_state_leader", Name: "leader"},
173 + {ID: "server_state_follower", Name: "follower"},
174 + {ID: "server_state_observer", Name: "observer"},
175 + {ID: "server_state_standalone", Name: "standalone"},
176 + },
177 + }
178 + chartThrottledOps = module.Chart{
179 + ID: "throttled_ops",
180 + Title: "Throttled Operations",
181 + Units: "ops/s",
182 + Fam: "server state",
183 + Ctx: "zookeeper.throttled_ops",
184 + Priority: prioThrottledOps,
185 + Dims: module.Dims{
186 + {ID: "throttled_ops", Name: "throttled", Algo: module.Incremental},
187 + },
188 + }
189 +
190 + chartPackets = module.Chart{
191 + ID: "packets",
192 + Title: "Packets",
193 + Units: "pps",
194 + Fam: "net",
195 + Ctx: "zookeeper.packets",
196 + Priority: prioPackets,
197 + Dims: module.Dims{
198 {ID: "packets_received", Name: "received", Algo: module.Incremental},
199 {ID: "packets_sent", Name: "sent", Algo: module.Incremental, Mul: -1},
200 },
56 - },
57 - {
58 - ID: "file_descriptor",
59 - Title: "Open File Descriptors",
60 - Units: "file descriptors",
61 - Fam: "file descriptors",
62 - Ctx: "zookeeper.file_descriptor",
63 - Dims: Dims{
201 + }
202 +
203 + chartFileDescriptors = module.Chart{
204 + ID: "file_descriptor",
205 + Title: "Open File Descriptors",
206 + Units: "file descriptors",
207 + Fam: "file descriptors",
208 + Ctx: "zookeeper.file_descriptor",
209 + Priority: prioFileDescriptors,
210 + Dims: module.Dims{
211 {ID: "open_file_descriptor_count", Name: "open"},
212 },
66 - Vars: Vars{
213 + Vars: module.Vars{
214 {ID: "max_file_descriptor_count"},
215 },
69 - },
70 - {
71 - ID: "nodes",
72 - Title: "Number of Nodes",
73 - Units: "nodes",
74 - Fam: "data tree",
75 - Ctx: "zookeeper.nodes",
76 - Dims: Dims{
216 + }
217 +
218 + chartNodesCount = module.Chart{
219 + ID: "nodes",
220 + Title: "Number of Nodes",
221 + Units: "nodes",
222 + Fam: "data tree",
223 + Ctx: "zookeeper.nodes",
224 + Priority: prioNodesCount,
225 + Dims: module.Dims{
226 {ID: "znode_count", Name: "znode"},
227 {ID: "ephemerals_count", Name: "ephemerals"},
228 },
80 - },
81 - {
82 - ID: "watches",
83 - Title: "Number of Watches",
84 - Units: "watches",
85 - Fam: "data tree",
86 - Ctx: "zookeeper.watches",
87 - Dims: Dims{
229 + }
230 + chartWatchesCount = module.Chart{
231 + ID: "watches",
232 + Title: "Number of Watches",
233 + Units: "watches",
234 + Fam: "data tree",
235 + Ctx: "zookeeper.watches",
236 + Priority: prioWatchesCount,
237 + Dims: module.Dims{
238 {ID: "watch_count", Name: "watches"},
239 },
90 - },
91 - {
92 - ID: "approximate_data_size",
93 - Title: "Approximate Data Tree Size",
94 - Units: "KiB",
95 - Fam: "data tree",
96 - Ctx: "zookeeper.approximate_data_size",
97 - Dims: Dims{
240 + }
241 + chartApproxDataSize = module.Chart{
242 + ID: "approximate_data_size",
243 + Title: "Approximate Data Tree Size",
244 + Units: "KiB",
245 + Fam: "data tree",
246 + Ctx: "zookeeper.approximate_data_size",
247 + Priority: prioApproxDataSize,
248 + Dims: module.Dims{
249 {ID: "approximate_data_size", Name: "size", Div: 1024},
250 },
100 - },
101 - {
102 - ID: "server_state",
103 - Title: "Server State",
104 - Units: "state",
105 - Fam: "server state",
106 - Ctx: "zookeeper.server_state",
107 - Dims: Dims{
108 - {ID: "server_state", Name: "state"},
109 - },
110 - },
111 -}
251 + }
252 +
253 + chartUptime = module.Chart{
254 + ID: "uptime",
255 + Title: "Uptime",
256 + Units: "seconds",
257 + Fam: "uptime",
258 + Ctx: "zookeeper.uptime",
259 + Priority: prioUptime,
260 + Dims: module.Dims{
261 + {ID: "uptime"},
262 + },
263 + }
264 +)
src/go/plugin/go.d/collector/zookeeper/collect.go
+64 -25
@@ -6,6 +6,8 @@ import (
6 "fmt"
7 "strconv"
8 "strings"
9 +
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/metrix"
11 )
12
13 func (c *Collector) collect() (map[string]int64, error) {
@@ -36,23 +38,23 @@ func (c *Collector) collectMntr() (map[string]int64, error) {
38 continue
39 }
40
39 - key, value := strings.TrimPrefix(parts[0], "zk_"), parts[1]
41 + key, value := parts[0], parts[1]
42 + if !zkMetrics[key] {
43 + continue
44 + }
45 + key = strings.TrimPrefix(key, "zk_")
46 +
47 switch key {
41 - case "version":
48 case "server_state":
43 - mx[key] = convertServerState(value)
44 - case "min_latency", "avg_latency", "max_latency":
45 - v, err := strconv.ParseFloat(value, 64)
46 - if err != nil {
47 - continue
49 + for _, v := range zkServerStates {
50 + mx[key+"_"+v] = metrix.Bool(v == value)
51 }
49 - mx[key] = int64(v * 1000)
52 + case "min_latency", "avg_latency", "max_latency":
53 + writeMetric(mx, key, value, 1000, 1)
54 + case "uptime":
55 + writeMetric(mx, key, value, 1, 1000) // ms->seconds
56 default:
51 - v, err := strconv.ParseFloat(value, 64)
52 - if err != nil {
53 - continue
54 - }
55 - mx[key] = int64(v)
57 + writeMetric(mx, key, value, 1, 1)
58 }
59 }
60
@@ -63,17 +65,54 @@ func (c *Collector) collectMntr() (map[string]int64, error) {
65 return mx, nil
66 }
67
66 -func convertServerState(state string) int64 {
67 - switch state {
68 - default:
69 - return 0
70 - case "leader":
71 - return 1
72 - case "follower":
73 - return 2
74 - case "observer":
75 - return 3
76 - case "standalone":
77 - return 4
68 +func writeMetric(mx map[string]int64, metric, value string, mul, div float64) {
69 + v, err := strconv.ParseFloat(value, 64)
70 + if err != nil {
71 + return
72 + }
73 + if mul != 0 {
74 + v *= mul
75 + }
76 + if div != 0 {
77 + v /= div
78 }
79 + mx[metric] = int64(v)
80 +
81 +}
82 +
83 +var zkServerStates = []string{
84 + "leader",
85 + "follower",
86 + "observer",
87 + "standalone",
88 +}
89 +
90 +var zkMetrics = map[string]bool{
91 + "zk_server_state": true,
92 +
93 + "zk_outstanding_requests": true,
94 + "zk_min_latency": true,
95 + "zk_avg_latency": true,
96 + "zk_max_latency": true,
97 + "zk_stale_requests": true,
98 + "zk_stale_requests_dropped": true,
99 +
100 + "zk_num_alive_connections": true,
101 + "zk_auth_failed_count": true,
102 + "zk_connection_drop_count": true,
103 + "zk_connection_rejected": true,
104 + "zk_global_sessions": true,
105 +
106 + "zk_packets_received": true,
107 + "zk_packets_sent": true,
108 +
109 + "zk_open_file_descriptor_count": true,
110 + "zk_max_file_descriptor_count": true,
111 + "zk_znode_count": true,
112 + "zk_ephemerals_count": true,
113 + "zk_watch_count": true,
114 + "zk_approximate_data_size": true,
115 +
116 + "zk_uptime": true,
117 + "zk_throttled_ops": true,
118 }
src/go/plugin/go.d/collector/zookeeper/collector.go
+1 -1
@@ -79,7 +79,7 @@ func (c *Collector) Check(context.Context) error {
79 return nil
80 }
81
82 -func (c *Collector) Charts() *Charts {
82 +func (c *Collector) Charts() *module.Charts {
83 return charts.Copy()
84 }
85
src/go/plugin/go.d/collector/zookeeper/collector_test.go
+174 -86
@@ -10,26 +10,26 @@ import (
10 "os"
11 "testing"
12
13 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
14 -
13 "github.com/stretchr/testify/assert"
14 "github.com/stretchr/testify/require"
15 +
16 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
17 )
18
19 var (
20 dataConfigJSON, _ = os.ReadFile("testdata/config.json")
21 dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")
22
23 - dataMntrMetrics, _ = os.ReadFile("testdata/mntr.txt")
24 - dataMntrNotInWhiteListResponse, _ = os.ReadFile("testdata/mntr_notinwhitelist.txt")
23 + dataVer39Mntr, _ = os.ReadFile("testdata/v3.9/mntr.txt")
24 + dataMntrNotInWhiteList, _ = os.ReadFile("testdata/mntr_notinwhitelist.txt")
25 )
26
27 func Test_testDataIsValid(t *testing.T) {
28 for name, data := range map[string][]byte{
29 "dataConfigJSON": dataConfigJSON,
30 "dataConfigYAML": dataConfigYAML,
31 - "dataMntrMetrics": dataMntrMetrics,
32 - "dataMntrNotInWhiteListResponse": dataMntrNotInWhiteListResponse,
31 + "dataVer39Mntr": dataVer39Mntr,
32 + "dataMntrNotInWhiteListResponse": dataMntrNotInWhiteList,
33 } {
34 assert.NotNil(t, data, name)
35 }
@@ -40,120 +40,208 @@ func TestCollector_ConfigurationSerialize(t *testing.T) {
40 }
41
42 func TestCollector_Init(t *testing.T) {
43 - collr := New()
43 + tests := map[string]struct {
44 + config Config
45 + wantFail bool
46 + }{
47 + "success on default config": {
48 + config: New().Config,
49 + },
50 + "fails on unset 'address'": {
51 + wantFail: true,
52 + config: Config{
53 + Address: "",
54 + },
55 + },
56 + }
57
45 - assert.NoError(t, collr.Init(context.Background()))
46 - assert.NotNil(t, collr.fetcher)
58 + for name, test := range tests {
59 + t.Run(name, func(t *testing.T) {
60 + collr := New()
61 + collr.Config = test.config
62 +
63 + if test.wantFail {
64 + assert.Error(t, collr.Init(context.Background()))
65 + } else {
66 + assert.NoError(t, collr.Init(context.Background()))
67 + }
68 + })
69 + }
70 }
71
49 -func TestCollector_InitErrorOnCreatingTLSConfig(t *testing.T) {
50 - collr := New()
51 - collr.UseTLS = true
52 - collr.TLSConfig.TLSCA = "testdata/tls"
53 -
54 - assert.Error(t, collr.Init(context.Background()))
72 +func TestCollector_Charts(t *testing.T) {
73 + assert.NotNil(t, New().Charts())
74 }
75
57 -func TestCollector_Check(t *testing.T) {
58 - collr := New()
59 - require.NoError(t, collr.Init(context.Background()))
60 - collr.fetcher = &mockZookeeperFetcher{data: dataMntrMetrics}
61 -
62 - assert.NoError(t, collr.Check(context.Background()))
63 -}
76 +func TestCollector_Cleanup(t *testing.T) {
77 + tests := map[string]struct {
78 + prepare func(t *testing.T) *Collector
79 + }{
80 + "not initialized": {
81 + prepare: func(t *testing.T) *Collector {
82 + return New()
83 + },
84 + },
85 + "initialized": {
86 + prepare: func(t *testing.T) *Collector {
87 + collr := New()
88 + require.NoError(t, collr.Init(context.Background()))
89 + return collr
90 + },
91 + },
92 + }
93
65 -func TestCollector_CheckErrorOnFetch(t *testing.T) {
66 - collr := New()
67 - require.NoError(t, collr.Init(context.Background()))
68 - collr.fetcher = &mockZookeeperFetcher{err: true}
94 + for name, test := range tests {
95 + t.Run(name, func(t *testing.T) {
96 + collr := test.prepare(t)
97
70 - assert.Error(t, collr.Check(context.Background()))
98 + assert.NotPanics(t, func() { collr.Cleanup(context.Background()) })
99 + })
100 + }
101 }
102
73 -func TestCollector_Charts(t *testing.T) {
74 - assert.NotNil(t, New().Charts())
75 -}
103 +func TestCollector_Check(t *testing.T) {
104 + tests := map[string]struct {
105 + prepare func() *mockZookeeperFetcher
106 + wantFail bool
107 + }{
108 + "success v3.9": {
109 + wantFail: false,
110 + prepare: prepareMockVer39Ok,
111 + },
112 + "fails if mntr is not whitelisted": {
113 + wantFail: true,
114 + prepare: prepareMockMntrNotInWhitelist,
115 + },
116 + "fails on empty response": {
117 + wantFail: true,
118 + prepare: prepareMockMntrEmptyResponse,
119 + },
120 + "fails on fetch error": {
121 + wantFail: true,
122 + prepare: prepareMockErrOnFetch,
123 + },
124 + }
125
77 -func TestCollector_Cleanup(t *testing.T) {
78 - New().Cleanup(context.Background())
126 + for name, test := range tests {
127 + t.Run(name, func(t *testing.T) {
128 + collr := New()
129 + require.NoError(t, collr.Init(context.Background()))
130 + collr.fetcher = test.prepare()
131 +
132 + if test.wantFail {
133 + assert.Error(t, collr.Check(context.Background()))
134 + } else {
135 + assert.NoError(t, collr.Check(context.Background()))
136 + }
137 + })
138 + }
139 }
140
141 func TestCollector_Collect(t *testing.T) {
82 - collr := New()
83 - require.NoError(t, collr.Init(context.Background()))
84 - collr.fetcher = &mockZookeeperFetcher{data: dataMntrMetrics}
85 -
86 - expected := map[string]int64{
87 - "approximate_data_size": 44,
88 - "avg_latency": 100,
89 - "ephemerals_count": 0,
90 - "max_file_descriptor_count": 1048576,
91 - "max_latency": 100,
92 - "min_latency": 100,
93 - "num_alive_connections": 1,
94 - "open_file_descriptor_count": 63,
95 - "outstanding_requests": 0,
96 - "packets_received": 92,
97 - "packets_sent": 182,
98 - "server_state": 4,
99 - "watch_count": 0,
100 - "znode_count": 5,
142 + tests := map[string]struct {
143 + prepare func() *mockZookeeperFetcher
144 + wantMetrics map[string]int64
145 + }{
146 + "success v3.9": {
147 + prepare: prepareMockVer39Ok,
148 + wantMetrics: map[string]int64{
149 + "approximate_data_size": 44,
150 + "auth_failed_count": 0,
151 + "avg_latency": 0,
152 + "connection_drop_count": 0,
153 + "connection_rejected": 0,
154 + "ephemerals_count": 0,
155 + "global_sessions": 0,
156 + "max_file_descriptor_count": 1048576,
157 + "max_latency": 0,
158 + "min_latency": 0,
159 + "num_alive_connections": 1,
160 + "open_file_descriptor_count": 77,
161 + "outstanding_requests": 0,
162 + "packets_received": 452,
163 + "packets_sent": 1353,
164 + "server_state_follower": 0,
165 + "server_state_leader": 0,
166 + "server_state_observer": 0,
167 + "server_state_standalone": 1,
168 + "stale_requests": 0,
169 + "stale_requests_dropped": 0,
170 + "throttled_ops": 0,
171 + "uptime": 488,
172 + "watch_count": 0,
173 + "znode_count": 5,
174 + },
175 + },
176 + "fails if mntr is not whitelisted": {
177 + prepare: prepareMockMntrNotInWhitelist,
178 + },
179 + "fails on empty response": {
180 + prepare: prepareMockMntrEmptyResponse,
181 + },
182 + "fails on fetch error": {
183 + prepare: prepareMockErrOnFetch,
184 + },
185 }
186
103 - mx := collr.Collect(context.Background())
187 + for name, test := range tests {
188 + t.Run(name, func(t *testing.T) {
189 + collr := New()
190 + require.NoError(t, collr.Init(context.Background()))
191 + collr.fetcher = test.prepare()
192
105 - assert.Equal(t, expected, mx)
106 - module.TestMetricsHasAllChartsDims(t, collr.Charts(), mx)
107 -}
193 + mx := collr.Collect(context.Background())
194
109 -func TestCollector_CollectMntrNotInWhiteList(t *testing.T) {
110 - collr := New()
111 - require.NoError(t, collr.Init(context.Background()))
112 - collr.fetcher = &mockZookeeperFetcher{data: dataMntrNotInWhiteListResponse}
113 -
114 - assert.Nil(t, collr.Collect(context.Background()))
195 + assert.Equal(t, test.wantMetrics, mx)
196 + if len(mx) > 0 {
197 + module.TestMetricsHasAllChartsDims(t, collr.Charts(), mx)
198 + }
199 + })
200 + }
201 }
202
117 -func TestCollector_CollectMntrEmptyResponse(t *testing.T) {
118 - collr := New()
119 - require.NoError(t, collr.Init(context.Background()))
120 - collr.fetcher = &mockZookeeperFetcher{}
121 -
122 - assert.Nil(t, collr.Collect(context.Background()))
203 +func prepareMockVer39Ok() *mockZookeeperFetcher {
204 + return &mockZookeeperFetcher{
205 + dataMntr: dataVer39Mntr,
206 + }
207 }
208
125 -func TestCollector_CollectMntrInvalidData(t *testing.T) {
126 - collr := New()
127 - require.NoError(t, collr.Init(context.Background()))
128 - collr.fetcher = &mockZookeeperFetcher{data: []byte("hello \nand good buy\n")}
129 -
130 - assert.Nil(t, collr.Collect(context.Background()))
209 +func prepareMockMntrNotInWhitelist() *mockZookeeperFetcher {
210 + return &mockZookeeperFetcher{
211 + dataMntr: dataMntrNotInWhiteList,
212 + }
213 }
214
133 -func TestCollector_CollectMntrReceiveError(t *testing.T) {
134 - collr := New()
135 - require.NoError(t, collr.Init(context.Background()))
136 - collr.fetcher = &mockZookeeperFetcher{err: true}
215 +func prepareMockMntrEmptyResponse() *mockZookeeperFetcher {
216 + return &mockZookeeperFetcher{}
217 +}
218
138 - assert.Nil(t, collr.Collect(context.Background()))
219 +func prepareMockErrOnFetch() *mockZookeeperFetcher {
220 + return &mockZookeeperFetcher{
221 + errOnFetch: true,
222 + }
223 }
224
225 type mockZookeeperFetcher struct {
142 - data []byte
143 - err bool
226 + dataMntr []byte
227 + errOnFetch bool
228 }
229
146 -func (m mockZookeeperFetcher) fetch(_ string) ([]string, error) {
147 - if m.err {
230 +func (m mockZookeeperFetcher) fetch(cmd string) ([]string, error) {
231 + if m.errOnFetch {
232 return nil, errors.New("mock fetch error")
233 }
234
235 + if cmd != "mntr" {
236 + return nil, errors.New("invalid command")
237 + }
238 +
239 var lines []string
152 - s := bufio.NewScanner(bytes.NewReader(m.data))
240 +
241 + s := bufio.NewScanner(bytes.NewReader(m.dataMntr))
242 for s.Scan() {
154 - if !isZKLine(s.Bytes()) || isMntrLineOK(s.Bytes()) {
155 - lines = append(lines, s.Text())
156 - }
243 + lines = append(lines, s.Text())
244 }
245 +
246 return lines, nil
247 }
src/go/plugin/go.d/collector/zookeeper/fetcher.go
+1 -36
@@ -3,9 +3,6 @@
3 package zookeeper
4
5 import (
6 - "bytes"
7 - "unsafe"
8 -
6 "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/socket"
7 )
8
@@ -24,9 +21,7 @@ func (c *zookeeperFetcher) fetch(command string) (rows []string, err error) {
21 defer func() { _ = c.Disconnect() }()
22
23 if err := c.Command(command, func(b []byte) (bool, error) {
27 - if !isZKLine(b) || isMntrLineOK(b) {
28 - rows = append(rows, string(b))
29 - }
24 + rows = append(rows, string(b))
25 return true, nil
26 }); err != nil {
27 return nil, err
@@ -34,33 +29,3 @@ func (c *zookeeperFetcher) fetch(command string) (rows []string, err error) {
29
30 return rows, nil
31 }
37 -
38 -func isZKLine(line []byte) bool {
39 - return bytes.HasPrefix(line, []byte("zk_"))
40 -}
41 -
42 -func isMntrLineOK(line []byte) bool {
43 - idx := bytes.LastIndexByte(line, '\t')
44 - return idx > 0 && collectedZKKeys[unsafeString(line)[:idx]]
45 -}
46 -
47 -func unsafeString(b []byte) string {
48 - return *((*string)(unsafe.Pointer(&b)))
49 -}
50 -
51 -var collectedZKKeys = map[string]bool{
52 - "zk_num_alive_connections": true,
53 - "zk_outstanding_requests": true,
54 - "zk_min_latency": true,
55 - "zk_avg_latency": true,
56 - "zk_max_latency": true,
57 - "zk_packets_received": true,
58 - "zk_packets_sent": true,
59 - "zk_open_file_descriptor_count": true,
60 - "zk_max_file_descriptor_count": true,
61 - "zk_znode_count": true,
62 - "zk_ephemerals_count": true,
63 - "zk_watch_count": true,
64 - "zk_approximate_data_size": true,
65 - "zk_server_state": true,
66 -}
src/go/plugin/go.d/collector/zookeeper/metadata.yaml
+57 -6
@@ -138,8 +138,8 @@ modules:
138 description: ""
139 availability: []
140 scopes:
141 - - name: global
142 - description: These metrics refer to the entire monitored application.
141 + - name: zookeeper server
142 + description: These metrics refer to ZooKeeper servers.
143 labels: []
144 metrics:
145 - name: zookeeper.requests
@@ -156,12 +156,63 @@ modules:
156 - name: min
157 - name: avg
158 - name: max
159 + - name: zookeeper.stale_requests
160 + description: Stale Requests
161 + unit: requests/s
162 + chart_type: line
163 + dimensions:
164 + - name: stale
165 + - name: zookeeper.stale_requests_dropped
166 + description: Stale Requests Dropped
167 + unit: requests/s
168 + chart_type: line
169 + dimensions:
170 + - name: dropped
171 - name: zookeeper.connections
172 description: Alive Connections
173 unit: connections
174 chart_type: line
175 dimensions:
176 - name: alive
177 + - name: zookeeper.connections_dropped
178 + description: Dropped Connections
179 + unit: connections/s
180 + chart_type: line
181 + dimensions:
182 + - name: dropped
183 + - name: zookeeper.connections_rejected
184 + description: Rejected Connections
185 + unit: connections/s
186 + chart_type: line
187 + dimensions:
188 + - name: rejected
189 + - name: zookeeper.auth_fails
190 + description: Auth Fails
191 + unit: fails/s
192 + chart_type: line
193 + dimensions:
194 + - name: auth
195 + - name: zookeeper.global_sessions
196 + description: Global Sessions
197 + unit: sessions
198 + chart_type: line
199 + dimensions:
200 + - name: global
201 + - name: zookeeper.server_state
202 + description: Server State
203 + unit: state
204 + chart_type: line
205 + dimensions:
206 + - name: leader
207 + - name: follower
208 + - name: observer
209 + - name: standalone
210 + - name: zookeeper.throttled_ops
211 + description: Throttled Operations
212 + unit: ops/s
213 + chart_type: line
214 + dimensions:
215 + - name: throttled
216 - name: zookeeper.packets
217 description: Packets
218 unit: pps
@@ -194,9 +245,9 @@ modules:
245 chart_type: line
246 dimensions:
247 - name: size
197 - - name: zookeeper.server_state
198 - description: Server State
199 - unit: state
248 + - name: zookeeper.uptime
249 + description: Uptime
250 + unit: seconds
251 chart_type: line
252 dimensions:
202 - - name: state
253 + - name: uptime
src/go/plugin/go.d/collector/zookeeper/testdata/v3.9/mntr.txt renamed
+119 -41
@@ -1,65 +1,95 @@
1 -zk_version 3.6.1--104dcb3e3fb464b30c5186d229e00af9f332524b, built on 04/21/2020 15:01 GMT
1 +zk_version 3.9.3-c26634f34490bb0ea7a09cc51e05ede3b4e320ee, built on 2024-10-17 23:21 UTC
2 zk_server_state standalone
3 zk_ephemerals_count 0
4 -zk_min_latency 0.1
5 -zk_avg_latency 0.1
4 zk_num_alive_connections 1
7 -zk_max_file_descriptor_count 1048576
5 +zk_avg_latency 0.0
6 zk_outstanding_requests 0
9 -zk_approximate_data_size 44
7 zk_znode_count 5
11 -zk_open_file_descriptor_count 63
8 zk_global_sessions 0
13 -zk_local_sessions 0
14 -zk_uptime 27595191
9 +zk_non_mtls_remote_conn_count 0
10 zk_last_client_response_size -1
16 -zk_max_latency 0.1
17 -zk_packets_sent 182
18 -zk_outstanding_tls_handshake 0
19 -zk_packets_received 92
11 +zk_packets_sent 1353
12 +zk_packets_received 452
13 zk_max_client_response_size -1
14 zk_connection_drop_probability 0.0
15 zk_watch_count 0
16 +zk_auth_failed_count 0
17 +zk_min_latency 0
18 +zk_max_file_descriptor_count 1048576
19 +zk_approximate_data_size 44
20 +zk_open_file_descriptor_count 77
21 +zk_local_sessions 0
22 +zk_uptime 488178
23 +zk_max_latency 0
24 +zk_outstanding_tls_handshake 0
25 zk_min_client_response_size -1
24 -zk_proposal_count 0
25 -zk_outstanding_changes_removed 0
26 +zk_non_mtls_local_conn_count 0
27 +zk_watch_bytes 0
28 zk_stale_requests_dropped 0
27 -zk_large_requests_rejected 0
29 +zk_throttled_ops 0
30 +zk_insecure_admin_count 0
31 zk_connection_rejected 0
32 zk_sessionless_connections_expired 0
30 -zk_looking_count 0
33 zk_dead_watchers_queued 0
34 zk_stale_requests 0
35 zk_connection_drop_count 0
34 -zk_learner_proposal_received_count 0
35 -zk_digest_mismatches_count 0
36 -zk_dead_watchers_cleared 0
36 zk_response_packet_cache_hits 0
38 -zk_bytes_received_count 368
37 +zk_bytes_received_count 1808
38 zk_add_dead_watcher_stall_time 0
39 zk_request_throttle_wait_count 0
40 +zk_requests_not_forwarded_to_commit_processor 0
41 zk_response_packet_cache_misses 0
42 -zk_ensemble_auth_success 0
42 zk_prep_processor_request_queued 0
44 -zk_learner_commit_received_count 0
43 zk_stale_replies 0
46 -zk_connection_request_count 0
44 +zk_response_bytes 0
45 zk_ensemble_auth_fail 0
46 zk_diff_count 0
49 -zk_response_packet_get_children_cache_misses 0
47 zk_connection_revalidate_count 0
48 zk_quit_leading_due_to_disloyal_voter 0
52 -zk_snap_count 0
49 zk_unrecoverable_error_count 0
50 +zk_unsuccessful_handshake 0
51 zk_commit_count 0
55 -zk_stale_sessions_expired 0
56 -zk_response_packet_get_children_cache_hits 0
57 -zk_sync_processor_request_queued 0
52 zk_outstanding_changes_queued 0
53 zk_request_commit_queued 0
54 zk_ensemble_auth_skip 0
55 +zk_skip_learner_request_to_next_processor_count 0
56 +zk_proposal_count 0
57 +zk_large_requests_rejected 0
58 +zk_outstanding_changes_removed 0
59 +zk_restore_error_count 0
60 +zk_cnxn_closed_without_zk_server_running 0
61 +zk_looking_count 0
62 +zk_snapshot_rate_limited_count 0
63 +zk_learner_proposal_received_count 0
64 +zk_digest_mismatches_count 0
65 +zk_dead_watchers_cleared 0
66 +zk_ensemble_auth_success 0
67 +zk_learner_commit_received_count 0
68 +zk_snapshot_error_count 0
69 +zk_connection_request_count 0
70 +zk_response_packet_get_children_cache_misses 0
71 +zk_snap_count 0
72 +zk_stale_sessions_expired 0
73 +zk_restore_rate_limited_count 0
74 +zk_response_packet_get_children_cache_hits 0
75 +zk_sync_processor_request_queued 0
76 zk_tls_handshake_exceeded 0
77 zk_revalidate_count 0
78 +zk_avg_socket_closing_time 0.0
79 +zk_min_socket_closing_time 0
80 +zk_max_socket_closing_time 0
81 +zk_cnt_socket_closing_time 0
82 +zk_sum_socket_closing_time 0
83 +zk_avg_proposal_process_time 0.0
84 +zk_min_proposal_process_time 0
85 +zk_max_proposal_process_time 0
86 +zk_cnt_proposal_process_time 0
87 +zk_sum_proposal_process_time 0
88 +zk_avg_leader_unavailable_time 0.0
89 +zk_min_leader_unavailable_time 0
90 +zk_max_leader_unavailable_time 0
91 +zk_cnt_leader_unavailable_time 0
92 +zk_sum_leader_unavailable_time 0
93 zk_avg_node_created_watch_count 0.0
94 zk_min_node_created_watch_count 0
95 zk_max_node_created_watch_count 0
@@ -115,16 +145,36 @@ zk_min_fsynctime 0
145 zk_max_fsynctime 0
146 zk_cnt_fsynctime 0
147 zk_sum_fsynctime 0
148 +zk_avg_inflight_snap_count 0.0
149 +zk_min_inflight_snap_count 0
150 +zk_max_inflight_snap_count 0
151 +zk_cnt_inflight_snap_count 0
152 +zk_sum_inflight_snap_count 0
153 zk_avg_reads_issued_from_session_queue 0.0
154 zk_min_reads_issued_from_session_queue 0
155 zk_max_reads_issued_from_session_queue 0
156 zk_cnt_reads_issued_from_session_queue 0
157 zk_sum_reads_issued_from_session_queue 0
123 -zk_avg_snapshottime 0.0
124 -zk_min_snapshottime 0
125 -zk_max_snapshottime 0
158 +zk_avg_restore_time 0.0
159 +zk_min_restore_time 0
160 +zk_max_restore_time 0
161 +zk_cnt_restore_time 0
162 +zk_sum_restore_time 0
163 +zk_avg_learner_request_processor_queue_size 0.0
164 +zk_min_learner_request_processor_queue_size 0
165 +zk_max_learner_request_processor_queue_size 0
166 +zk_cnt_learner_request_processor_queue_size 0
167 +zk_sum_learner_request_processor_queue_size 0
168 +zk_avg_snapshottime 1.0
169 +zk_min_snapshottime 1
170 +zk_max_snapshottime 1
171 zk_cnt_snapshottime 1
127 -zk_sum_snapshottime 0
172 +zk_sum_snapshottime 1
173 +zk_avg_unavailable_time 0.0
174 +zk_min_unavailable_time 0
175 +zk_max_unavailable_time 0
176 +zk_cnt_unavailable_time 0
177 +zk_sum_unavailable_time 0
178 zk_avg_startup_txns_loaded 0.0
179 zk_min_startup_txns_loaded 0
180 zk_max_startup_txns_loaded 0
@@ -165,11 +215,16 @@ zk_min_commit_process_time 0
215 zk_max_commit_process_time 0
216 zk_cnt_commit_process_time 0
217 zk_sum_commit_process_time 0
168 -zk_avg_dbinittime 6.0
169 -zk_min_dbinittime 6
170 -zk_max_dbinittime 6
218 +zk_avg_dbinittime 11.0
219 +zk_min_dbinittime 11
220 +zk_max_dbinittime 11
221 zk_cnt_dbinittime 1
172 -zk_sum_dbinittime 6
222 +zk_sum_dbinittime 11
223 +zk_avg_inflight_diff_count 0.0
224 +zk_min_inflight_diff_count 0
225 +zk_max_inflight_diff_count 0
226 +zk_cnt_inflight_diff_count 0
227 +zk_sum_inflight_diff_count 0
228 zk_avg_netty_queued_buffer_capacity 0.0
229 zk_min_netty_queued_buffer_capacity 0
230 zk_max_netty_queued_buffer_capacity 0
@@ -210,6 +265,11 @@ zk_min_concurrent_request_processing_in_commit_processor 0
265 zk_max_concurrent_request_processing_in_commit_processor 0
266 zk_cnt_concurrent_request_processing_in_commit_processor 0
267 zk_sum_concurrent_request_processing_in_commit_processor 0
268 +zk_avg_observer_sync_time 0.0
269 +zk_min_observer_sync_time 0
270 +zk_max_observer_sync_time 0
271 +zk_cnt_observer_sync_time 0
272 +zk_sum_observer_sync_time 0
273 zk_avg_node_changed_watch_count 0.0
274 zk_min_node_changed_watch_count 0
275 zk_max_node_changed_watch_count 0
@@ -220,11 +280,11 @@ zk_min_sync_process_time 0
280 zk_max_sync_process_time 0
281 zk_cnt_sync_process_time 0
282 zk_sum_sync_process_time 0
223 -zk_avg_startup_snap_load_time 5.0
224 -zk_min_startup_snap_load_time 5
225 -zk_max_startup_snap_load_time 5
283 +zk_avg_startup_snap_load_time 1.0
284 +zk_min_startup_snap_load_time 1
285 +zk_max_startup_snap_load_time 1
286 zk_cnt_startup_snap_load_time 1
227 -zk_sum_startup_snap_load_time 5
287 +zk_sum_startup_snap_load_time 1
288 zk_avg_prep_processor_queue_time_ms 0.0
289 zk_min_prep_processor_queue_time_ms 0
290 zk_max_prep_processor_queue_time_ms 0
@@ -234,6 +294,15 @@ zk_p50_prep_processor_queue_time_ms 0
294 zk_p95_prep_processor_queue_time_ms 0
295 zk_p99_prep_processor_queue_time_ms 0
296 zk_p999_prep_processor_queue_time_ms 0
297 +zk_avg_jvm_pause_time_ms 0.0
298 +zk_min_jvm_pause_time_ms 0
299 +zk_max_jvm_pause_time_ms 0
300 +zk_cnt_jvm_pause_time_ms 0
301 +zk_sum_jvm_pause_time_ms 0
302 +zk_p50_jvm_pause_time_ms 0
303 +zk_p95_jvm_pause_time_ms 0
304 +zk_p99_jvm_pause_time_ms 0
305 +zk_p999_jvm_pause_time_ms 0
306 zk_avg_close_session_prep_time 0.0
307 zk_min_close_session_prep_time 0
308 zk_max_close_session_prep_time 0
@@ -270,6 +339,15 @@ zk_p50_local_write_committed_time_ms 0
339 zk_p95_local_write_committed_time_ms 0
340 zk_p99_local_write_committed_time_ms 0
341 zk_p999_local_write_committed_time_ms 0
342 +zk_avg_request_throttle_queue_time_ms 0.0
343 +zk_min_request_throttle_queue_time_ms 0
344 +zk_max_request_throttle_queue_time_ms 0
345 +zk_cnt_request_throttle_queue_time_ms 0
346 +zk_sum_request_throttle_queue_time_ms 0
347 +zk_p50_request_throttle_queue_time_ms 0
348 +zk_p95_request_throttle_queue_time_ms 0
349 +zk_p99_request_throttle_queue_time_ms 0
350 +zk_p999_request_throttle_queue_time_ms 0
351 zk_avg_readlatency 0.0
352 zk_min_readlatency 0
353 zk_max_readlatency 0
@@ -413,4 +491,4 @@ zk_sum_write_commitproc_time_ms 0
491 zk_p50_write_commitproc_time_ms 0
492 zk_p95_write_commitproc_time_ms 0
493 zk_p99_write_commitproc_time_ms 0
416 -zk_p999_write_commitproc_time_ms 0
\ No newline at end of file
494 +zk_p999_write_commitproc_time_ms 0