@cryptotaxi247 / netdata-1 / commits / b0c21cb08

improvement(go.d/nats): add accstatz metrics (#19262)

Ilya Mashchenko committed Dec 21, 2024 at 21:12 UTC b0c21cb080ff60530be9416d6d1a46417f832dd5
12 files changed +596 -174
src/go/plugin/go.d/collector/nats/charts.go
+162 -19
@@ -4,6 +4,7 @@ package nats
4
5 import (
6 "fmt"
7 + "strings"
8
9 "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
10 )
@@ -11,13 +12,21 @@ import (
12 const (
13 prioServerTraffic = module.Priority + iota
14 prioServerMessages
14 - prioServerConnectionsCurrent
15 + prioServerConnections
16 prioServerConnectionsRate
17 prioHttpEndpointRequests
18 prioServerHealthProbeStatus
19 prioServerCpuUsage
20 prioServerMemoryUsage
21 prioServerUptime
22 +
23 + prioAccountTraffic
24 + prioAccountMessages
25 + prioAccountConnections
26 + prioAccountConnectionsRate
27 + prioAccountSubscriptions
28 + prioAccountSlowConsumers
29 + prioAccountLeafNodes
30 )
31
32 var serverCharts = func() module.Charts {
@@ -31,7 +40,7 @@ var serverCharts = func() module.Charts {
40 chartServerMemUsage.Copy(),
41 chartServerUptime.Copy(),
42 }
34 - charts = append(charts, httpEndpointCharts()...)
43 + charts = append(charts, httpEndpointsCharts()...)
44 return charts
45 }()
46
@@ -45,8 +54,8 @@ var (
54 Priority: prioServerTraffic,
55 Type: module.Area,
56 Dims: module.Dims{
48 - {ID: "in_bytes", Name: "in", Algo: module.Incremental},
49 - {ID: "out_bytes", Name: "out", Mul: -1, Algo: module.Incremental},
57 + {ID: "varz_srv_in_bytes", Name: "received", Algo: module.Incremental},
58 + {ID: "varz_srv_out_bytes", Name: "sent", Mul: -1, Algo: module.Incremental},
59 },
60 }
61 chartServerMessages = module.Chart{
@@ -57,19 +66,19 @@ var (
66 Ctx: "nats.server_messages",
67 Priority: prioServerMessages,
68 Dims: module.Dims{
60 - {ID: "in_msgs", Name: "in", Algo: module.Incremental},
61 - {ID: "out_msgs", Name: "out", Mul: -1, Algo: module.Incremental},
69 + {ID: "varz_srv_in_msgs", Name: "received", Algo: module.Incremental},
70 + {ID: "varz_srv_out_msgs", Name: "sent", Mul: -1, Algo: module.Incremental},
71 },
72 }
73 chartServerConnectionsCurrent = module.Chart{
65 - ID: "server_connections_current",
66 - Title: "Server Current Connections",
74 + ID: "server_connections",
75 + Title: "Server Active Connections",
76 Units: "connections",
77 Fam: "connections",
69 - Ctx: "nats.server_connections_current",
70 - Priority: prioServerConnectionsCurrent,
78 + Ctx: "nats.server_connections",
79 + Priority: prioServerConnections,
80 Dims: module.Dims{
72 - {ID: "connections", Name: "active"},
81 + {ID: "varz_srv_connections", Name: "active"},
82 },
83 }
84 chartServerConnectionsRate = module.Chart{
@@ -80,7 +89,7 @@ var (
89 Ctx: "nats.server_connections_rate",
90 Priority: prioServerConnectionsRate,
91 Dims: module.Dims{
83 - {ID: "total_connections", Name: "connections", Algo: module.Incremental},
92 + {ID: "varz_srv_total_connections", Name: "connections", Algo: module.Incremental},
93 },
94 }
95 chartServerHealthProbeStatus = module.Chart{
@@ -91,8 +100,8 @@ var (
100 Ctx: "nats.server_health_probe_status",
101 Priority: prioServerHealthProbeStatus,
102 Dims: module.Dims{
94 - {ID: "healthz_status_ok", Name: "ok"},
95 - {ID: "healthz_status_error", Name: "error"},
103 + {ID: "varz_srv_healthz_status_ok", Name: "ok"},
104 + {ID: "varz_srv_healthz_status_error", Name: "error"},
105 },
106 }
107 chartServerCpuUsage = module.Chart{
@@ -104,7 +113,7 @@ var (
113 Priority: prioServerCpuUsage,
114 Type: module.Area,
115 Dims: module.Dims{
107 - {ID: "cpu", Name: "used"},
116 + {ID: "varz_srv_cpu", Name: "used"},
117 },
118 }
119 chartServerMemUsage = module.Chart{
@@ -116,7 +125,7 @@ var (
125 Priority: prioServerMemoryUsage,
126 Type: module.Area,
127 Dims: module.Dims{
119 - {ID: "mem", Name: "used"},
128 + {ID: "varz_srv_mem", Name: "used"},
129 },
130 }
131 chartServerUptime = module.Chart{
@@ -127,15 +136,17 @@ var (
136 Ctx: "nats.server_uptime",
137 Priority: prioServerUptime,
138 Dims: module.Dims{
130 - {ID: "uptime", Name: "uptime"},
139 + {ID: "varz_srv_uptime", Name: "uptime"},
140 },
141 }
142 )
143
135 -func httpEndpointCharts() module.Charts {
144 +func httpEndpointsCharts() module.Charts {
145 var charts module.Charts
146 +
147 for _, path := range httpEndpoints {
148 chart := httpEndpointRequestsChartTmpl.Copy()
149 +
150 chart.ID = fmt.Sprintf(chart.ID, path)
151 chart.Labels = []module.Label{
152 {Key: "http_endpoint", Value: path},
@@ -145,6 +156,7 @@ func httpEndpointCharts() module.Charts {
156 }
157 charts = append(charts, chart)
158 }
159 +
160 return charts
161 }
162
@@ -156,6 +168,137 @@ var httpEndpointRequestsChartTmpl = module.Chart{
168 Ctx: "nats.http_endpoint_requests",
169 Priority: prioHttpEndpointRequests,
170 Dims: module.Dims{
159 - {ID: "http_endpoint_%s_req", Name: "requests", Algo: module.Incremental},
171 + {ID: "varz_http_endpoint_%s_req", Name: "requests", Algo: module.Incremental},
172 },
173 }
174 +
175 +var accountChartsTmpl = module.Charts{
176 + accountTrafficTmpl.Copy(),
177 + accountMessagesTmpl.Copy(),
178 + accountConnectionsCurrentTmpl.Copy(),
179 + accountConnectionsRateTmpl.Copy(),
180 + accountSubscriptionsTmpl.Copy(),
181 + accountSlowConsumersTmpl.Copy(),
182 + accountLeadNodesTmpl.Copy(),
183 +}
184 +
185 +var (
186 + accountTrafficTmpl = module.Chart{
187 + ID: "account_%s_traffic",
188 + Title: "Account Traffic",
189 + Units: "bytes/s",
190 + Fam: "acc traffic",
191 + Ctx: "nats.account_traffic",
192 + Priority: prioAccountTraffic,
193 + Type: module.Area,
194 + Dims: module.Dims{
195 + {ID: "accstatz_acc_%s_received_bytes", Name: "received", Algo: module.Incremental},
196 + {ID: "accstatz_acc_%s_sent_bytes", Name: "sent", Mul: -1, Algo: module.Incremental},
197 + },
198 + }
199 + accountMessagesTmpl = module.Chart{
200 + ID: "account_%s_messages",
201 + Title: "Account Messages",
202 + Units: "messages/s",
203 + Fam: "acc traffic",
204 + Ctx: "nats.account_messages",
205 + Priority: prioAccountMessages,
206 + Type: module.Line,
207 + Dims: module.Dims{
208 + {ID: "accstatz_acc_%s_received_msgs", Name: "received", Algo: module.Incremental},
209 + {ID: "accstatz_acc_%s_sent_msgs", Name: "sent", Mul: -1, Algo: module.Incremental},
210 + },
211 + }
212 + accountConnectionsCurrentTmpl = module.Chart{
213 + ID: "account_%s_connections",
214 + Title: "Account Active Connections",
215 + Units: "connections",
216 + Fam: "acc connections",
217 + Ctx: "nats.account_connections",
218 + Priority: prioAccountConnections,
219 + Type: module.Line,
220 + Dims: module.Dims{
221 + {ID: "accstatz_acc_%s_conns", Name: "active"},
222 + },
223 + }
224 + accountConnectionsRateTmpl = module.Chart{
225 + ID: "account_%s_connections_rate",
226 + Title: "Account Connections",
227 + Units: "connections/s",
228 + Fam: "acc connections",
229 + Ctx: "nats.account_connections_rate",
230 + Priority: prioAccountConnectionsRate,
231 + Type: module.Line,
232 + Dims: module.Dims{
233 + {ID: "accstatz_acc_%s_total_conns", Name: "connections", Algo: module.Incremental},
234 + },
235 + }
236 + accountSubscriptionsTmpl = module.Chart{
237 + ID: "account_%s_subscriptions",
238 + Title: "Account Active Subscriptions",
239 + Units: "subscriptions",
240 + Fam: "acc subscriptions",
241 + Ctx: "nats.account_subscriptions",
242 + Priority: prioAccountSubscriptions,
243 + Type: module.Line,
244 + Dims: module.Dims{
245 + {ID: "accstatz_acc_%s_num_subs", Name: "active"},
246 + },
247 + }
248 + accountSlowConsumersTmpl = module.Chart{
249 + ID: "account_%s_slow_consumers",
250 + Title: "Account Slow Consumers",
251 + Units: "consumers/s",
252 + Fam: "acc consumers",
253 + Ctx: "nats.account_slow_consumers",
254 + Priority: prioAccountSlowConsumers,
255 + Type: module.Line,
256 + Dims: module.Dims{
257 + {ID: "accstatz_acc_%s_slow_consumers", Name: "slow", Algo: module.Incremental},
258 + },
259 + }
260 + accountLeadNodesTmpl = module.Chart{
261 + ID: "account_%s_leaf_nodes",
262 + Title: "Account Leaf Nodes",
263 + Units: "servers",
264 + Fam: "acc leaf nodes",
265 + Ctx: "nats.account_leaf_nodes",
266 + Priority: prioAccountLeafNodes,
267 + Type: module.Line,
268 + Dims: module.Dims{
269 + {ID: "accstatz_acc_%s_leaf_nodes", Name: "leafnode"},
270 + },
271 + }
272 +)
273 +
274 +func (c *Collector) addAccountCharts(acc string) {
275 + charts := accountChartsTmpl.Copy()
276 +
277 + for _, chart := range *charts {
278 + chart.ID = fmt.Sprintf(chart.ID, acc)
279 + chart.Labels = []module.Label{
280 + {Key: "account", Value: acc},
281 + }
282 + for _, dim := range chart.Dims {
283 + dim.ID = fmt.Sprintf(dim.ID, acc)
284 + }
285 + }
286 +
287 + if err := c.Charts().Add(*charts...); err != nil {
288 + c.Warningf("failed to add charts for account %s: %s", acc, err)
289 + }
290 +}
291 +
292 +func (c *Collector) removeAccountCharts(acc string) {
293 + px := fmt.Sprintf("accstatz_acc_%s_", acc)
294 + c.removeCharts(px)
295 +}
296 +
297 +func (c *Collector) removeCharts(prefix string) {
298 + for _, chart := range *c.Charts() {
299 + if strings.HasPrefix(chart.ID, prefix) {
300 + chart.MarkRemove()
301 + chart.MarkNotCreated()
302 + }
303 + }
304 +}
src/go/plugin/go.d/collector/nats/collect.go
+87 -33
@@ -10,26 +10,51 @@ import (
10 "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"
11 )
12
13 -const (
14 - urlPathVarz = "/varz"
15 - urlPathHealthz = "/healthz"
16 -)
17 -
13 func (c *Collector) collect() (map[string]int64, error) {
14 mx := make(map[string]int64)
15
21 - if err := c.collectVarz(mx); err != nil {
22 - return nil, err
23 - }
16 if err := c.collectHealthz(mx); err != nil {
17 return nil, err
18 }
19 + if err := c.collectVarz(mx); err != nil {
20 + return mx, err
21 + }
22 + if err := c.collectAccstatz(mx); err != nil {
23 + return mx, err
24 + }
25
26 return mx, nil
27 }
28
29 +func (c *Collector) collectHealthz(mx map[string]int64) error {
30 + req, err := web.NewHTTPRequestWithPath(c.RequestConfig, urlPathHealthz)
31 + if err != nil {
32 + return err
33 + }
34 +
35 + switch c.HealthzCheck {
36 + case "js-enabled-only":
37 + req.URL.Scheme = urlQueryHealthzJsEnabledOnly
38 + case "js-server-only":
39 + req.URL.Scheme = urlQueryHealthzJsServerOnly
40 + }
41 +
42 + var resp healthzResponse
43 + client := web.DoHTTP(c.httpClient).OnNokCode(func(resp *http.Response) (bool, error) { return true, nil })
44 + if err := client.RequestJSON(req, &resp); err != nil {
45 + return err
46 + }
47 + if resp.Status == nil {
48 + return fmt.Errorf("healthz response missing status")
49 + }
50 +
51 + mx["varz_srv_healthz_status_ok"] = metrix.Bool(*resp.Status == "ok")
52 + mx["varz_srv_healthz_status_error"] = metrix.Bool(*resp.Status != "ok")
53 +
54 + return nil
55 +}
56 +
57 func (c *Collector) collectVarz(mx map[string]int64) error {
32 - // https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#general-information
58 req, err := web.NewHTTPRequestWithPath(c.RequestConfig, urlPathVarz)
59 if err != nil {
60 return err
@@ -40,46 +65,75 @@ func (c *Collector) collectVarz(mx map[string]int64) error {
65 return err
66 }
67
43 - mx["uptime"] = int64(resp.Now.Sub(resp.Start).Seconds())
44 - mx["in_msgs"] = resp.InMsgs
45 - mx["out_msgs"] = resp.OutMsgs
46 - mx["in_bytes"] = resp.InBytes
47 - mx["out_bytes"] = resp.OutBytes
48 - mx["slow_consumers"] = resp.SlowConsumers
49 - mx["subscriptions"] = int64(resp.Subscriptions)
50 - mx["connections"] = int64(resp.Connections)
51 - mx["total_connections"] = int64(resp.TotalConnections)
52 - mx["routes"] = int64(resp.Routes)
53 - mx["remotes"] = int64(resp.Remotes)
54 - mx["cpu"] = int64(resp.CPU)
55 - mx["mem"] = resp.Mem
68 + mx["varz_srv_uptime"] = int64(resp.Now.Sub(resp.Start).Seconds())
69 + mx["varz_srv_in_msgs"] = resp.InMsgs
70 + mx["varz_srv_out_msgs"] = resp.OutMsgs
71 + mx["varz_srv_in_bytes"] = resp.InBytes
72 + mx["varz_srv_out_bytes"] = resp.OutBytes
73 + mx["varz_srv_slow_consumers"] = resp.SlowConsumers
74 + mx["varz_srv_subscriptions"] = int64(resp.Subscriptions)
75 + mx["varz_srv_connections"] = int64(resp.Connections)
76 + mx["varz_srv_total_connections"] = int64(resp.TotalConnections)
77 + mx["varz_srv_routes"] = int64(resp.Routes)
78 + mx["varz_srv_remotes"] = int64(resp.Remotes)
79 + mx["varz_srv_cpu"] = int64(resp.CPU)
80 + mx["varz_srv_mem"] = resp.Mem
81
82 for _, path := range httpEndpoints {
83 v := resp.HTTPReqStats[path]
59 - mx[fmt.Sprintf("http_endpoint_%s_req", path)] = int64(v)
84 + mx[fmt.Sprintf("varz_http_endpoint_%s_req", path)] = int64(v)
85 }
86
87 return nil
88 }
89
65 -func (c *Collector) collectHealthz(mx map[string]int64) error {
66 - // https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#health
67 - req, err := web.NewHTTPRequestWithPath(c.RequestConfig, urlPathHealthz)
90 +func (c *Collector) collectAccstatz(mx map[string]int64) error {
91 + req, err := web.NewHTTPRequestWithPath(c.RequestConfig, urlPathAccstatz)
92 if err != nil {
93 return err
94 }
95
72 - var resp healthzResponse
73 - client := web.DoHTTP(c.httpClient).OnNokCode(func(resp *http.Response) (bool, error) { return true, nil })
74 - if err := client.RequestJSON(req, &resp); err != nil {
96 + req.URL.RawQuery = urlQueryAccstatz
97 +
98 + var resp accstatzResponse
99 + if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
100 return err
101 }
77 - if resp.Status == nil {
78 - return fmt.Errorf("healthz response missing status")
102 +
103 + seen := make(map[string]bool)
104 +
105 + for _, acc := range resp.AccStats {
106 + if acc.Account == "" {
107 + continue
108 + }
109 +
110 + seen[acc.Account] = true
111 +
112 + px := fmt.Sprintf("accstatz_acc_%s_", acc.Account)
113 +
114 + mx[px+"conns"] = int64(acc.Conns)
115 + mx[px+"total_conns"] = int64(acc.TotalConns)
116 + mx[px+"num_subs"] = int64(acc.NumSubs)
117 + mx[px+"leaf_nodes"] = int64(acc.LeafNodes)
118 + mx[px+"slow_consumers"] = acc.SlowConsumers
119 + mx[px+"received_bytes"] = acc.Received.Bytes
120 + mx[px+"received_msgs"] = acc.Received.Msgs
121 + mx[px+"sent_bytes"] = acc.Sent.Bytes
122 + mx[px+"sent_msgs"] = acc.Sent.Msgs
123 }
124
81 - mx["healthz_status_ok"] = metrix.Bool(*resp.Status == "ok")
82 - mx["healthz_status_error"] = metrix.Bool(*resp.Status != "ok")
125 + for acc := range seen {
126 + if !c.seenAccounts[acc] {
127 + c.seenAccounts[acc] = true
128 + c.addAccountCharts(acc)
129 + }
130 + }
131 + for acc := range c.seenAccounts {
132 + if !seen[acc] {
133 + delete(c.seenAccounts, acc)
134 + c.removeAccountCharts(acc)
135 + }
136 + }
137
138 return nil
139 }
src/go/plugin/go.d/collector/nats/collector.go
+6 -1
@@ -37,14 +37,17 @@ func New() *Collector {
37 Timeout: confopt.Duration(time.Second),
38 },
39 },
40 + HealthzCheck: "default",
41 },
41 - charts: serverCharts.Copy(),
42 + charts: serverCharts.Copy(),
43 + seenAccounts: make(map[string]bool),
44 }
45 }
46
47 type Config struct {
48 Vnode string `yaml:"vnode,omitempty" json:"vnode"`
49 UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
50 + HealthzCheck string `yaml:"healthz_check,omitempty" json:"healthz_check"`
51 web.HTTPConfig `yaml:",inline" json:""`
52 }
53
@@ -55,6 +58,8 @@ type Collector struct {
58 charts *module.Charts
59
60 httpClient *http.Client
61 +
62 + seenAccounts map[string]bool
63 }
64
65 func (c *Collector) Configuration() any {
src/go/plugin/go.d/collector/nats/collector_test.go
+69 -34
@@ -20,16 +20,18 @@ var (
20 dataConfigJSON, _ = os.ReadFile("testdata/config.json")
21 dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")
22
23 - dataVer210Varz, _ = os.ReadFile("testdata/v2.10.24/varz.json")
23 dataVer210HealthzOk, _ = os.ReadFile("testdata/v2.10.24/healthz-ok.json")
24 + dataVer210Varz, _ = os.ReadFile("testdata/v2.10.24/varz.json")
25 + dataVer210Accstatz, _ = os.ReadFile("testdata/v2.10.24/accstatz.json")
26 )
27
28 func Test_testDataIsValid(t *testing.T) {
29 for name, data := range map[string][]byte{
30 "dataConfigJSON": dataConfigJSON,
31 "dataConfigYAML": dataConfigYAML,
31 - "dataVer210Varz": dataVer210Varz,
32 "dataVer210HealthzOk": dataVer210HealthzOk,
33 + "dataVer210Varz": dataVer210Varz,
34 + "dataVer210Accstatz": dataVer210Accstatz,
35 } {
36 require.NotNil(t, data, name)
37 }
@@ -125,37 +127,64 @@ func TestCollector_Collect(t *testing.T) {
127 }{
128 "success on valid response": {
129 prepare: caseOk,
128 - wantNumOfCharts: len(serverCharts),
130 + wantNumOfCharts: len(serverCharts) + len(accountChartsTmpl)*3,
131 wantMetrics: map[string]int64{
130 - "connections": 0,
131 - "cpu": 0,
132 - "healthz_status_error": 0,
133 - "healthz_status_ok": 1,
134 - "http_endpoint_/_req": 3,
135 - "http_endpoint_/accountz_req": 2,
136 - "http_endpoint_/accstatz_req": 2,
137 - "http_endpoint_/connz_req": 2,
138 - "http_endpoint_/gatewayz_req": 2,
139 - "http_endpoint_/healthz_req": 2017,
140 - "http_endpoint_/ipqueuesz_req": 0,
141 - "http_endpoint_/jsz_req": 3,
142 - "http_endpoint_/leafz_req": 2,
143 - "http_endpoint_/raftz_req": 0,
144 - "http_endpoint_/routez_req": 2,
145 - "http_endpoint_/stacksz_req": 0,
146 - "http_endpoint_/subsz_req": 1,
147 - "http_endpoint_/varz_req": 3750,
148 - "in_bytes": 0,
149 - "in_msgs": 0,
150 - "mem": 21725184,
151 - "out_bytes": 0,
152 - "out_msgs": 0,
153 - "remotes": 0,
154 - "routes": 0,
155 - "slow_consumers": 0,
156 - "subscriptions": 57,
157 - "total_connections": 0,
158 - "uptime": 27513,
132 + "accstatz_acc_$G_conns": 0,
133 + "accstatz_acc_$G_leaf_nodes": 0,
134 + "accstatz_acc_$G_num_subs": 5,
135 + "accstatz_acc_$G_received_bytes": 0,
136 + "accstatz_acc_$G_received_msgs": 0,
137 + "accstatz_acc_$G_sent_bytes": 0,
138 + "accstatz_acc_$G_sent_msgs": 0,
139 + "accstatz_acc_$G_slow_consumers": 0,
140 + "accstatz_acc_$G_total_conns": 0,
141 + "accstatz_acc_$SYS_conns": 0,
142 + "accstatz_acc_$SYS_leaf_nodes": 0,
143 + "accstatz_acc_$SYS_num_subs": 220,
144 + "accstatz_acc_$SYS_received_bytes": 0,
145 + "accstatz_acc_$SYS_received_msgs": 0,
146 + "accstatz_acc_$SYS_sent_bytes": 0,
147 + "accstatz_acc_$SYS_sent_msgs": 0,
148 + "accstatz_acc_$SYS_slow_consumers": 0,
149 + "accstatz_acc_$SYS_total_conns": 0,
150 + "accstatz_acc_default_conns": 44,
151 + "accstatz_acc_default_leaf_nodes": 0,
152 + "accstatz_acc_default_num_subs": 1133,
153 + "accstatz_acc_default_received_bytes": 62023455,
154 + "accstatz_acc_default_received_msgs": 916392,
155 + "accstatz_acc_default_sent_bytes": 529749990,
156 + "accstatz_acc_default_sent_msgs": 2546732,
157 + "accstatz_acc_default_slow_consumers": 1,
158 + "accstatz_acc_default_total_conns": 44,
159 + "varz_http_endpoint_/_req": 5710,
160 + "varz_http_endpoint_/accountz_req": 2201,
161 + "varz_http_endpoint_/accstatz_req": 6,
162 + "varz_http_endpoint_/connz_req": 3649,
163 + "varz_http_endpoint_/gatewayz_req": 2204,
164 + "varz_http_endpoint_/healthz_req": 3430,
165 + "varz_http_endpoint_/ipqueuesz_req": 0,
166 + "varz_http_endpoint_/jsz_req": 2958,
167 + "varz_http_endpoint_/leafz_req": 9,
168 + "varz_http_endpoint_/raftz_req": 0,
169 + "varz_http_endpoint_/routez_req": 2202,
170 + "varz_http_endpoint_/stacksz_req": 0,
171 + "varz_http_endpoint_/subsz_req": 4412,
172 + "varz_http_endpoint_/varz_req": 7114,
173 + "varz_srv_connections": 44,
174 + "varz_srv_cpu": 10,
175 + "varz_srv_healthz_status_error": 0,
176 + "varz_srv_healthz_status_ok": 1,
177 + "varz_srv_in_bytes": 62024985,
178 + "varz_srv_in_msgs": 916475,
179 + "varz_srv_mem": 95731712,
180 + "varz_srv_out_bytes": 529775656,
181 + "varz_srv_out_msgs": 2546840,
182 + "varz_srv_remotes": 0,
183 + "varz_srv_routes": 0,
184 + "varz_srv_slow_consumers": 1,
185 + "varz_srv_subscriptions": 1358,
186 + "varz_srv_total_connections": 74932,
187 + "varz_srv_uptime": 339394,
188 },
189 },
190 "fail on unexpected JSON response": {
@@ -201,10 +230,16 @@ func caseOk(t *testing.T) (*Collector, func()) {
230 srv := httptest.NewServer(http.HandlerFunc(
231 func(w http.ResponseWriter, r *http.Request) {
232 switch r.URL.Path {
204 - case urlPathVarz:
205 - _, _ = w.Write(dataVer210Varz)
233 case urlPathHealthz:
234 _, _ = w.Write(dataVer210HealthzOk)
235 + case urlPathVarz:
236 + _, _ = w.Write(dataVer210Varz)
237 + case urlPathAccstatz:
238 + if r.URL.RawQuery != urlQueryAccstatz {
239 + w.WriteHeader(http.StatusNotFound)
240 + return
241 + }
242 + _, _ = w.Write(dataVer210Accstatz)
243 default:
244 w.WriteHeader(http.StatusNotFound)
245 }
src/go/plugin/go.d/collector/nats/config_schema.json
+15
@@ -30,6 +30,17 @@
30 "description": "If set, the client will not follow HTTP redirects automatically.",
31 "type": "boolean"
32 },
33 + "healthz_check": {
34 + "title": "Health Check Mode",
35 + "description": "Controls the behavior of the `/healthz` endpoint [health check](https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#health).",
36 + "type": "string",
37 + "enum": [
38 + "default",
39 + "js-enabled-only",
40 + "js-server-only"
41 + ],
42 + "default": "default"
43 + },
44 "vnode": {
45 "title": "Vnode",
46 "description": "Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).",
@@ -127,6 +138,9 @@
138 "vnode": {
139 "ui:placeholder": "To use this option, first create a Virtual Node and then reference its name here."
140 },
141 + "healthz_check": {
142 + "ui:help": "`default` performs a full health check, ensuring the server can accept connections and that JetStream is functional, including checking accounts, streams, and consumers. `js-enabled-only` returns an error if JetStream is disabled. `js-server-only` checks if the server can accept connections and that JetStream is enabled, but skips health checks of accounts, streams, and consumers."
143 + },
144 "timeout": {
145 "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
146 },
@@ -152,6 +166,7 @@
166 "url",
167 "timeout",
168 "not_follow_redirects",
169 + "healthz_check",
170 "vnode"
171 ]
172 },
src/go/plugin/go.d/collector/nats/metadata.yaml
+60 -6
@@ -76,6 +76,10 @@ modules:
76 description: HTTP request timeout.
77 default_value: 1
78 required: false
79 + - name: healthz_check
80 + description: "Controls the behavior of the `/healthz` endpoint [health check](https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#health)."
81 + default_value: "default"
82 + required: false
83 - name: username
84 description: Username for basic HTTP authentication.
85 default_value: ""
@@ -188,17 +192,17 @@ modules:
192 unit: bytes/s
193 chart_type: area
194 dimensions:
191 - - name: in
192 - - name: out
195 + - name: received
196 + - name: sent
197 - name: nats.server_messages
198 description: Server Messages
199 unit: messages/s
200 chart_type: line
201 dimensions:
198 - - name: in
199 - - name: out
200 - - name: nats.server_connections_current
201 - description: Server Current Connections
202 + - name: received
203 + - name: sent
204 + - name: nats.server_connections
205 + description: Server Active Connections
206 unit: connections
207 chart_type: line
208 dimensions:
@@ -234,6 +238,56 @@ modules:
238 chart_type: line
239 dimensions:
240 - name: uptime
241 + - name: account
242 + description: These metrics refer to [accounts](https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#account-statistics).
243 + labels:
244 + - name: account
245 + description: "Account name."
246 + metrics:
247 + - name: nats.account_traffic
248 + description: Account Traffic
249 + unit: bytes/s
250 + chart_type: area
251 + dimensions:
252 + - name: received
253 + - name: sent
254 + - name: nats.account_messages
255 + description: Account Messages
256 + unit: messages/s
257 + chart_type: line
258 + dimensions:
259 + - name: received
260 + - name: sent
261 + - name: nats.account_connections
262 + description: Account Active Connections
263 + unit: connections
264 + chart_type: line
265 + dimensions:
266 + - name: active
267 + - name: nats.account_connections_rate
268 + description: Account Connections
269 + unit: connections/s
270 + chart_type: line
271 + dimensions:
272 + - name: connections
273 + - name: nats.account_subscriptions
274 + description: Account Active Subscriptions
275 + unit: subscriptions
276 + chart_type: line
277 + dimensions:
278 + - name: active
279 + - name: nats.account_slow_consumers
280 + description: Account Slow Consumers
281 + unit: consumers/s
282 + chart_type: line
283 + dimensions:
284 + - name: slow
285 + - name: nats.account_leaf_nodes
286 + description: Account Leaf Nodes
287 + unit: servers
288 + chart_type: line
289 + dimensions:
290 + - name: leafnode
291 - name: http endpoint
292 description: These metrics refer to HTTP endpoints.
293 labels:
src/go/plugin/go.d/collector/nats/restapi.go
+58 -36
@@ -4,10 +4,50 @@ package nats
4
5 import (
6 "time"
7 +
8 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"
9 )
10
11 // https://docs.nats.io/running-a-nats-service/nats_admin/monitoring
12
13 +const (
14 + // https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#general-information
15 + urlPathVarz = "/varz"
16 + // https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#health
17 + urlPathHealthz = "/healthz"
18 + // https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#account-statistics
19 + urlPathAccstatz = "/accstatz"
20 +)
21 +
22 +var (
23 + urlQueryHealthzJsEnabledOnly = web.URLQuery("js-enabled-only", "true")
24 + urlQueryHealthzJsServerOnly = web.URLQuery("js-server-only", "true")
25 + urlQueryAccstatz = web.URLQuery("unused", "1")
26 +)
27 +
28 +// //https://github.com/nats-io/nats-server/blob/v2.10.24/server/server.go#L2851
29 +var httpEndpoints = []string{
30 + "/",
31 + "/varz",
32 + "/connz",
33 + "/routez",
34 + "/gatewayz",
35 + "/leafz",
36 + "/subsz",
37 + "/stacksz",
38 + "/accountz",
39 + "/accstatz",
40 + "/jsz",
41 + "/healthz",
42 + "/ipqueuesz",
43 + "/raftz",
44 +}
45 +
46 +// https://github.com/nats-io/nats-server/blob/v2.10.24/server/monitor.go#L3125
47 +type healthzResponse struct {
48 + Status *string `json:"status"`
49 +}
50 +
51 // https://github.com/nats-io/nats-server/blob/v2.10.24/server/monitor.go#L1164
52 type varzResponse struct {
53 ID string `json:"server_id"`
@@ -19,24 +59,9 @@ type varzResponse struct {
59 IP string `json:"ip,omitempty"`
60 MaxConn int `json:"max_connections"`
61 MaxSubs int `json:"max_subscriptions,omitempty"`
22 - PingInterval time.Duration `json:"ping_interval"`
23 - MaxPingsOut int `json:"ping_max"`
24 - HTTPHost string `json:"http_host"`
25 - HTTPPort int `json:"http_port"`
26 - HTTPBasePath string `json:"http_base_path"`
27 - HTTPSPort int `json:"https_port"`
28 - AuthTimeout float64 `json:"auth_timeout"`
29 - MaxControlLine int32 `json:"max_control_line"`
30 - MaxPayload int `json:"max_payload"`
31 - MaxPending int64 `json:"max_pending"`
32 - TLSTimeout float64 `json:"tls_timeout"`
33 - WriteDeadline time.Duration `json:"write_deadline"`
62 Start time.Time `json:"start"`
63 Now time.Time `json:"now"`
36 - Uptime string `json:"uptime"`
64 Mem int64 `json:"mem"`
38 - Cores int `json:"cores"`
39 - MaxProcs int `json:"gomaxprocs"`
65 CPU float64 `json:"cpu"`
66 Connections int `json:"connections"`
67 TotalConnections uint64 `json:"total_connections"`
@@ -52,25 +77,22 @@ type varzResponse struct {
77 HTTPReqStats map[string]uint64 `json:"http_req_stats"`
78 }
79
55 -// //https://github.com/nats-io/nats-server/blob/v2.10.24/server/server.go#L2851
56 -var httpEndpoints = []string{
57 - "/",
58 - "/varz",
59 - "/connz",
60 - "/routez",
61 - "/gatewayz",
62 - "/leafz",
63 - "/subsz",
64 - "/stacksz",
65 - "/accountz",
66 - "/accstatz",
67 - "/jsz",
68 - "/healthz",
69 - "/ipqueuesz",
70 - "/raftz",
71 -}
72 -
73 -// https://github.com/nats-io/nats-server/blob/v2.10.24/server/monitor.go#L3125
74 -type healthzResponse struct {
75 - Status *string `json:"status"`
80 +// https://github.com/nats-io/nats-server/blob/v2.10.24/server/monitor.go#L2279
81 +type accstatzResponse struct {
82 + AccStats []struct {
83 + Account string `json:"acc"`
84 + Conns int `json:"conns"`
85 + TotalConns int `json:"total_conns"`
86 + LeafNodes int `json:"leafnodes"`
87 + NumSubs uint32 `json:"num_subscriptions"`
88 + Sent struct {
89 + Msgs int64 `json:"msgs"`
90 + Bytes int64 `json:"bytes"`
91 + } `json:"sent"`
92 + Received struct {
93 + Msgs int64 `json:"msgs"`
94 + Bytes int64 `json:"bytes"`
95 + } `json:"received"`
96 + SlowConsumers int64 `json:"slow_consumers"`
97 + } `json:"account_statz"`
98 }
src/go/plugin/go.d/collector/nats/testdata/config.json
+1
@@ -1,6 +1,7 @@
1 {
2 "vnode": "ok",
3 "update_every": 123,
4 + "healthz_check": "ok",
5 "url": "ok",
6 "body": "ok",
7 "method": "ok",
src/go/plugin/go.d/collector/nats/testdata/config.yaml
+1
@@ -1,5 +1,6 @@
1 vnode: "ok"
2 update_every: 123
3 +healthz_check: "ok"
4 url: "ok"
5 body: "ok"
6 method: "ok"
src/go/plugin/go.d/collector/nats/testdata/v2.10.24/accstatz.json new
+54
@@ -0,0 +1,54 @@
1 +{
2 + "server_id": "NDR5FR76SWSTAP5LSKUNYG7ADXPFLXXWNC7ALU3WGX6WLFMMCIDQAD4J",
3 + "now": "2024-12-21T15:43:05.910221284Z",
4 + "account_statz": [
5 + {
6 + "acc": "$SYS",
7 + "conns": 0,
8 + "leafnodes": 0,
9 + "total_conns": 0,
10 + "num_subscriptions": 220,
11 + "sent": {
12 + "msgs": 0,
13 + "bytes": 0
14 + },
15 + "received": {
16 + "msgs": 0,
17 + "bytes": 0
18 + },
19 + "slow_consumers": 0
20 + },
21 + {
22 + "acc": "$G",
23 + "conns": 0,
24 + "leafnodes": 0,
25 + "total_conns": 0,
26 + "num_subscriptions": 5,
27 + "sent": {
28 + "msgs": 0,
29 + "bytes": 0
30 + },
31 + "received": {
32 + "msgs": 0,
33 + "bytes": 0
34 + },
35 + "slow_consumers": 0
36 + },
37 + {
38 + "acc": "default",
39 + "conns": 44,
40 + "leafnodes": 0,
41 + "total_conns": 44,
42 + "num_subscriptions": 1133,
43 + "sent": {
44 + "msgs": 2546732,
45 + "bytes": 529749990
46 + },
47 + "received": {
48 + "msgs": 916392,
49 + "bytes": 62023455
50 + },
51 + "slow_consumers": 1
52 + }
53 + ]
54 +}
src/go/plugin/go.d/collector/nats/testdata/v2.10.24/varz.json
+79 -45
@@ -1,73 +1,107 @@
1 {
2 - "server_id": "NASZPQXJ3BIJOGQHV5ZEWGI6EH3YRQPI2Z5GJRA4AZ47TC4PX4OJGY63",
3 - "server_name": "NASZPQXJ3BIJOGQHV5ZEWGI6EH3YRQPI2Z5GJRA4AZ47TC4PX4OJGY63",
2 + "server_id": "NDR5FR76SWSTAP5LSKUNYG7ADXPFLXXWNC7ALU3WGX6WLFMMCIDQAD4J",
3 + "server_name": "us-south-nats-demo",
4 "version": "2.10.24",
5 "proto": 1,
6 "git_commit": "1d6f7ea",
7 "go": "go1.23.4",
8 "host": "0.0.0.0",
9 "port": 4222,
10 - "max_connections": 65536,
10 + "auth_required": true,
11 + "max_connections": 250000,
12 + "max_subscriptions": 200000,
13 "ping_interval": 120000000000,
14 "ping_max": 2,
15 "http_host": "0.0.0.0",
14 - "http_port": 8222,
16 + "http_port": 0,
17 "http_base_path": "",
16 - "https_port": 0,
17 - "auth_timeout": 2,
18 + "https_port": 8222,
19 + "auth_timeout": 6,
20 "max_control_line": 4096,
21 "max_payload": 1048576,
22 "max_pending": 67108864,
21 - "cluster": {
22 - "name": "my_cluster",
23 - "addr": "0.0.0.0",
24 - "cluster_port": 6222,
25 - "auth_timeout": 2,
26 - "tls_timeout": 2,
27 - "pool_size": 3
28 - },
23 + "cluster": {},
24 "gateway": {},
30 - "leaf": {},
31 - "mqtt": {},
32 - "websocket": {},
33 - "jetstream": {},
34 - "tls_timeout": 2,
25 + "leaf": {
26 + "host": "0.0.0.0",
27 + "port": 7422,
28 + "auth_timeout": 6,
29 + "tls_timeout": 5,
30 + "tls_required": true
31 + },
32 + "mqtt": {
33 + "host": "0.0.0.0",
34 + "port": 1883,
35 + "no_auth_user": "demo-user",
36 + "tls_timeout": 5,
37 + "ack_wait": 60000000000,
38 + "max_ack_pending": 1024
39 + },
40 + "websocket": {
41 + "host": "0.0.0.0",
42 + "port": 8443,
43 + "no_auth_user": "demo-user",
44 + "handshake_timeout": 5000000000,
45 + "compression": true
46 + },
47 + "jetstream": {
48 + "config": {
49 + "max_memory": 10737418240,
50 + "max_storage": 440234147840,
51 + "store_dir": "/var/jetstream/jetstream",
52 + "sync_interval": 120000000000,
53 + "compress_ok": true
54 + },
55 + "stats": {
56 + "memory": 0,
57 + "storage": 1819021225,
58 + "reserved_memory": 0,
59 + "reserved_storage": 1620615736,
60 + "accounts": 1,
61 + "ha_assets": 0,
62 + "api": {
63 + "total": 324897,
64 + "errors": 323
65 + }
66 + }
67 + },
68 + "tls_timeout": 5,
69 "write_deadline": 10000000000,
36 - "start": "2024-12-19T11:51:48.038140697Z",
37 - "now": "2024-12-19T19:30:21.110744698Z",
38 - "uptime": "7h38m33s",
39 - "mem": 21725184,
70 + "start": "2024-12-17T17:27:05.96540148Z",
71 + "now": "2024-12-21T15:43:40.410706574Z",
72 + "uptime": "3d22h16m34s",
73 + "mem": 95731712,
74 "cores": 16,
75 "gomaxprocs": 16,
42 - "cpu": 0,
43 - "connections": 0,
44 - "total_connections": 0,
76 + "cpu": 10,
77 + "connections": 44,
78 + "total_connections": 74932,
79 "routes": 0,
80 "remotes": 0,
81 "leafnodes": 0,
48 - "in_msgs": 0,
49 - "out_msgs": 0,
50 - "in_bytes": 0,
51 - "out_bytes": 0,
52 - "slow_consumers": 0,
53 - "subscriptions": 57,
82 + "in_msgs": 916475,
83 + "out_msgs": 2546840,
84 + "in_bytes": 62024985,
85 + "out_bytes": 529775656,
86 + "slow_consumers": 1,
87 + "subscriptions": 1358,
88 "http_req_stats": {
55 - "/": 3,
56 - "/accountz": 2,
57 - "/accstatz": 2,
58 - "/connz": 2,
59 - "/gatewayz": 2,
60 - "/healthz": 2017,
61 - "/jsz": 3,
62 - "/leafz": 2,
63 - "/routez": 2,
64 - "/subsz": 1,
65 - "/varz": 3750
89 + "/": 5710,
90 + "/accountz": 2201,
91 + "/accstatz": 6,
92 + "/connz": 3649,
93 + "/gatewayz": 2204,
94 + "/healthz": 3430,
95 + "/jsz": 2958,
96 + "/leafz": 9,
97 + "/routez": 2202,
98 + "/subsz": 4412,
99 + "/varz": 7114
100 },
67 - "config_load_time": "2024-12-19T11:51:48.038140697Z",
101 + "config_load_time": "2024-12-17T17:27:05.96540148Z",
102 "system_account": "$SYS",
103 "slow_consumer_stats": {
70 - "clients": 0,
104 + "clients": 1,
105 "routes": 0,
106 "gateways": 0,
107 "leafs": 0
src/go/plugin/go.d/pkg/web/request_config.go
+4
@@ -103,3 +103,7 @@ func NewHTTPRequestWithPath(cfg RequestConfig, urlPath string) (*http.Request, e
103
104 return NewHTTPRequest(cfg)
105 }
106 +
107 +func URLQuery(key, value string) string {
108 + return url.Values{key: []string{value}}.Encode()
109 +}