@cryptotaxi247 / netdata-1 / commits / e417c525b

go.d add typesense collector (#18538)

Ilya Mashchenko committed Sep 13, 2024 at 12:12 UTC e417c525b181146931befb66319d2e725ec5ec0d
19 files changed +1205 -2
src/go/plugin/go.d/README.md
+3 -2
@@ -147,10 +147,11 @@ see the appropriate collector readme.
147 | [tomcat](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/tomcat) | Tomcat |
148 | [tor](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/tor) | Tor |
149 | [traefik](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/traefik) | Traefik |
150 +| [typesense](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/typesense) | Typesense |
151 | [unbound](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/unbound) | Unbound |
152 | [upsd](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/upsd) | UPSd (Nut) |
152 -| [uwsgi](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/uwsgi) | uWSGI |
153 -| [varnish](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/varnish) | Varnish |
153 +| [uwsgi](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/uwsgi) | uWSGI |
154 +| [varnish](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/varnish) | Varnish |
155 | [vcsa](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/vcsa) | vCenter Server Appliance |
156 | [vernemq](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/vernemq) | VerneMQ |
157 | [vsphere](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/vsphere) | VMware vCenter Server |
src/go/plugin/go.d/agent/discovery/sd/discoverer/dockerd/docker.go
+2
@@ -199,6 +199,8 @@ func (d *Discoverer) buildTargetGroup(cntr types.Container) model.TargetGroup {
199 }
200 tgt.Address = net.JoinHostPort(tgt.IPAddress, tgt.PrivatePort)
201
202 + d.Infof("%+v\n", tgt)
203 +
204 hash, err := calcHash(tgt)
205 if err != nil {
206 continue
src/go/plugin/go.d/config/go.d.conf
+1
@@ -111,6 +111,7 @@ modules:
111 # tomcat: yes
112 # tor: yes
113 # traefik: yes
114 +# typesense: yes
115 # upsd: yes
116 # unbound: yes
117 # uwsgi: yes
src/go/plugin/go.d/config/go.d/sd/docker.conf
+8
@@ -82,6 +82,8 @@ classify:
82 expr: '{{ and (eq .PrivatePort "9051") (match "sp" .Image "*/tor */tor:*") }}'
83 - tags: "tomcat"
84 expr: '{{ match "sp" .Image "tomcat tomcat:* */tomcat */tomcat:*" }}'
85 + - tags: "typesense"
86 + expr: '{{ match "sp" .Image "typesense/typesense typesense/typesense:*" }}'
87 - tags: "varnish"
88 expr: '{{ match "sp" .Image "varnish varnish:*" }}'
89 - tags: "vernemq"
@@ -254,6 +256,12 @@ compose:
256 module: tomcat
257 name: docker_{{.Name}}
258 url: http://{{.Address}}
259 + - selector: "typesense"
260 + template: |
261 + module: typesense
262 + name: docker_{{.Name}}
263 + url: http://{{.Address}}
264 + api_key: {{ trimPrefix "--api-key=" (regexFind "--api-key=[^ ]+" .Command) -}}
265 - selector: "tor"
266 template: |
267 module: tor
src/go/plugin/go.d/config/go.d/sd/net_listeners.conf
+8
@@ -130,6 +130,8 @@ classify:
130 expr: '{{ and (eq .Port "9051") (eq .Comm "tor") }}'
131 - tags: "traefik"
132 expr: '{{ and (eq .Port "80" "8080") (eq .Comm "traefik") }}'
133 + - tags: "typesense"
134 + expr: '{{ and (eq .Port "8108") (eq .Comm "typesense-server") }}'
135 - tags: "unbound"
136 expr: '{{ and (eq .Port "8953") (eq .Comm "unbound") }}'
137 - tags: "upsd"
@@ -488,6 +490,12 @@ compose:
490 module: traefik
491 name: local
492 url: http://{{.Address}}/metrics
493 + - selector: "typesense"
494 + template: |
495 + module: typesense
496 + name: local
497 + url: http://{{.Address}}
498 + api_key: {{ trimPrefix "--api-key=" (regexFind "--api-key=[^ ]+" .Cmdline) -}}
499 - selector: "tomcat"
500 template: |
501 module: tomcat
src/go/plugin/go.d/config/go.d/typesense.conf new
+6
@@ -0,0 +1,6 @@
1 +## All available configuration options, their descriptions and default values:
2 +## https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/typesense#readme
3 +
4 +#jobs:
5 +# - name: local
6 +# url: http://127.0.0.1:8108
src/go/plugin/go.d/modules/init.go
+1
@@ -103,6 +103,7 @@ import (
103 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/tomcat"
104 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/tor"
105 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/traefik"
106 + _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/typesense"
107 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/unbound"
108 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/upsd"
109 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/uwsgi"
src/go/plugin/go.d/modules/typesense/charts.go new
+105
@@ -0,0 +1,105 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package typesense
4 +
5 +import "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
6 +
7 +var baseCharts = module.Charts{
8 + healthStatusChart.Copy(),
9 +}
10 +
11 +const precision = 1000
12 +
13 +const (
14 + prioHealthStatus = module.Priority + iota
15 +
16 + prioTotalRequests
17 + prioRequestsByOperation
18 + prioLatencyByOperation
19 + prioOverloadedRequests
20 +)
21 +
22 +var healthStatusChart = module.Chart{
23 + ID: "health_status",
24 + Title: "Health Status",
25 + Units: "status",
26 + Fam: "health",
27 + Ctx: "typesense.health_status",
28 + Type: module.Line,
29 + Priority: prioHealthStatus,
30 + Dims: module.Dims{
31 + {ID: "health_status_ok", Name: "ok"},
32 + {ID: "health_status_out_of_disk", Name: "out_of_disk"},
33 + {ID: "health_status_out_of_memory", Name: "out_of_memory"},
34 + },
35 +}
36 +
37 +var statsCharts = module.Charts{
38 + totalRequestsChart.Copy(),
39 + requestsByOperationChart.Copy(),
40 + overloadedRequestsChart.Copy(),
41 + latencyByOperationChart.Copy(),
42 +}
43 +
44 +var (
45 + totalRequestsChart = module.Chart{
46 + ID: "total_requests",
47 + Title: "Total Requests",
48 + Units: "requests/s",
49 + Fam: "requests",
50 + Ctx: "typesense.total_requests",
51 + Type: module.Line,
52 + Priority: prioTotalRequests,
53 + Dims: module.Dims{
54 + {ID: "total_requests_per_second", Name: "requests", Div: precision},
55 + },
56 + }
57 + requestsByOperationChart = module.Chart{
58 + ID: "requests_by_type",
59 + Title: "Requests by Operation",
60 + Units: "requests/s",
61 + Fam: "requests",
62 + Ctx: "typesense.requests_by_operation",
63 + Type: module.Line,
64 + Priority: prioRequestsByOperation,
65 + Dims: module.Dims{
66 + {ID: "search_requests_per_second", Name: "search", Div: precision},
67 + {ID: "write_requests_per_second", Name: "write", Div: precision},
68 + {ID: "import_requests_per_second", Name: "import", Div: precision},
69 + {ID: "delete_requests_per_second", Name: "delete", Div: precision},
70 + },
71 + }
72 + latencyByOperationChart = module.Chart{
73 + ID: "latency_by_operation",
74 + Title: "Latency by Operation",
75 + Units: "milliseconds",
76 + Fam: "requests",
77 + Ctx: "typesense.latency_by_operation",
78 + Type: module.Line,
79 + Priority: prioLatencyByOperation,
80 + Dims: module.Dims{
81 + {ID: "search_latency_ms", Name: "search"},
82 + {ID: "write_latency_ms", Name: "write"},
83 + {ID: "import_latency_ms", Name: "import"},
84 + {ID: "delete_latency_ms", Name: "delete"},
85 + },
86 + }
87 + overloadedRequestsChart = module.Chart{
88 + ID: "overloaded_requests",
89 + Title: "Overloaded Requests",
90 + Units: "requests/s",
91 + Fam: "requests",
92 + Ctx: "typesense.overloaded_requests",
93 + Type: module.Line,
94 + Priority: prioOverloadedRequests,
95 + Dims: module.Dims{
96 + {ID: "overloaded_requests_per_second", Name: "overloaded", Div: precision},
97 + },
98 + }
99 +)
100 +
101 +func (ts *Typesense) addStatsCharts() {
102 + if err := ts.charts.Add(*statsCharts.Copy()...); err != nil {
103 + ts.Warningf("error adding stats charts: %v", err)
104 + }
105 +}
src/go/plugin/go.d/modules/typesense/collect.go new
+152
@@ -0,0 +1,152 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package typesense
4 +
5 +import (
6 + "encoding/json"
7 + "fmt"
8 + "io"
9 + "net/http"
10 + "strings"
11 +
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/stm"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"
14 +)
15 +
16 +const (
17 + urlPathHealth = "/health"
18 + urlPathStats = "/stats.json"
19 +)
20 +
21 +// https://typesense.org/docs/27.0/api/cluster-operations.html#health
22 +type healthResponse struct {
23 + Ok *bool `json:"ok"`
24 + Err string `json:"resource_error"`
25 +}
26 +
27 +// https://typesense.org/docs/27.0/api/cluster-operations.html#api-stats
28 +type statsResponse struct {
29 + DeleteLatencyMs float64 `json:"delete_latency_ms" stm:"delete_latency_ms"`
30 + DeleteRequestsPerSecond float64 `json:"delete_requests_per_second" stm:"delete_requests_per_second,1000,1"`
31 + ImportLatencyMs float64 `json:"import_latency_ms" stm:"import_latency_ms"`
32 + ImportRequestsPerSecond float64 `json:"import_requests_per_second" stm:"import_requests_per_second,1000,1"`
33 + OverloadedRequestsPerSecond float64 `json:"overloaded_requests_per_second" stm:"overloaded_requests_per_second,1000,1"`
34 + PendingWriteBatches float64 `json:"pending_write_batches" stm:"pending_write_batches"`
35 + SearchLatencyMs float64 `json:"search_latency_ms" stm:"search_latency_ms"`
36 + SearchRequestsPerSecond float64 `json:"search_requests_per_second" stm:"search_requests_per_second,1000,1"`
37 + TotalRequestsPerSecond float64 `json:"total_requests_per_second" stm:"total_requests_per_second,1000,1"`
38 + WriteLatencyMs float64 `json:"write_latency_ms" stm:"write_latency_ms"`
39 + WriteRequestsPerSecond float64 `json:"write_requests_per_second" stm:"write_requests_per_second,1000,1"`
40 +}
41 +
42 +func (ts *Typesense) collect() (map[string]int64, error) {
43 + mx := make(map[string]int64)
44 +
45 + if err := ts.collectHealth(mx); err != nil {
46 + return nil, err
47 + }
48 +
49 + if err := ts.collectStats(mx); err != nil {
50 + return nil, err
51 + }
52 +
53 + return mx, nil
54 +}
55 +
56 +func (ts *Typesense) collectHealth(mx map[string]int64) error {
57 + req, err := web.NewHTTPRequestWithPath(ts.Request, urlPathHealth)
58 + if err != nil {
59 + return fmt.Errorf("creating health request: %w", err)
60 + }
61 +
62 + var resp healthResponse
63 + if err := ts.doOKDecode(req, &resp); err != nil {
64 + return err
65 + }
66 +
67 + px := "health_status_"
68 +
69 + for _, v := range []string{"ok", "out_of_disk", "out_of_memory"} {
70 + mx[px+v] = 0
71 + }
72 +
73 + if resp.Ok == nil {
74 + return fmt.Errorf("unexpected response: no health status found")
75 + }
76 +
77 + if resp.Err != "" {
78 + mx[px+strings.ToLower(resp.Err)] = 1
79 + } else if *resp.Ok {
80 + mx[px+"ok"] = 1
81 + }
82 +
83 + return nil
84 +}
85 +
86 +func (ts *Typesense) collectStats(mx map[string]int64) error {
87 + if !ts.doStats || ts.APIKey == "" {
88 + return nil
89 + }
90 +
91 + req, err := web.NewHTTPRequestWithPath(ts.Request, urlPathStats)
92 + if err != nil {
93 + return fmt.Errorf("creating stats request: %w", err)
94 + }
95 +
96 + req.Header.Set("X-TYPESENSE-API-KEY", ts.APIKey)
97 +
98 + var resp statsResponse
99 + if err := ts.doOKDecode(req, &resp); err != nil {
100 + if !isStatusUnauthorized(err) {
101 + return err
102 + }
103 +
104 + ts.doStats = false
105 + ts.Warning(err)
106 +
107 + return nil
108 + }
109 +
110 + ts.once.Do(ts.addStatsCharts)
111 +
112 + for k, v := range stm.ToMap(resp) {
113 + mx[k] = v
114 + }
115 +
116 + return nil
117 +}
118 +
119 +func (ts *Typesense) doOKDecode(req *http.Request, in interface{}) error {
120 + resp, err := ts.httpClient.Do(req)
121 + if err != nil {
122 + return fmt.Errorf("error on HTTP request '%s': %v", req.URL, err)
123 + }
124 + defer closeBody(resp)
125 +
126 + if resp.StatusCode != http.StatusOK {
127 + // {"message": "Forbidden - a valid `x-typesense-api-key` header must be sent."}
128 + var msg struct {
129 + Msg string `json:"message"`
130 + }
131 + if err := json.NewDecoder(resp.Body).Decode(&msg); err == nil {
132 + return fmt.Errorf("'%s' returned HTTP status code: %d (msg: '%s')", req.URL, resp.StatusCode, msg.Msg)
133 + }
134 + return fmt.Errorf("'%s' returned HTTP status code: %d", req.URL, resp.StatusCode)
135 + }
136 +
137 + if err := json.NewDecoder(resp.Body).Decode(in); err != nil {
138 + return fmt.Errorf("error on decoding response from '%s': %v", req.URL, err)
139 + }
140 + return nil
141 +}
142 +
143 +func closeBody(resp *http.Response) {
144 + if resp != nil && resp.Body != nil {
145 + _, _ = io.Copy(io.Discard, resp.Body)
146 + _ = resp.Body.Close()
147 + }
148 +}
149 +
150 +func isStatusUnauthorized(err error) bool {
151 + return strings.Contains(err.Error(), "code: 401")
152 +}
src/go/plugin/go.d/modules/typesense/config_schema.json new
+193
@@ -0,0 +1,193 @@
1 +{
2 + "jsonSchema": {
3 + "$schema": "http://json-schema.org/draft-07/schema#",
4 + "title": "Typesense collector configuration.",
5 + "type": "object",
6 + "properties": {
7 + "update_every": {
8 + "title": "Update every",
9 + "description": "Data collection interval, measured in seconds.",
10 + "type": "integer",
11 + "minimum": 1,
12 + "default": 1
13 + },
14 + "url": {
15 + "title": "URL",
16 + "description": "The base URL of the Typesense server.",
17 + "type": "string",
18 + "default": "http://127.0.0.1:8108",
19 + "format": "uri"
20 + },
21 + "timeout": {
22 + "title": "Timeout",
23 + "description": "The timeout in seconds for the HTTP request.",
24 + "type": "number",
25 + "minimum": 0.5,
26 + "default": 1
27 + },
28 + "not_follow_redirects": {
29 + "title": "Not follow redirects",
30 + "description": "If set, the client will not follow HTTP redirects automatically.",
31 + "type": "boolean"
32 + },
33 + "api_key": {
34 + "title": "API Key",
35 + "description": "The Typesense [API Key](https://typesense.org/docs/0.20.0/api/api-keys.html#api-keys) (`X-TYPESENSE-API-KEY`).",
36 + "type": "string",
37 + "sensitive": true
38 + },
39 + "username": {
40 + "title": "Username",
41 + "description": "The username for basic authentication.",
42 + "type": "string",
43 + "sensitive": true
44 + },
45 + "password": {
46 + "title": "Password",
47 + "description": "The password for basic authentication.",
48 + "type": "string",
49 + "sensitive": true
50 + },
51 + "proxy_url": {
52 + "title": "Proxy URL",
53 + "description": "The URL of the proxy server.",
54 + "type": "string"
55 + },
56 + "proxy_username": {
57 + "title": "Proxy username",
58 + "description": "The username for proxy authentication.",
59 + "type": "string",
60 + "sensitive": true
61 + },
62 + "proxy_password": {
63 + "title": "Proxy password",
64 + "description": "The password for proxy authentication.",
65 + "type": "string",
66 + "sensitive": true
67 + },
68 + "headers": {
69 + "title": "Headers",
70 + "description": "Additional HTTP headers to include in the request.",
71 + "type": [
72 + "object",
73 + "null"
74 + ],
75 + "additionalProperties": {
76 + "type": "string"
77 + }
78 + },
79 + "tls_skip_verify": {
80 + "title": "Skip TLS verification",
81 + "description": "If set, TLS certificate verification will be skipped.",
82 + "type": "boolean"
83 + },
84 + "tls_ca": {
85 + "title": "TLS CA",
86 + "description": "The path to the CA certificate file for TLS verification.",
87 + "type": "string",
88 + "pattern": "^$|^/"
89 + },
90 + "tls_cert": {
91 + "title": "TLS certificate",
92 + "description": "The path to the client certificate file for TLS authentication.",
93 + "type": "string",
94 + "pattern": "^$|^/"
95 + },
96 + "tls_key": {
97 + "title": "TLS key",
98 + "description": "The path to the client key file for TLS authentication.",
99 + "type": "string",
100 + "pattern": "^$|^/"
101 + },
102 + "body": {
103 + "title": "Body",
104 + "type": "string"
105 + },
106 + "method": {
107 + "title": "Method",
108 + "type": "string"
109 + }
110 + },
111 + "required": [
112 + "url"
113 + ],
114 + "additionalProperties": false,
115 + "patternProperties": {
116 + "^name$": {}
117 + }
118 + },
119 + "uiSchema": {
120 + "uiOptions": {
121 + "fullPage": true
122 + },
123 + "body": {
124 + "ui:widget": "hidden"
125 + },
126 + "method": {
127 + "ui:widget": "hidden"
128 + },
129 + "timeout": {
130 + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
131 + },
132 + "api_key": {
133 + "ui:widget": "password"
134 + },
135 + "username": {
136 + "ui:widget": "password"
137 + },
138 + "proxy_username": {
139 + "ui:widget": "password"
140 + },
141 + "password": {
142 + "ui:widget": "password"
143 + },
144 + "proxy_password": {
145 + "ui:widget": "password"
146 + },
147 + "ui:flavour": "tabs",
148 + "ui:options": {
149 + "tabs": [
150 + {
151 + "title": "Base",
152 + "fields": [
153 + "update_every",
154 + "url",
155 + "timeout",
156 + "not_follow_redirects"
157 + ]
158 + },
159 + {
160 + "title": "Auth",
161 + "fields": [
162 + "api_key",
163 + "username",
164 + "password"
165 + ]
166 + },
167 + {
168 + "title": "TLS",
169 + "fields": [
170 + "tls_skip_verify",
171 + "tls_ca",
172 + "tls_cert",
173 + "tls_key"
174 + ]
175 + },
176 + {
177 + "title": "Proxy",
178 + "fields": [
179 + "proxy_url",
180 + "proxy_username",
181 + "proxy_password"
182 + ]
183 + },
184 + {
185 + "title": "Headers",
186 + "fields": [
187 + "headers"
188 + ]
189 + }
190 + ]
191 + }
192 + }
193 +}
src/go/plugin/go.d/modules/typesense/init.go new
+3
@@ -0,0 +1,3 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package typesense
src/go/plugin/go.d/modules/typesense/metadata.yaml new
+222
@@ -0,0 +1,222 @@
1 +plugin_name: go.d.plugin
2 +modules:
3 + - meta:
4 + id: collector-go.d.plugin-typesense
5 + plugin_name: go.d.plugin
6 + module_name: typesense
7 + monitored_instance:
8 + name: NGINX
9 + link: https://typesense.org/
10 + categories:
11 + - data-collection.search-engines
12 + icon_filename: typesense.svg
13 + related_resources:
14 + integrations:
15 + list: []
16 + alternative_monitored_instances: []
17 + info_provided_to_referring_integrations:
18 + description: ""
19 + keywords:
20 + - typesense
21 + - search engine
22 + most_popular: false
23 + overview:
24 + data_collection:
25 + metrics_description: |
26 + This collector monitors the overall health status and performance of your Typesense servers.
27 + It gathers detailed metrics, including the total number of requests processed, the breakdown of different request types, and the average latency experienced by each request.
28 + method_description: |
29 + It gathers metrics by periodically issuing HTTP GET requests to the Typesense server:
30 +
31 + - [/health](https://typesense.org/docs/27.0/api/cluster-operations.html#health) endpoint to check server health.
32 + - [/stats.json](https://typesense.org/docs/27.0/api/cluster-operations.html#api-stats) endpoint to collect data on requests and latency.
33 + default_behavior:
34 + auto_detection:
35 + description: |
36 + The collector can automatically detect Typesense instances running on:
37 +
38 + - localhost that are listening on port 8108
39 + - within Docker containers
40 + limits:
41 + description: ""
42 + performance_impact:
43 + description: ""
44 + additional_permissions:
45 + description: ""
46 + multi_instance: true
47 + supported_platforms:
48 + include: []
49 + exclude: []
50 + setup:
51 + prerequisites:
52 + list:
53 + - title: API Key Configuration
54 + description: |
55 + While optional, configuring an [API key](https://typesense.org/docs/0.20.0/api/api-keys.html#api-keys) is highly recommended to enable the collector to gather [stats metrics](https://typesense.org/docs/27.0/api/cluster-operations.html#api-stats), including request counts and latency.
56 + Without an API key, the collector will only collect health status information.
57 +
58 + > If you're running Typesense with the API key provided as a command-line parameter (e.g., `--api-key=XYZ`), Netdata can automatically detect and use this key for queries.
59 + > In this case, no additional configuration is required.
60 + configuration:
61 + file:
62 + name: go.d/typesense.conf
63 + options:
64 + description: |
65 + The following options can be defined globally: update_every, autodetection_retry.
66 + folding:
67 + title: Config options
68 + enabled: true
69 + list:
70 + - name: update_every
71 + description: Data collection frequency.
72 + default_value: 1
73 + required: false
74 + - name: autodetection_retry
75 + description: Recheck interval in seconds. Zero means no recheck will be scheduled.
76 + default_value: 0
77 + required: false
78 + - name: url
79 + description: Server URL.
80 + default_value: http://127.0.0.1:8108
81 + required: true
82 + - name: timeout
83 + description: HTTP request timeout.
84 + default_value: 1
85 + required: false
86 + - name: api_key
87 + description: "The Typesense [API Key](https://typesense.org/docs/0.20.0/api/api-keys.html#api-keys) (`X-TYPESENSE-API-KEY`)."
88 + default_value: ""
89 + required: false
90 + - name: username
91 + description: Username for basic HTTP authentication.
92 + default_value: ""
93 + required: false
94 + - name: password
95 + description: Password for basic HTTP authentication.
96 + default_value: ""
97 + required: false
98 + - name: proxy_url
99 + description: Proxy URL.
100 + default_value: ""
101 + required: false
102 + - name: proxy_username
103 + description: Username for proxy basic HTTP authentication.
104 + default_value: ""
105 + required: false
106 + - name: proxy_password
107 + description: Password for proxy basic HTTP authentication.
108 + default_value: ""
109 + required: false
110 + - name: method
111 + description: HTTP request method.
112 + default_value: GET
113 + required: false
114 + - name: body
115 + description: HTTP request body.
116 + default_value: ""
117 + required: false
118 + - name: headers
119 + description: HTTP request headers.
120 + default_value: ""
121 + required: false
122 + - name: not_follow_redirects
123 + description: Redirect handling policy. Controls whether the client follows redirects.
124 + default_value: false
125 + required: false
126 + - name: tls_skip_verify
127 + description: Server certificate chain and hostname validation policy. Controls whether the client performs this check.
128 + default_value: false
129 + required: false
130 + - name: tls_ca
131 + description: Certification authority that the client uses when verifying the server's certificates.
132 + default_value: ""
133 + required: false
134 + - name: tls_cert
135 + description: Client TLS certificate.
136 + default_value: ""
137 + required: false
138 + - name: tls_key
139 + description: Client TLS key.
140 + default_value: ""
141 + required: false
142 + examples:
143 + folding:
144 + title: Config
145 + enabled: true
146 + list:
147 + - name: Basic
148 + description: A basic example configuration.
149 + folding:
150 + enabled: false
151 + config: |
152 + jobs:
153 + - name: local
154 + url: http://127.0.0.1:8108
155 + api_key: XYZ
156 + - name: Multi-instance
157 + description: |
158 + > **Note**: When you define multiple jobs, their names must be unique.
159 +
160 + Collecting metrics from local and remote instances.
161 + config: |
162 + jobs:
163 + - name: local
164 + url: http://127.0.0.1:8108
165 + api_key: XYZ
166 +
167 + - name: remote
168 + url: http://192.0.2.1:8108
169 + api_key: XYZ
170 + troubleshooting:
171 + problems:
172 + list: []
173 + alerts: []
174 + metrics:
175 + folding:
176 + title: Metrics
177 + enabled: false
178 + description: ""
179 + availability: []
180 + scopes:
181 + - name: global
182 + description: These metrics refer to the entire monitored application.
183 + labels: []
184 + metrics:
185 + - name: typesense.health_status
186 + description: Health Status
187 + unit: status
188 + chart_type: line
189 + dimensions:
190 + - name: ok
191 + - name: out_of_disk
192 + - name: out_of_memory
193 + - name: typesense.total_requests
194 + description: Total Requests
195 + unit: requests/s
196 + chart_type: line
197 + dimensions:
198 + - name: requests
199 + - name: typesense.requests_by_operation
200 + description: Requests by Operation
201 + unit: requests/s
202 + chart_type: line
203 + dimensions:
204 + - name: search
205 + - name: write
206 + - name: import
207 + - name: delete
208 + - name: typesense.latency_by_operation
209 + description: Latency by Operation
210 + unit: milliseconds
211 + chart_type: line
212 + dimensions:
213 + - name: search
214 + - name: write
215 + - name: import
216 + - name: delete
217 + - name: typesense.overloaded_requests
218 + description: Overloaded Requests
219 + unit: requests/s
220 + chart_type: line
221 + dimensions:
222 + - name: overloaded
src/go/plugin/go.d/modules/typesense/testdata/config.json new
+21
@@ -0,0 +1,21 @@
1 +{
2 + "update_every": 123,
3 + "url": "ok",
4 + "api_key": "ok",
5 + "body": "ok",
6 + "method": "ok",
7 + "headers": {
8 + "ok": "ok"
9 + },
10 + "username": "ok",
11 + "password": "ok",
12 + "proxy_url": "ok",
13 + "proxy_username": "ok",
14 + "proxy_password": "ok",
15 + "timeout": 123.123,
16 + "not_follow_redirects": true,
17 + "tls_ca": "ok",
18 + "tls_cert": "ok",
19 + "tls_key": "ok",
20 + "tls_skip_verify": true
21 +}
src/go/plugin/go.d/modules/typesense/testdata/config.yaml new
+18
@@ -0,0 +1,18 @@
1 +update_every: 123
2 +url: "ok"
3 +api_key: "ok"
4 +body: "ok"
5 +method: "ok"
6 +headers:
7 + ok: "ok"
8 +username: "ok"
9 +password: "ok"
10 +proxy_url: "ok"
11 +proxy_username: "ok"
12 +proxy_password: "ok"
13 +timeout: 123.123
14 +not_follow_redirects: yes
15 +tls_ca: "ok"
16 +tls_cert: "ok"
17 +tls_key: "ok"
18 +tls_skip_verify: yes
src/go/plugin/go.d/modules/typesense/testdata/v27.0/health_nok.json new
+4
@@ -0,0 +1,4 @@
1 +{
2 + "ok": false,
3 + "resource_error": "OUT_OF_DISK"
4 +}
src/go/plugin/go.d/modules/typesense/testdata/v27.0/health_ok.json new
+3
@@ -0,0 +1,3 @@
1 +{
2 + "ok": true
3 +}
src/go/plugin/go.d/modules/typesense/testdata/v27.0/stats.json new
+15
@@ -0,0 +1,15 @@
1 +{
2 + "delete_latency_ms": 1,
3 + "delete_requests_per_second": 1.1,
4 + "import_latency_ms": 1,
5 + "import_requests_per_second": 1.1,
6 + "latency_ms": {},
7 + "overloaded_requests_per_second": 1.1,
8 + "pending_write_batches": 1,
9 + "requests_per_second": {},
10 + "search_latency_ms": 1,
11 + "search_requests_per_second": 1.1,
12 + "total_requests_per_second": 1.1,
13 + "write_latency_ms": 1,
14 + "write_requests_per_second": 1.1
15 +}
src/go/plugin/go.d/modules/typesense/typesense.go new
+122
@@ -0,0 +1,122 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package typesense
4 +
5 +import (
6 + _ "embed"
7 + "errors"
8 + "fmt"
9 + "net/http"
10 + "sync"
11 + "time"
12 +
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"
15 +)
16 +
17 +//go:embed "config_schema.json"
18 +var configSchema string
19 +
20 +func init() {
21 + module.Register("typesense", module.Creator{
22 + Create: func() module.Module { return New() },
23 + JobConfigSchema: configSchema,
24 + Config: func() any { return &Config{} },
25 + })
26 +}
27 +
28 +func New() *Typesense {
29 + return &Typesense{
30 + Config: Config{
31 + HTTP: web.HTTP{
32 + Request: web.Request{
33 + URL: "http://127.0.0.1:8108",
34 + },
35 + Client: web.Client{
36 + Timeout: web.Duration(time.Second),
37 + },
38 + },
39 + },
40 + charts: baseCharts.Copy(),
41 + doStats: true,
42 + }
43 +}
44 +
45 +type Config struct {
46 + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
47 + web.HTTP `yaml:",inline" json:""`
48 + APIKey string `yaml:"api_key,omitempty" json:"api_key"`
49 +}
50 +
51 +type Typesense struct {
52 + module.Base
53 + Config `yaml:",inline" json:""`
54 +
55 + charts *module.Charts
56 + once sync.Once
57 +
58 + httpClient *http.Client
59 +
60 + doStats bool
61 +}
62 +
63 +func (ts *Typesense) Configuration() any {
64 + return ts.Config
65 +}
66 +
67 +func (ts *Typesense) Init() error {
68 + if ts.URL == "" {
69 + ts.Error("typesense URL not configured")
70 + return errors.New("typesense URL not configured")
71 + }
72 +
73 + httpClient, err := web.NewHTTPClient(ts.Client)
74 + if err != nil {
75 + return fmt.Errorf("initialize http client: %w", err)
76 + }
77 +
78 + ts.httpClient = httpClient
79 +
80 + if ts.APIKey == "" {
81 + ts.Warning("API key not set in configuration. Only health status will be collected.")
82 + }
83 + ts.Debugf("using URL %s", ts.URL)
84 + ts.Debugf("using timeout: %s", ts.Timeout)
85 +
86 + return nil
87 +}
88 +
89 +func (ts *Typesense) Check() error {
90 + mx, err := ts.collect()
91 + if err != nil {
92 + ts.Error(err)
93 + return err
94 + }
95 + if len(mx) == 0 {
96 + return errors.New("no metrics collected")
97 +
98 + }
99 + return nil
100 +}
101 +
102 +func (ts *Typesense) Charts() *module.Charts {
103 + return ts.charts
104 +}
105 +
106 +func (ts *Typesense) Collect() map[string]int64 {
107 + mx, err := ts.collect()
108 + if err != nil {
109 + ts.Error(err)
110 + }
111 +
112 + if len(mx) == 0 {
113 + return nil
114 + }
115 + return mx
116 +}
117 +
118 +func (ts *Typesense) Cleanup() {
119 + if ts.httpClient != nil {
120 + ts.httpClient.CloseIdleConnections()
121 + }
122 +}
src/go/plugin/go.d/modules/typesense/typesense_test.go new
+318
@@ -0,0 +1,318 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package typesense
4 +
5 +import (
6 + "net/http"
7 + "net/http/httptest"
8 + "os"
9 + "testing"
10 +
11 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"
13 +
14 + "github.com/stretchr/testify/assert"
15 + "github.com/stretchr/testify/require"
16 +)
17 +
18 +const testApiKey = "XYZ"
19 +
20 +var (
21 + dataConfigJSON, _ = os.ReadFile("testdata/config.json")
22 + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")
23 +
24 + dataVer27HealthOk, _ = os.ReadFile("testdata/v27.0/health_ok.json")
25 + dataVer27HealthNok, _ = os.ReadFile("testdata/v27.0/health_nok.json")
26 + dataVer27Stats, _ = os.ReadFile("testdata/v27.0/stats.json")
27 +)
28 +
29 +func Test_testDataIsValid(t *testing.T) {
30 + for name, data := range map[string][]byte{
31 + "dataConfigJSON": dataConfigJSON,
32 + "dataConfigYAML": dataConfigYAML,
33 + "dataVer27HealthOk": dataVer27HealthOk,
34 + "dataVer27HealthNok": dataVer27HealthNok,
35 + "dataVer27Stats": dataVer27Stats,
36 + } {
37 + require.NotNil(t, data, name)
38 +
39 + }
40 +}
41 +
42 +func TestTypesense_ConfigurationSerialize(t *testing.T) {
43 + module.TestConfigurationSerialize(t, &Typesense{}, dataConfigJSON, dataConfigYAML)
44 +}
45 +
46 +func TestTypesense_Init(t *testing.T) {
47 + tests := map[string]struct {
48 + wantFail bool
49 + config Config
50 + }{
51 + "success with default": {
52 + wantFail: false,
53 + config: New().Config,
54 + },
55 + "fail when URL not set": {
56 + wantFail: true,
57 + config: Config{
58 + HTTP: web.HTTP{
59 + Request: web.Request{URL: ""},
60 + },
61 + },
62 + },
63 + }
64 +
65 + for name, test := range tests {
66 + t.Run(name, func(t *testing.T) {
67 + ts := New()
68 + ts.Config = test.config
69 +
70 + if test.wantFail {
71 + assert.Error(t, ts.Init())
72 + } else {
73 + assert.NoError(t, ts.Init())
74 + }
75 + })
76 + }
77 +}
78 +
79 +func TestTypesense_Check(t *testing.T) {
80 + tests := map[string]struct {
81 + wantFail bool
82 + prepare func(t *testing.T) (ts *Typesense, cleanup func())
83 + }{
84 + "success with valid API key": {
85 + wantFail: false,
86 + prepare: caseOk,
87 + },
88 + "success without API key": {
89 + wantFail: false,
90 + prepare: caseOkNoApiKey,
91 + },
92 + "fail on unexpected JSON response": {
93 + wantFail: true,
94 + prepare: caseUnexpectedJsonResponse,
95 + },
96 + "fail on invalid data response": {
97 + wantFail: true,
98 + prepare: caseInvalidDataResponse,
99 + },
100 + "fail on connection refused": {
101 + wantFail: true,
102 + prepare: caseConnectionRefused,
103 + },
104 + "fail on 404 response": {
105 + wantFail: true,
106 + prepare: case404,
107 + },
108 + }
109 +
110 + for name, test := range tests {
111 + t.Run(name, func(t *testing.T) {
112 + ts, cleanup := test.prepare(t)
113 + defer cleanup()
114 +
115 + if test.wantFail {
116 + assert.Error(t, ts.Check())
117 + } else {
118 + assert.NoError(t, ts.Check())
119 + }
120 + })
121 + }
122 +}
123 +
124 +func TestTypesense_Charts(t *testing.T) {
125 + assert.NotNil(t, New().Charts())
126 +}
127 +
128 +func TestTypesense_Collect(t *testing.T) {
129 + tests := map[string]struct {
130 + prepare func(t *testing.T) (ts *Typesense, cleanup func())
131 + wantNumOfCharts int
132 + wantMetrics map[string]int64
133 + }{
134 + "success with valid API key": {
135 + prepare: caseOk,
136 + wantNumOfCharts: len(baseCharts) + len(statsCharts),
137 + wantMetrics: map[string]int64{
138 + "delete_latency_ms": 1,
139 + "delete_requests_per_second": 1100,
140 + "health_status_ok": 1,
141 + "health_status_out_of_disk": 0,
142 + "health_status_out_of_memory": 0,
143 + "import_latency_ms": 1,
144 + "import_requests_per_second": 1100,
145 + "overloaded_requests_per_second": 1100,
146 + "pending_write_batches": 1,
147 + "search_latency_ms": 1,
148 + "search_requests_per_second": 1100,
149 + "total_requests_per_second": 1100,
150 + "write_latency_ms": 1,
151 + "write_requests_per_second": 1100,
152 + },
153 + },
154 + "success without API key": {
155 + prepare: caseOkNoApiKey,
156 + wantNumOfCharts: len(baseCharts),
157 + wantMetrics: map[string]int64{
158 + "health_status_ok": 0,
159 + "health_status_out_of_disk": 1,
160 + "health_status_out_of_memory": 0,
161 + },
162 + },
163 + "fail on unexpected JSON response": {
164 + prepare: caseUnexpectedJsonResponse,
165 + wantMetrics: nil,
166 + },
167 + "fail on invalid data response": {
168 + prepare: caseInvalidDataResponse,
169 + wantMetrics: nil,
170 + },
171 + "fail on connection refused": {
172 + prepare: caseConnectionRefused,
173 + wantMetrics: nil,
174 + },
175 + "fail on 404 response": {
176 + prepare: case404,
177 + wantMetrics: nil,
178 + },
179 + }
180 +
181 + for name, test := range tests {
182 + t.Run(name, func(t *testing.T) {
183 + ts, cleanup := test.prepare(t)
184 + defer cleanup()
185 +
186 + _ = ts.Check()
187 +
188 + mx := ts.Collect()
189 +
190 + require.Equal(t, test.wantMetrics, mx)
191 +
192 + if len(test.wantMetrics) > 0 {
193 + assert.Equal(t, test.wantNumOfCharts, len(*ts.Charts()), "want charts")
194 +
195 + module.TestMetricsHasAllChartsDims(t, ts.Charts(), mx)
196 + }
197 + })
198 + }
199 +}
200 +
201 +func caseOk(t *testing.T) (*Typesense, func()) {
202 + t.Helper()
203 + srv := httptest.NewServer(http.HandlerFunc(
204 + func(w http.ResponseWriter, r *http.Request) {
205 + switch r.URL.Path {
206 + case urlPathHealth:
207 + _, _ = w.Write(dataVer27HealthOk)
208 + case urlPathStats:
209 + if r.Header.Get("X-TYPESENSE-API-KEY") != testApiKey {
210 + msg := "{\"message\": \"Forbidden - a valid `x-typesense-api-key` header must be sent.\"}"
211 + _, _ = w.Write([]byte(msg))
212 + w.WriteHeader(http.StatusUnauthorized)
213 + } else {
214 + _, _ = w.Write(dataVer27Stats)
215 + }
216 + default:
217 + w.WriteHeader(http.StatusNotFound)
218 + }
219 + }))
220 + ts := New()
221 + ts.URL = srv.URL
222 + ts.APIKey = testApiKey
223 + require.NoError(t, ts.Init())
224 +
225 + return ts, srv.Close
226 +}
227 +
228 +func caseOkNoApiKey(t *testing.T) (*Typesense, func()) {
229 + t.Helper()
230 + srv := httptest.NewServer(http.HandlerFunc(
231 + func(w http.ResponseWriter, r *http.Request) {
232 + switch r.URL.Path {
233 + case urlPathHealth:
234 + _, _ = w.Write(dataVer27HealthNok)
235 + case urlPathStats:
236 + if r.Header.Get("X-TYPESENSE-API-KEY") != testApiKey {
237 + msg := "{\"message\": \"Forbidden - a valid `x-typesense-api-key` header must be sent.\"}"
238 + _, _ = w.Write([]byte(msg))
239 + w.WriteHeader(http.StatusUnauthorized)
240 + } else {
241 + _, _ = w.Write(dataVer27Stats)
242 + }
243 + default:
244 + w.WriteHeader(http.StatusNotFound)
245 + }
246 + }))
247 + ts := New()
248 + ts.URL = srv.URL
249 + ts.APIKey = ""
250 + require.NoError(t, ts.Init())
251 +
252 + return ts, srv.Close
253 +}
254 +
255 +func caseUnexpectedJsonResponse(t *testing.T) (*Typesense, func()) {
256 + t.Helper()
257 + resp := `
258 +{
259 + "elephant": {
260 + "burn": false,
261 + "mountain": true,
262 + "fog": false,
263 + "skin": -1561907625,
264 + "burst": "anyway",
265 + "shadow": 1558616893
266 + },
267 + "start": "ever",
268 + "base": 2093056027,
269 + "mission": -2007590351,
270 + "victory": 999053756,
271 + "die": false
272 +}
273 +`
274 + srv := httptest.NewServer(http.HandlerFunc(
275 + func(w http.ResponseWriter, r *http.Request) {
276 + _, _ = w.Write([]byte(resp))
277 + }))
278 + ts := New()
279 + ts.URL = srv.URL
280 + require.NoError(t, ts.Init())
281 +
282 + return ts, srv.Close
283 +}
284 +
285 +func caseInvalidDataResponse(t *testing.T) (*Typesense, func()) {
286 + t.Helper()
287 + srv := httptest.NewServer(http.HandlerFunc(
288 + func(w http.ResponseWriter, r *http.Request) {
289 + _, _ = w.Write([]byte("hello and\n goodbye"))
290 + }))
291 + ts := New()
292 + ts.URL = srv.URL
293 + require.NoError(t, ts.Init())
294 +
295 + return ts, srv.Close
296 +}
297 +
298 +func caseConnectionRefused(t *testing.T) (*Typesense, func()) {
299 + t.Helper()
300 + ts := New()
301 + ts.URL = "http://127.0.0.1:65001"
302 + require.NoError(t, ts.Init())
303 +
304 + return ts, func() {}
305 +}
306 +
307 +func case404(t *testing.T) (*Typesense, func()) {
308 + t.Helper()
309 + srv := httptest.NewServer(http.HandlerFunc(
310 + func(w http.ResponseWriter, r *http.Request) {
311 + w.WriteHeader(http.StatusNotFound)
312 + }))
313 + ts := New()
314 + ts.URL = srv.URL
315 + require.NoError(t, ts.Init())
316 +
317 + return ts, srv.Close
318 +}