@cryptotaxi247 / netdata-1 / commits / b53fb3699

feat(go.d/sd): Add HTTP service discovery (#22256)

Ilya Mashchenko committed Apr 23, 2026 at 18:03 UTC b53fb3699d0bca2ee0c6d75e44ec8b75ac833cfb
15 files changed +1665
docs/fleet-configuration-management.md
+3
@@ -193,8 +193,11 @@ The go.d.plugin provides auto-discovery for 150+ applications through multiple m
193 - **[Docker container discovery](https://learn.netdata.cloud/docs/collecting-metrics/container-services/docker) (dockersd)** - Discovers applications running in Docker containers
194 - **[Kubernetes service discovery](https://learn.netdata.cloud/docs/netdata-agent/installation/kubernetes) (k8ssd)** - Discovers services running in Kubernetes pods
195 - **[SNMP device discovery](https://learn.netdata.cloud/docs/collecting-metrics/network-devices/snmp) (snmpsd)** - Discovers and profiles SNMP-enabled network devices
196 +- **HTTP service discovery (http)** - Fetches JSON or YAML discovery items from an HTTP endpoint and creates collector jobs through service templates
197 - **Configuration file scanning** - Detects applications based on their configuration files
198
199 +HTTP service discovery templates must generate job configs with `name` and `module`. The `module` is the collector name, such as `httpcheck` or `ping`; it can be omitted only in curated rules where the service rule ID is the intended collector module.
200 +
201 **Platform-specific behavior**:
202
203 **Linux systems (non-Kubernetes)**:
src/go/plugin/agent/discovery/sd/pipeline/funcmap.go
+5
@@ -12,6 +12,7 @@ import (
12
13 "github.com/Masterminds/sprig/v3"
14 "github.com/bmatcuk/doublestar/v4"
15 + "gopkg.in/yaml.v2"
16 )
17
18 func newFuncMap() template.FuncMap {
@@ -26,6 +27,10 @@ func newFuncMap() template.FuncMap {
27 v, _ := strconv.Atoi(port)
28 return prometheusPortAllocations[v]
29 },
30 + "toYaml": func(v any) (string, error) {
31 + bs, err := yaml.Marshal(v)
32 + return string(bs), err
33 + },
34 }
35
36 maps.Copy(fm, extra)
src/go/plugin/agent/discovery/sd/pipeline/services_test.go
+99
@@ -119,3 +119,102 @@ func TestServiceEngine_compose(t *testing.T) {
119 })
120 }
121 }
122 +
123 +func TestServiceEngine_composeHTTPItems(t *testing.T) {
124 + tests := map[string]struct {
125 + configYAML string
126 + target model.Target
127 + wantConfigs []confgroup.Config
128 + }{
129 + "full job pass-through preserves module": {
130 + configYAML: `
131 +- id: "passthrough"
132 + match: '{{ true }}'
133 + config_template: |
134 + {{ .Item | toYaml }}
135 +`,
136 + target: &itemTarget{Item: map[string]any{
137 + "module": "nginx",
138 + "name": "local",
139 + "url": "http://127.0.0.1/stub_status",
140 + }},
141 + wantConfigs: []confgroup.Config{
142 + {"module": "nginx", "name": "local", "url": "http://127.0.0.1/stub_status"},
143 + },
144 + },
145 + "full job pass-through preserves numeric fields": {
146 + configYAML: `
147 +- id: "passthrough"
148 + match: '{{ true }}'
149 + config_template: |
150 + {{ .Item | toYaml }}
151 +`,
152 + target: &itemTarget{Item: map[string]any{
153 + "module": "httpcheck",
154 + "name": "api",
155 + "url": "http://127.0.0.1/health",
156 + "port": float64(80),
157 + }},
158 + wantConfigs: []confgroup.Config{
159 + {"module": "httpcheck", "name": "api", "url": "http://127.0.0.1/health", "port": 80},
160 + },
161 + },
162 + "endpoint object fills module from rule id": {
163 + configYAML: `
164 +- id: "httpcheck"
165 + match: '{{ hasKey .Item "url" }}'
166 + config_template: |
167 + name: {{ .Item.name }}
168 + url: {{ .Item.url }}
169 +`,
170 + target: &itemTarget{Item: map[string]any{
171 + "name": "api",
172 + "url": "http://127.0.0.1/health",
173 + }},
174 + wantConfigs: []confgroup.Config{
175 + {"module": "httpcheck", "name": "api", "url": "http://127.0.0.1/health"},
176 + },
177 + },
178 + "scalar endpoint string uses TUID": {
179 + configYAML: `
180 +- id: "httpcheck"
181 + match: '{{ kindIs "string" .Item }}'
182 + config_template: |
183 + name: {{ .TUID }}
184 + url: {{ .Item }}
185 +`,
186 + target: &itemTarget{Item: "http://127.0.0.1/health", tuid: "http_item_1"},
187 + wantConfigs: []confgroup.Config{
188 + {"module": "httpcheck", "name": "http_item_1", "url": "http://127.0.0.1/health"},
189 + },
190 + },
191 + }
192 +
193 + for name, test := range tests {
194 + t.Run(name, func(t *testing.T) {
195 + var cfg []ServiceRuleConfig
196 + err := yaml.Unmarshal([]byte(test.configYAML), &cfg)
197 + require.NoErrorf(t, err, "yaml unmarshalling of services config")
198 +
199 + svr, err := newServiceEngine(cfg)
200 + require.NoErrorf(t, err, "service engine creation")
201 +
202 + assert.Equal(t, test.wantConfigs, svr.compose(test.target))
203 + })
204 + }
205 +}
206 +
207 +type itemTarget struct {
208 + model.Base
209 + Item any
210 + tuid string
211 +}
212 +
213 +func (t itemTarget) TUID() string {
214 + if t.tuid != "" {
215 + return t.tuid
216 + }
217 + return "item"
218 +}
219 +
220 +func (t itemTarget) Hash() uint64 { return 1 }
src/go/plugin/go.d/config/go.d/sd/http.conf new
+73
@@ -0,0 +1,73 @@
1 +## ===================================================================
2 +## WARNING: HTTP DISCOVERY IS DISABLED BY DEFAULT
3 +## To enable, change "disabled: yes" to "disabled: no" below
4 +## AND configure the endpoint and service rules for your environment.
5 +## ===================================================================
6 +
7 +disabled: yes
8 +
9 +discoverer:
10 + http:
11 + ## HTTP endpoint that returns either:
12 + ## - a bare array: [ ... ]
13 + ## - an envelope: { "items": [ ... ] }
14 + #url: "https://example.com/netdata/go.d/jobs.yaml"
15 +
16 + ## How often to fetch the endpoint (default: 1m).
17 + ## Set to 0 for a one-shot fetch. One-shot mode fetches once when this
18 + ## SD pipeline starts; it does not refetch on SD reload unless the pipeline
19 + ## itself is recreated.
20 + #interval: "1m"
21 +
22 + ## Response format: auto, json, or yaml (default: auto).
23 + ## Auto uses Content-Type when clear, then tries JSON before YAML.
24 + #format: auto
25 +
26 + ## Standard HTTP options from go.d collectors are supported.
27 + #headers:
28 + # Accept: application/yaml
29 + #username: ""
30 + #password: ""
31 +
32 + ## If bearer_token_file points under /var/run/secrets/ and Netdata is not
33 + ## running in Kubernetes, missing token files are ignored.
34 + #bearer_token_file: ""
35 +
36 + #timeout: "2s"
37 + #proxy_url: ""
38 + #tls_skip_verify: no
39 +
40 +services:
41 + ## Full job pass-through. The fetched item must already be a go.d job config.
42 + ## Pass-through items must include both name and module. The module is the
43 + ## collector name, for example httpcheck, ping, or nginx.
44 + ## `toYaml` serializes the decoded item to YAML for config_template parsing.
45 + ## `.Item` is remote response data; `.TUID` and `.Hash` are target methods and
46 + ## do not interact with same-named keys inside `.Item`.
47 + - id: "passthrough"
48 + match: '{{ true }}'
49 + config_template: |
50 + {{ .Item | toYaml }}
51 +
52 + ## Example for an array of endpoint strings:
53 + ## items:
54 + ## - "https://example.com/health"
55 + ##
56 + ## - id: "httpcheck"
57 + ## # Because id is httpcheck, module may be omitted from this curated rule.
58 + ## match: '{{ kindIs "string" .Item }}'
59 + ## config_template: |
60 + ## name: {{ .TUID }}
61 + ## url: {{ .Item }}
62 +
63 + ## Example for an array of endpoint objects:
64 + ## items:
65 + ## - name: "api"
66 + ## url: "https://api.example.com/health"
67 + ##
68 + ## - id: "httpcheck"
69 + ## # Because id is httpcheck, module may be omitted from this curated rule.
70 + ## match: '{{ and (kindIs "map" .Item) (hasKey .Item "url") }}'
71 + ## config_template: |
72 + ## name: {{ .Item.name }}
73 + ## url: {{ .Item.url }}
src/go/plugin/go.d/discovery/sdext/config_schema_http.json new
+286
@@ -0,0 +1,286 @@
1 +{
2 + "jsonSchema": {
3 + "$schema": "http://json-schema.org/draft-07/schema#",
4 + "title": "HTTP Service Discovery",
5 + "description": "Fetches JSON or YAML service discovery items from an HTTP endpoint.",
6 + "type": "object",
7 + "properties": {
8 + "discoverer": {
9 + "title": "Discoverer",
10 + "type": "object",
11 + "properties": {
12 + "http": {
13 + "title": "HTTP",
14 + "type": "object",
15 + "properties": {
16 + "url": {
17 + "title": "URL",
18 + "description": "HTTP endpoint that returns a bare array or an object with an items array.",
19 + "type": "string",
20 + "format": "uri"
21 + },
22 + "method": {
23 + "title": "Method",
24 + "description": "HTTP method used to fetch discovery items.",
25 + "type": "string"
26 + },
27 + "headers": {
28 + "title": "Headers",
29 + "description": "HTTP request headers.",
30 + "type": "object",
31 + "additionalProperties": {
32 + "type": "string"
33 + }
34 + },
35 + "body": {
36 + "title": "Body",
37 + "description": "Optional HTTP request body.",
38 + "type": "string"
39 + },
40 + "username": {
41 + "title": "Username",
42 + "description": "Basic authentication username.",
43 + "type": "string"
44 + },
45 + "password": {
46 + "title": "Password",
47 + "description": "Basic authentication password.",
48 + "type": "string"
49 + },
50 + "bearer_token_file": {
51 + "title": "Bearer token file",
52 + "description": "Path to a file containing a bearer token. The file is read for each request.",
53 + "type": "string"
54 + },
55 + "interval": {
56 + "title": "Poll interval",
57 + "description": "How often to fetch the endpoint. Set to 0 for a one-shot fetch.",
58 + "type": "string",
59 + "pattern": "^[0-9]+(\\.[0-9]+)?(ms|s|m|h|d|w|mo|y)?$",
60 + "default": "1m"
61 + },
62 + "timeout": {
63 + "title": "Timeout",
64 + "description": "HTTP request timeout, in seconds.",
65 + "type": "number",
66 + "minimum": 0.1,
67 + "default": 2
68 + },
69 + "format": {
70 + "title": "Response format",
71 + "description": "Response format. Auto uses Content-Type when clear, then tries JSON before YAML.",
72 + "type": "string",
73 + "enum": [
74 + "auto",
75 + "json",
76 + "yaml"
77 + ],
78 + "default": "auto"
79 + },
80 + "not_follow_redirects": {
81 + "title": "Do not follow redirects",
82 + "type": "boolean"
83 + },
84 + "proxy_url": {
85 + "title": "Proxy URL",
86 + "type": "string"
87 + },
88 + "proxy_username": {
89 + "title": "Proxy username",
90 + "type": "string"
91 + },
92 + "proxy_password": {
93 + "title": "Proxy password",
94 + "type": "string"
95 + },
96 + "tls_ca": {
97 + "title": "TLS CA",
98 + "description": "Path to CA certificate used to verify the endpoint.",
99 + "type": "string"
100 + },
101 + "tls_cert": {
102 + "title": "TLS certificate",
103 + "description": "Path to client TLS certificate.",
104 + "type": "string"
105 + },
106 + "tls_key": {
107 + "title": "TLS key",
108 + "description": "Path to client TLS key.",
109 + "type": "string"
110 + },
111 + "tls_skip_verify": {
112 + "title": "Skip TLS verification",
113 + "type": "boolean"
114 + },
115 + "force_http2": {
116 + "title": "Force HTTP/2",
117 + "type": "boolean"
118 + }
119 + },
120 + "required": [
121 + "url"
122 + ]
123 + }
124 + },
125 + "required": [
126 + "http"
127 + ]
128 + },
129 + "services": {
130 + "title": "Service rules",
131 + "description": "- Match fetched HTTP discovery items and generate collector configurations.\n- Generated job configs must include `name` and `module`; `module` is the collector name, for example `httpcheck` or `ping`.\n- If a generated config omits `module`, the rule ID is used as `module`; rely on this only for curated rules where the rule ID is the intended collector.\n- `.Item` contains the fetched item. String items expose the string as `.Item`; object items expose fields under `.Item`.",
132 + "type": "array",
133 + "minItems": 1,
134 + "items": {
135 + "type": "object",
136 + "properties": {
137 + "id": {
138 + "title": "Rule ID",
139 + "description": "Unique identifier for this rule. If config_template omits module, this value is used as the module. Use that fallback only when the rule ID is the intended collector name.",
140 + "type": "string"
141 + },
142 + "match": {
143 + "title": "Match expression",
144 + "description": "Go template expression that must evaluate to 'true' for the rule to match the fetched item.",
145 + "type": "string"
146 + },
147 + "config_template": {
148 + "title": "Config template",
149 + "description": "Go template that generates the data collection job configuration in YAML format. Generated configs must include name and module unless module is intentionally filled from the rule ID.",
150 + "type": "string"
151 + }
152 + },
153 + "required": [
154 + "id",
155 + "match"
156 + ]
157 + }
158 + }
159 + },
160 + "required": [
161 + "discoverer",
162 + "services"
163 + ]
164 + },
165 + "uiSchema": {
166 + "uiOptions": {
167 + "fullPage": true
168 + },
169 + "ui:flavour": "tabs",
170 + "ui:options": {
171 + "tabs": [
172 + {
173 + "title": "Discovery",
174 + "fields": [
175 + "discoverer"
176 + ]
177 + },
178 + {
179 + "title": "Services",
180 + "fields": [
181 + "services"
182 + ]
183 + }
184 + ]
185 + },
186 + "discoverer": {
187 + "http": {
188 + "ui:flavour": "tabs",
189 + "ui:options": {
190 + "tabs": [
191 + {
192 + "title": "Request",
193 + "fields": [
194 + "url",
195 + "format",
196 + "not_follow_redirects",
197 + "interval",
198 + "timeout",
199 + "method",
200 + "body"
201 + ]
202 + },
203 + {
204 + "title": "Auth",
205 + "fields": [
206 + "username",
207 + "password",
208 + "bearer_token_file"
209 + ]
210 + },
211 + {
212 + "title": "TLS",
213 + "fields": [
214 + "tls_ca",
215 + "tls_cert",
216 + "tls_key",
217 + "tls_skip_verify"
218 + ]
219 + },
220 + {
221 + "title": "Proxy",
222 + "fields": [
223 + "proxy_url",
224 + "proxy_username",
225 + "proxy_password",
226 + "force_http2"
227 + ]
228 + },
229 + {
230 + "title": "Headers",
231 + "fields": [
232 + "headers"
233 + ]
234 + }
235 + ]
236 + },
237 + "url": {
238 + "ui:placeholder": "https://example.com/netdata/go.d/jobs.yaml"
239 + },
240 + "method": {
241 + "ui:widget": "hidden"
242 + },
243 + "body": {
244 + "ui:widget": "hidden"
245 + },
246 + "interval": {
247 + "ui:placeholder": "1m",
248 + "ui:help": "Examples: 30s, 1m, 5m. Use 0 for one-shot fetch."
249 + },
250 + "format": {
251 + "ui:placeholder": "auto",
252 + "ui:widget": "radio",
253 + "ui:options": {
254 + "inline": true
255 + }
256 + },
257 + "password": {
258 + "ui:widget": "password"
259 + },
260 + "proxy_password": {
261 + "ui:widget": "password"
262 + },
263 + "force_http2": {
264 + "ui:widget": "hidden"
265 + }
266 + }
267 + },
268 + "services": {
269 + "ui:listFlavour": "list",
270 + "items": {
271 + "id": {
272 + "ui:placeholder": "httpcheck"
273 + },
274 + "match": {
275 + "ui:widget": "textarea",
276 + "ui:placeholder": "{{ true }}",
277 + "ui:help": "| Field | Description |\n|-------|-------------|\n| `.Item` | Fetched item. Object fields are under `.Item`, for example `.Item.url`. String items expose the string itself. |\n| `.TUID` | Stable target unique ID. |\n| `.Hash` | Stable content-sensitive target hash. |"
278 + },
279 + "config_template": {
280 + "ui:widget": "textarea",
281 + "ui:placeholder": "name: {{ .TUID }}\nmodule: httpcheck\nurl: {{ .Item }}"
282 + }
283 + }
284 + }
285 + }
286 +}
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/config.go new
+87
@@ -0,0 +1,87 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package httpsd
4 +
5 +import (
6 + "errors"
7 + "fmt"
8 + "net/url"
9 + "strings"
10 + "time"
11 +
12 + "github.com/netdata/netdata/go/plugins/pkg/confopt"
13 + "github.com/netdata/netdata/go/plugins/pkg/web"
14 +)
15 +
16 +const (
17 + defaultInterval = time.Minute
18 + defaultTimeout = 2 * time.Second
19 +
20 + responseBodyLimit = 10 * 1024 * 1024
21 +
22 + formatAuto = "auto"
23 + formatJSON = "json"
24 + formatYAML = "yaml"
25 +)
26 +
27 +type Config struct {
28 + Source string `yaml:"-" json:"-"`
29 +
30 + web.HTTPConfig `yaml:",inline" json:""`
31 +
32 + Interval *confopt.LongDuration `yaml:"interval,omitempty" json:"interval,omitempty"`
33 + Format string `yaml:"format,omitempty" json:"format,omitempty"`
34 +}
35 +
36 +func (c Config) validate() error {
37 + if strings.TrimSpace(c.URL) == "" {
38 + return errors.New("url is required")
39 + }
40 +
41 + u, err := url.Parse(c.URL)
42 + if err != nil {
43 + return fmt.Errorf("invalid url: %w", err)
44 + }
45 + switch u.Scheme {
46 + case "http", "https":
47 + default:
48 + return fmt.Errorf("unsupported url scheme %q", u.Scheme)
49 + }
50 + if u.Host == "" {
51 + return errors.New("url host is required")
52 + }
53 +
54 + switch c.format() {
55 + case formatAuto, formatJSON, formatYAML:
56 + default:
57 + return fmt.Errorf("unsupported format %q", c.Format)
58 + }
59 +
60 + if c.Interval != nil && c.Interval.Duration() < 0 {
61 + return errors.New("interval cannot be negative")
62 + }
63 +
64 + return nil
65 +}
66 +
67 +func (c Config) interval() time.Duration {
68 + if c.Interval == nil {
69 + return defaultInterval
70 + }
71 + return c.Interval.Duration()
72 +}
73 +
74 +func (c Config) clientConfig() web.ClientConfig {
75 + cfg := c.ClientConfig
76 + if cfg.Timeout.Duration() <= 0 {
77 + cfg.Timeout = confopt.Duration(defaultTimeout)
78 + }
79 + return cfg
80 +}
81 +
82 +func (c Config) format() string {
83 + if v := strings.TrimSpace(strings.ToLower(c.Format)); v != "" {
84 + return v
85 + }
86 + return formatAuto
87 +}
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/discoverer.go new
+152
@@ -0,0 +1,152 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package httpsd
4 +
5 +import (
6 + "context"
7 + "errors"
8 + "fmt"
9 + "io"
10 + "log/slog"
11 + "net/http"
12 + "time"
13 +
14 + "github.com/netdata/netdata/go/plugins/logger"
15 + "github.com/netdata/netdata/go/plugins/pkg/web"
16 + "github.com/netdata/netdata/go/plugins/plugin/agent/discovery/sd/model"
17 +)
18 +
19 +const (
20 + shortName = "http"
21 + fullName = "sd:http"
22 +)
23 +
24 +func NewDiscoverer(cfg Config) (*Discoverer, error) {
25 + if err := cfg.validate(); err != nil {
26 + return nil, err
27 + }
28 +
29 + client, err := web.NewHTTPClient(cfg.clientConfig())
30 + if err != nil {
31 + return nil, err
32 + }
33 +
34 + d := &Discoverer{
35 + Logger: logger.New().With(
36 + slog.String("component", "service discovery"),
37 + slog.String("discoverer", shortName),
38 + ),
39 + client: client,
40 + request: cfg.RequestConfig,
41 + interval: cfg.interval(),
42 + parser: responseParser{format: cfg.format()},
43 + source: sourceString(cfg),
44 + }
45 +
46 + return d, nil
47 +}
48 +
49 +type Discoverer struct {
50 + *logger.Logger
51 + model.Base
52 +
53 + client *http.Client
54 + request web.RequestConfig
55 +
56 + interval time.Duration
57 + parser responseParser
58 + source string
59 +}
60 +
61 +func (d *Discoverer) String() string {
62 + return fullName
63 +}
64 +
65 +func (d *Discoverer) Discover(ctx context.Context, in chan<- []model.TargetGroup) {
66 + d.Info("instance is started")
67 + d.Debugf("used config: interval: %s, response body limit: %d, source: %s", d.interval, responseBodyLimit, d.source)
68 + defer func() { d.Info("instance is stopped") }()
69 +
70 + d.discover(ctx, in)
71 +
72 + if d.interval <= 0 {
73 + return
74 + }
75 +
76 + tk := time.NewTicker(d.interval)
77 + defer tk.Stop()
78 +
79 + for {
80 + select {
81 + case <-ctx.Done():
82 + return
83 + case <-tk.C:
84 + d.discover(ctx, in)
85 + }
86 + }
87 +}
88 +
89 +func (d *Discoverer) discover(ctx context.Context, in chan<- []model.TargetGroup) {
90 + tgg, err := d.fetchTargetGroup(ctx)
91 + if err != nil {
92 + if !errors.Is(err, context.Canceled) {
93 + d.Warning(err)
94 + }
95 + return
96 + }
97 +
98 + model.SendTargetGroup(ctx, in, tgg)
99 +}
100 +
101 +func (d *Discoverer) fetchTargetGroup(ctx context.Context) (model.TargetGroup, error) {
102 + req, err := web.NewHTTPRequest(d.request)
103 + if err != nil {
104 + return nil, fmt.Errorf("create HTTP request: %w", err)
105 + }
106 + req = req.WithContext(ctx)
107 + safeURL := sanitizedURL(req.URL.String())
108 +
109 + resp, err := d.client.Do(req)
110 + if err != nil {
111 + return nil, fmt.Errorf("HTTP request to %q failed: %w", safeURL, err)
112 + }
113 + if resp.Body != nil {
114 + defer func() { _ = resp.Body.Close() }()
115 + }
116 +
117 + if resp.StatusCode != http.StatusOK {
118 + return nil, fmt.Errorf("%s %q returned HTTP status code: %d", req.Method, safeURL, resp.StatusCode)
119 + }
120 +
121 + bs, err := readResponseBody(resp.Body, responseBodyLimit)
122 + if err != nil {
123 + return nil, fmt.Errorf("read response from %q: %w", safeURL, err)
124 + }
125 +
126 + items, err := d.parser.parse(bs, resp.Header.Get("Content-Type"))
127 + if err != nil {
128 + return nil, fmt.Errorf("parse response from %q: %w", safeURL, err)
129 + }
130 +
131 + targets, err := targetsFromItems(d.source, items)
132 + if err != nil {
133 + return nil, err
134 + }
135 +
136 + return &targetGroup{
137 + source: d.source,
138 + targets: targets,
139 + }, nil
140 +}
141 +
142 +func readResponseBody(r io.Reader, maxBytes int64) ([]byte, error) {
143 + lr := io.LimitReader(r, maxBytes+1)
144 + bs, err := io.ReadAll(lr)
145 + if err != nil {
146 + return nil, err
147 + }
148 + if int64(len(bs)) > maxBytes {
149 + return nil, fmt.Errorf("response body exceeds limit (%d bytes)", maxBytes)
150 + }
151 + return bs, nil
152 +}
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/discoverer_test.go new
+399
@@ -0,0 +1,399 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package httpsd
4 +
5 +import (
6 + "context"
7 + "fmt"
8 + "io"
9 + "net/http"
10 + "net/http/httptest"
11 + "os"
12 + "strings"
13 + "sync"
14 + "testing"
15 + "time"
16 +
17 + "github.com/netdata/netdata/go/plugins/pkg/confopt"
18 + "github.com/netdata/netdata/go/plugins/pkg/web"
19 + "github.com/netdata/netdata/go/plugins/plugin/agent/discovery/sd/model"
20 +
21 + "github.com/stretchr/testify/assert"
22 + "github.com/stretchr/testify/require"
23 +)
24 +
25 +func TestNewDiscoverer(t *testing.T) {
26 + tests := map[string]struct {
27 + cfg Config
28 + wantErr bool
29 + validate func(*testing.T, *Discoverer)
30 + }{
31 + "valid defaults": {
32 + cfg: Config{
33 + HTTPConfig: web.HTTPConfig{
34 + RequestConfig: web.RequestConfig{URL: "http://127.0.0.1"},
35 + },
36 + },
37 + validate: func(t *testing.T, d *Discoverer) {
38 + assert.Equal(t, defaultInterval, d.interval)
39 + assert.Equal(t, defaultTimeout, d.client.Timeout)
40 + assert.Equal(t, formatAuto, d.parser.format)
41 + },
42 + },
43 + "explicit one shot": {
44 + cfg: Config{
45 + HTTPConfig: web.HTTPConfig{
46 + RequestConfig: web.RequestConfig{URL: "http://127.0.0.1"},
47 + },
48 + Interval: durationPtr(0),
49 + },
50 + validate: func(t *testing.T, d *Discoverer) {
51 + assert.Zero(t, d.interval)
52 + },
53 + },
54 + "negative interval": {
55 + cfg: Config{
56 + HTTPConfig: web.HTTPConfig{
57 + RequestConfig: web.RequestConfig{URL: "http://127.0.0.1"},
58 + },
59 + Interval: durationPtr(-time.Second),
60 + },
61 + wantErr: true,
62 + },
63 + "explicit format": {
64 + cfg: Config{
65 + HTTPConfig: web.HTTPConfig{
66 + RequestConfig: web.RequestConfig{URL: "http://127.0.0.1"},
67 + },
68 + Format: "yaml",
69 + },
70 + validate: func(t *testing.T, d *Discoverer) {
71 + assert.Equal(t, formatYAML, d.parser.format)
72 + },
73 + },
74 + "negative timeout uses default": {
75 + cfg: Config{
76 + HTTPConfig: web.HTTPConfig{
77 + RequestConfig: web.RequestConfig{URL: "http://127.0.0.1"},
78 + ClientConfig: web.ClientConfig{Timeout: confopt.Duration(-time.Second)},
79 + },
80 + },
81 + validate: func(t *testing.T, d *Discoverer) {
82 + assert.Equal(t, defaultTimeout, d.client.Timeout)
83 + },
84 + },
85 + "missing url": {
86 + wantErr: true,
87 + },
88 + "unsupported scheme": {
89 + cfg: Config{
90 + HTTPConfig: web.HTTPConfig{
91 + RequestConfig: web.RequestConfig{URL: "ftp://127.0.0.1"},
92 + },
93 + },
94 + wantErr: true,
95 + },
96 + "unsupported format": {
97 + cfg: Config{
98 + HTTPConfig: web.HTTPConfig{
99 + RequestConfig: web.RequestConfig{URL: "http://127.0.0.1"},
100 + },
101 + Format: "toml",
102 + },
103 + wantErr: true,
104 + },
105 + }
106 +
107 + for name, tc := range tests {
108 + t.Run(name, func(t *testing.T) {
109 + d, err := NewDiscoverer(tc.cfg)
110 +
111 + if tc.wantErr {
112 + assert.Error(t, err)
113 + return
114 + }
115 +
116 + require.NoError(t, err)
117 + require.NotNil(t, d)
118 + if tc.validate != nil {
119 + tc.validate(t, d)
120 + }
121 + })
122 + }
123 +}
124 +
125 +func TestDiscoverer_fetchTargetGroup(t *testing.T) {
126 + var gotMethod, gotBody, gotHeader, gotAuth string
127 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
128 + gotMethod = r.Method
129 + gotHeader = r.Header.Get("X-Test")
130 + gotAuth = r.Header.Get("Authorization")
131 + bs, _ := ioReadAllString(r)
132 + gotBody = bs
133 +
134 + w.Header().Set("Content-Type", "application/json")
135 + _, _ = fmt.Fprint(w, `[{"name":"api","url":"http://127.0.0.1"}]`)
136 + }))
137 + defer srv.Close()
138 +
139 + d, err := NewDiscoverer(Config{
140 + HTTPConfig: web.HTTPConfig{
141 + RequestConfig: web.RequestConfig{
142 + URL: srv.URL,
143 + Method: http.MethodPost,
144 + Body: "request-body",
145 + Username: "user",
146 + Password: "pass",
147 + Headers: map[string]string{"X-Test": "value"},
148 + },
149 + },
150 + })
151 + require.NoError(t, err)
152 +
153 + tgg, err := d.fetchTargetGroup(context.Background())
154 + require.NoError(t, err)
155 +
156 + assert.Equal(t, http.MethodPost, gotMethod)
157 + assert.Equal(t, "request-body", gotBody)
158 + assert.Equal(t, "value", gotHeader)
159 + assert.NotEmpty(t, gotAuth)
160 + assert.Equal(t, fullName, tgg.Provider())
161 + assert.Len(t, tgg.Targets(), 1)
162 + assert.Contains(t, tgg.Source(), "discoverer=http,url="+srv.URL+",hash=")
163 +}
164 +
165 +func TestDiscoverer_NotFollowRedirects(t *testing.T) {
166 + var finalHit bool
167 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
168 + switch r.URL.Path {
169 + case "/":
170 + http.Redirect(w, r, "/final", http.StatusFound)
171 + case "/final":
172 + finalHit = true
173 + w.Header().Set("Content-Type", "application/json")
174 + _, _ = fmt.Fprint(w, `[]`)
175 + default:
176 + http.NotFound(w, r)
177 + }
178 + }))
179 + defer srv.Close()
180 +
181 + d, err := NewDiscoverer(Config{
182 + HTTPConfig: web.HTTPConfig{
183 + RequestConfig: web.RequestConfig{URL: srv.URL},
184 + ClientConfig: web.ClientConfig{NotFollowRedirect: true},
185 + },
186 + })
187 + require.NoError(t, err)
188 +
189 + _, err = d.fetchTargetGroup(context.Background())
190 + require.Error(t, err)
191 + assert.Contains(t, err.Error(), "redirect")
192 + assert.False(t, finalHit)
193 +}
194 +
195 +func TestDiscoverer_BearerTokenFileReread(t *testing.T) {
196 + tokenFile := t.TempDir() + "/token"
197 + require.NoError(t, os.WriteFile(tokenFile, []byte("token-1"), 0o600))
198 +
199 + var mu sync.Mutex
200 + var auths []string
201 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
202 + mu.Lock()
203 + auths = append(auths, r.Header.Get("Authorization"))
204 + mu.Unlock()
205 +
206 + w.Header().Set("Content-Type", "application/json")
207 + _, _ = fmt.Fprint(w, `[]`)
208 + }))
209 + defer srv.Close()
210 +
211 + d, err := NewDiscoverer(Config{
212 + HTTPConfig: web.HTTPConfig{
213 + RequestConfig: web.RequestConfig{
214 + URL: srv.URL,
215 + BearerTokenFile: tokenFile,
216 + },
217 + },
218 + })
219 + require.NoError(t, err)
220 +
221 + _, err = d.fetchTargetGroup(context.Background())
222 + require.NoError(t, err)
223 + require.NoError(t, os.WriteFile(tokenFile, []byte("token-2"), 0o600))
224 + _, err = d.fetchTargetGroup(context.Background())
225 + require.NoError(t, err)
226 +
227 + mu.Lock()
228 + defer mu.Unlock()
229 + require.Len(t, auths, 2)
230 + assert.Equal(t, "Bearer token-1", auths[0])
231 + assert.Equal(t, "Bearer token-2", auths[1])
232 +}
233 +
234 +func TestDiscoverer_ResponseBodyLimit(t *testing.T) {
235 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
236 + chunk := strings.Repeat("x", 1024)
237 + for written := int64(0); written <= responseBodyLimit; written += int64(len(chunk)) {
238 + if _, err := w.Write([]byte(chunk)); err != nil {
239 + return
240 + }
241 + }
242 + }))
243 + defer srv.Close()
244 +
245 + d, err := NewDiscoverer(Config{
246 + HTTPConfig: web.HTTPConfig{
247 + RequestConfig: web.RequestConfig{URL: srv.URL},
248 + },
249 + })
250 + require.NoError(t, err)
251 +
252 + _, err = d.fetchTargetGroup(context.Background())
253 + assert.Error(t, err)
254 + assert.Contains(t, err.Error(), "response body exceeds limit")
255 +}
256 +
257 +func TestDiscoverer_ErrorUsesSanitizedURL(t *testing.T) {
258 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
259 + http.Error(w, "nope", http.StatusInternalServerError)
260 + }))
261 + defer srv.Close()
262 +
263 + rawURL := strings.Replace(srv.URL, "http://", "http://user:pass@", 1) + "/path?token=secret#fragment"
264 + d, err := NewDiscoverer(Config{
265 + HTTPConfig: web.HTTPConfig{
266 + RequestConfig: web.RequestConfig{URL: rawURL},
267 + },
268 + })
269 + require.NoError(t, err)
270 +
271 + _, err = d.fetchTargetGroup(context.Background())
272 + require.Error(t, err)
273 +
274 + assert.Contains(t, err.Error(), "/path")
275 + assert.NotContains(t, err.Error(), "user")
276 + assert.NotContains(t, err.Error(), "pass")
277 + assert.NotContains(t, err.Error(), "token")
278 + assert.NotContains(t, err.Error(), "secret")
279 + assert.NotContains(t, err.Error(), "fragment")
280 +}
281 +
282 +func TestDiscoverer_DiscoverFailureDoesNotEmit(t *testing.T) {
283 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
284 + http.Error(w, "nope", http.StatusInternalServerError)
285 + }))
286 + defer srv.Close()
287 +
288 + d, err := NewDiscoverer(Config{
289 + HTTPConfig: web.HTTPConfig{
290 + RequestConfig: web.RequestConfig{URL: srv.URL},
291 + },
292 + Interval: durationPtr(0),
293 + })
294 + require.NoError(t, err)
295 +
296 + ch := make(chan []model.TargetGroup, 1)
297 + d.Discover(context.Background(), ch)
298 +
299 + select {
300 + case got := <-ch:
301 + t.Fatalf("expected no emission, got %v", got)
302 + default:
303 + }
304 +}
305 +
306 +func TestDiscoverer_DiscoverEmptySuccessEmitsEmptyTargetGroup(t *testing.T) {
307 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
308 + w.Header().Set("Content-Type", "application/json")
309 + _, _ = fmt.Fprint(w, `[]`)
310 + }))
311 + defer srv.Close()
312 +
313 + d, err := NewDiscoverer(Config{
314 + HTTPConfig: web.HTTPConfig{
315 + RequestConfig: web.RequestConfig{URL: srv.URL},
316 + },
317 + Interval: durationPtr(0),
318 + })
319 + require.NoError(t, err)
320 +
321 + ch := make(chan []model.TargetGroup, 1)
322 + d.Discover(context.Background(), ch)
323 +
324 + select {
325 + case got := <-ch:
326 + require.Len(t, got, 1)
327 + assert.Empty(t, got[0].Targets())
328 + case <-time.After(time.Second):
329 + t.Fatal("expected empty target group emission")
330 + }
331 +}
332 +
333 +func TestDiscoverer_DiscoverPollsInterval(t *testing.T) {
334 + var (
335 + mu sync.Mutex
336 + calls int
337 + )
338 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
339 + mu.Lock()
340 + calls++
341 + mu.Unlock()
342 +
343 + w.Header().Set("Content-Type", "application/json")
344 + _, _ = fmt.Fprint(w, `[]`)
345 + }))
346 + defer srv.Close()
347 +
348 + d, err := NewDiscoverer(Config{
349 + HTTPConfig: web.HTTPConfig{
350 + RequestConfig: web.RequestConfig{URL: srv.URL},
351 + },
352 + Interval: durationPtr(10 * time.Millisecond),
353 + })
354 + require.NoError(t, err)
355 +
356 + ctx, cancel := context.WithCancel(context.Background())
357 + defer cancel()
358 +
359 + ch := make(chan []model.TargetGroup, 4)
360 + done := make(chan struct{})
361 + go func() {
362 + defer close(done)
363 + d.Discover(ctx, ch)
364 + }()
365 +
366 + for range 2 {
367 + select {
368 + case got := <-ch:
369 + require.Len(t, got, 1)
370 + assert.Empty(t, got[0].Targets())
371 + case <-time.After(time.Second):
372 + t.Fatal("expected target group emission")
373 + }
374 + }
375 +
376 + cancel()
377 + select {
378 + case <-done:
379 + case <-time.After(time.Second):
380 + t.Fatal("discoverer did not stop after context cancellation")
381 + }
382 +
383 + mu.Lock()
384 + defer mu.Unlock()
385 + assert.GreaterOrEqual(t, calls, 2)
386 +}
387 +
388 +func durationPtr(d time.Duration) *confopt.LongDuration {
389 + v := confopt.LongDuration(d)
390 + return &v
391 +}
392 +
393 +func ioReadAllString(r *http.Request) (string, error) {
394 + bs, err := io.ReadAll(r.Body)
395 + if err != nil {
396 + return "", err
397 + }
398 + return string(bs), nil
399 +}
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/parser.go new
+279
@@ -0,0 +1,279 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package httpsd
4 +
5 +import (
6 + "bytes"
7 + "encoding/json"
8 + "errors"
9 + "fmt"
10 + "hash/fnv"
11 + "io"
12 + "sort"
13 + "strings"
14 +
15 + "github.com/netdata/netdata/go/plugins/plugin/agent/discovery/sd/model"
16 +
17 + "gopkg.in/yaml.v2"
18 +)
19 +
20 +type responseParser struct {
21 + format string
22 +}
23 +
24 +func (p responseParser) parse(bs []byte, contentType string) ([]any, error) {
25 + switch p.format {
26 + case formatJSON:
27 + return parseItemsJSON(bs)
28 + case formatYAML:
29 + return parseItemsYAML(bs)
30 + case formatAuto:
31 + return p.parseAuto(bs, contentType)
32 + default:
33 + return nil, fmt.Errorf("unsupported format %q", p.format)
34 + }
35 +}
36 +
37 +func (p responseParser) parseAuto(bs []byte, contentType string) ([]any, error) {
38 + switch detectContentTypeFormat(contentType) {
39 + case formatJSON:
40 + return parseItemsJSON(bs)
41 + case formatYAML:
42 + return parseItemsYAML(bs)
43 + }
44 +
45 + items, jsonErr := parseItemsJSON(bs)
46 + if jsonErr == nil {
47 + return items, nil
48 + }
49 +
50 + items, yamlErr := parseItemsYAML(bs)
51 + if yamlErr == nil {
52 + return items, nil
53 + }
54 +
55 + return nil, fmt.Errorf("parse response as json: %v; parse response as yaml: %v", jsonErr, yamlErr)
56 +}
57 +
58 +func detectContentTypeFormat(contentType string) string {
59 + if i := strings.IndexByte(contentType, ';'); i >= 0 {
60 + contentType = contentType[:i]
61 + }
62 + contentType = strings.TrimSpace(strings.ToLower(contentType))
63 +
64 + switch {
65 + case contentType == "application/json",
66 + contentType == "text/json",
67 + strings.HasSuffix(contentType, "+json"):
68 + return formatJSON
69 + case contentType == "application/yaml",
70 + contentType == "application/x-yaml",
71 + contentType == "text/yaml",
72 + contentType == "text/x-yaml",
73 + strings.HasSuffix(contentType, "+yaml"),
74 + strings.HasSuffix(contentType, "+x-yaml"):
75 + return formatYAML
76 + default:
77 + return ""
78 + }
79 +}
80 +
81 +func parseItemsJSON(bs []byte) ([]any, error) {
82 + dec := json.NewDecoder(bytes.NewReader(bs))
83 +
84 + var data any
85 + if err := dec.Decode(&data); err != nil {
86 + return nil, err
87 + }
88 + var extra any
89 + if err := dec.Decode(&extra); err == nil {
90 + return nil, errors.New("multiple JSON values are not supported")
91 + } else if !errors.Is(err, io.EOF) {
92 + return nil, err
93 + }
94 +
95 + return extractItems(data)
96 +}
97 +
98 +func parseItemsYAML(bs []byte) ([]any, error) {
99 + dec := yaml.NewDecoder(bytes.NewReader(bs))
100 +
101 + var data any
102 + if err := dec.Decode(&data); err != nil {
103 + return nil, err
104 + }
105 + var extra any
106 + if err := dec.Decode(&extra); err == nil {
107 + return nil, errors.New("multiple YAML documents are not supported")
108 + } else if !errors.Is(err, io.EOF) {
109 + return nil, err
110 + }
111 +
112 + data, err := normalizeYAMLValue(data)
113 + if err != nil {
114 + return nil, err
115 + }
116 +
117 + return extractItems(data)
118 +}
119 +
120 +func extractItems(data any) ([]any, error) {
121 + switch v := data.(type) {
122 + case []any:
123 + return normalizeItems(v)
124 + case map[string]any:
125 + items, ok := v["items"]
126 + if !ok {
127 + return nil, errors.New("unsupported response envelope: missing items field")
128 + }
129 + arr, ok := items.([]any)
130 + if !ok {
131 + return nil, fmt.Errorf("unsupported response envelope: items must be an array, got %T", items)
132 + }
133 + return normalizeItems(arr)
134 + default:
135 + return nil, fmt.Errorf("unsupported response format: expected array or object with items array, got %T", data)
136 + }
137 +}
138 +
139 +func normalizeItems(items []any) ([]any, error) {
140 + out := make([]any, 0, len(items))
141 + for i, item := range items {
142 + norm, err := normalizeItem(item)
143 + if err != nil {
144 + return nil, fmt.Errorf("item[%d]: %w", i, err)
145 + }
146 + out = append(out, norm)
147 + }
148 + return out, nil
149 +}
150 +
151 +func normalizeItem(item any) (any, error) {
152 + switch v := item.(type) {
153 + case map[string]any:
154 + return normalizeMap(v)
155 + case string:
156 + return v, nil
157 + default:
158 + return nil, fmt.Errorf("unsupported item type %T", item)
159 + }
160 +}
161 +
162 +func normalizeYAMLValue(v any) (any, error) {
163 + switch vv := v.(type) {
164 + case map[any]any:
165 + m := make(map[string]any, len(vv))
166 + for k, iv := range vv {
167 + ks, ok := k.(string)
168 + if !ok {
169 + return nil, fmt.Errorf("yaml map key must be string, got %T", k)
170 + }
171 + norm, err := normalizeYAMLValue(iv)
172 + if err != nil {
173 + return nil, err
174 + }
175 + m[ks] = norm
176 + }
177 + return m, nil
178 + case map[string]any:
179 + return normalizeMap(vv)
180 + case []any:
181 + arr := make([]any, 0, len(vv))
182 + for _, iv := range vv {
183 + norm, err := normalizeYAMLValue(iv)
184 + if err != nil {
185 + return nil, err
186 + }
187 + arr = append(arr, norm)
188 + }
189 + return arr, nil
190 + default:
191 + return v, nil
192 + }
193 +}
194 +
195 +func normalizeMap(src map[string]any) (map[string]any, error) {
196 + m := make(map[string]any, len(src))
197 + for k, v := range src {
198 + norm, err := normalizeYAMLValue(v)
199 + if err != nil {
200 + return nil, err
201 + }
202 + m[k] = norm
203 + }
204 + return m, nil
205 +}
206 +
207 +func targetsFromItems(source string, items []any) ([]model.Target, error) {
208 + targets := make([]model.Target, 0, len(items))
209 + for i, item := range items {
210 + canonical, err := canonicalJSON(item)
211 + if err != nil {
212 + return nil, fmt.Errorf("item[%d]: canonical encoding: %w", i, err)
213 + }
214 +
215 + tgt := &target{
216 + label: itemLabel(item, canonical),
217 + Item: item,
218 + }
219 + hash, err := model.CalcHash(struct {
220 + Source string
221 + Item string
222 + }{
223 + Source: source,
224 + Item: string(canonical),
225 + })
226 + if err != nil {
227 + return nil, fmt.Errorf("item[%d]: target hash: %w", i, err)
228 + }
229 + tgt.hash = hash
230 + targets = append(targets, tgt)
231 + }
232 + return targets, nil
233 +}
234 +
235 +func itemLabel(item any, canonical []byte) string {
236 + if m, ok := item.(map[string]any); ok {
237 + if v, ok := m["name"].(string); ok {
238 + if name := strings.TrimSpace(v); name != "" {
239 + return name
240 + }
241 + }
242 + }
243 + return fmt.Sprintf("item-%x", hashBytes(canonical))
244 +}
245 +
246 +func canonicalJSON(v any) ([]byte, error) {
247 + return json.Marshal(canonicalValue(v))
248 +}
249 +
250 +func canonicalValue(v any) any {
251 + switch vv := v.(type) {
252 + case map[string]any:
253 + keys := make([]string, 0, len(vv))
254 + for k := range vv {
255 + keys = append(keys, k)
256 + }
257 + sort.Strings(keys)
258 +
259 + m := make(map[string]any, len(vv))
260 + for _, k := range keys {
261 + m[k] = canonicalValue(vv[k])
262 + }
263 + return m
264 + case []any:
265 + arr := make([]any, 0, len(vv))
266 + for _, iv := range vv {
267 + arr = append(arr, canonicalValue(iv))
268 + }
269 + return arr
270 + default:
271 + return v
272 + }
273 +}
274 +
275 +func hashBytes(bs []byte) uint64 {
276 + h := fnv.New64a()
277 + _, _ = h.Write(bs)
278 + return h.Sum64()
279 +}
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/parser_test.go new
+189
@@ -0,0 +1,189 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package httpsd
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/stretchr/testify/assert"
9 + "github.com/stretchr/testify/require"
10 +)
11 +
12 +func TestResponseParser_parse(t *testing.T) {
13 + tests := map[string]struct {
14 + format string
15 + contentType string
16 + body string
17 + want []any
18 + wantErr bool
19 + wantErrText string
20 + }{
21 + "json bare array": {
22 + format: formatJSON,
23 + contentType: "application/json",
24 + body: `[{"name":"api","url":"http://127.0.0.1"}, "http://127.0.0.2"]`,
25 + want: []any{
26 + map[string]any{"name": "api", "url": "http://127.0.0.1"},
27 + "http://127.0.0.2",
28 + },
29 + },
30 + "yaml bare array": {
31 + format: formatYAML,
32 + body: `
33 +- name: api
34 + url: http://127.0.0.1
35 +- http://127.0.0.2
36 +`,
37 + want: []any{
38 + map[string]any{"name": "api", "url": "http://127.0.0.1"},
39 + "http://127.0.0.2",
40 + },
41 + },
42 + "json envelope": {
43 + format: formatJSON,
44 + body: `{"items":[{"name":"api"}]}`,
45 + want: []any{map[string]any{"name": "api"}},
46 + },
47 + "yaml envelope": {
48 + format: formatYAML,
49 + body: `
50 +items:
51 + - name: api
52 +`,
53 + want: []any{map[string]any{"name": "api"}},
54 + },
55 + "auto content type json": {
56 + format: formatAuto,
57 + contentType: "application/vnd.netdata.discovery+json; charset=utf-8",
58 + body: `[{"name":"api"}]`,
59 + want: []any{map[string]any{"name": "api"}},
60 + },
61 + "auto content type yaml": {
62 + format: formatAuto,
63 + contentType: "application/x-yaml",
64 + body: `
65 +- name: api
66 +`,
67 + want: []any{map[string]any{"name": "api"}},
68 + },
69 + "auto json first without content type": {
70 + format: formatAuto,
71 + body: `[{"name":"api"}]`,
72 + want: []any{map[string]any{"name": "api"}},
73 + },
74 + "unsupported envelope": {
75 + format: formatYAML,
76 + body: `jobs: []`,
77 + wantErr: true,
78 + },
79 + "unsupported item type": {
80 + format: formatJSON,
81 + body: `[1]`,
82 + wantErr: true,
83 + },
84 + "malformed": {
85 + format: formatAuto,
86 + body: `[`,
87 + wantErr: true,
88 + },
89 + "json trailing garbage": {
90 + format: formatJSON,
91 + body: `{"items":[]}garbage`,
92 + wantErr: true,
93 + wantErrText: "invalid character",
94 + },
95 + "json multiple values": {
96 + format: formatJSON,
97 + body: `{"items":[]} {"items":[]}`,
98 + wantErr: true,
99 + wantErrText: "multiple JSON values",
100 + },
101 + "yaml multiple documents": {
102 + format: formatYAML,
103 + body: `
104 +items: []
105 +---
106 +items: []
107 +`,
108 + wantErr: true,
109 + wantErrText: "multiple YAML documents",
110 + },
111 + }
112 +
113 + for name, tc := range tests {
114 + t.Run(name, func(t *testing.T) {
115 + items, err := responseParser{format: tc.format}.parse([]byte(tc.body), tc.contentType)
116 +
117 + if tc.wantErr {
118 + assert.Error(t, err)
119 + if tc.wantErrText != "" {
120 + assert.Contains(t, err.Error(), tc.wantErrText)
121 + }
122 + } else {
123 + require.NoError(t, err)
124 + assert.Equal(t, tc.want, items)
125 + }
126 + })
127 + }
128 +}
129 +
130 +func TestTargetsFromItems_HashStability(t *testing.T) {
131 + items1, err := parseItemsJSON([]byte(`[{"name":"api","url":"http://127.0.0.1","headers":{"b":"2","a":"1"}}]`))
132 + require.NoError(t, err)
133 + items2, err := parseItemsJSON([]byte(`[{"headers":{"a":"1","b":"2"},"url":"http://127.0.0.1","name":"api"}]`))
134 + require.NoError(t, err)
135 +
136 + targets1, err := targetsFromItems("source", items1)
137 + require.NoError(t, err)
138 + targets2, err := targetsFromItems("source", items2)
139 + require.NoError(t, err)
140 +
141 + require.Len(t, targets1, 1)
142 + require.Len(t, targets2, 1)
143 + assert.Equal(t, targets1[0].Hash(), targets2[0].Hash())
144 + assert.Equal(t, targets1[0].TUID(), targets2[0].TUID())
145 +}
146 +
147 +func TestTargetsFromItems_ContentChangesHash(t *testing.T) {
148 + items1, err := parseItemsJSON([]byte(`[{"name":"api","url":"http://127.0.0.1"}]`))
149 + require.NoError(t, err)
150 + items2, err := parseItemsJSON([]byte(`[{"name":"api","url":"http://127.0.0.2"}]`))
151 + require.NoError(t, err)
152 +
153 + targets1, err := targetsFromItems("source", items1)
154 + require.NoError(t, err)
155 + targets2, err := targetsFromItems("source", items2)
156 + require.NoError(t, err)
157 +
158 + require.Len(t, targets1, 1)
159 + require.Len(t, targets2, 1)
160 + assert.NotEqual(t, targets1[0].Hash(), targets2[0].Hash())
161 +}
162 +
163 +func TestTargetsFromItems_StringItem(t *testing.T) {
164 + targets, err := targetsFromItems("source", []any{"http://127.0.0.1"})
165 + require.NoError(t, err)
166 +
167 + require.Len(t, targets, 1)
168 + tgt := targets[0].(*target)
169 + assert.Equal(t, "http://127.0.0.1", tgt.Item)
170 + assert.NotEmpty(t, tgt.TUID())
171 + assert.Contains(t, tgt.TUID(), "http_item-")
172 +}
173 +
174 +func TestSourceString_SanitizesURLAndIsStable(t *testing.T) {
175 + cfg := Config{}
176 + cfg.URL = "https://user:pass@example.com/path?token=secret#fragment"
177 + cfg.Headers = map[string]string{"X-Test": "value"}
178 +
179 + src1 := sourceString(cfg)
180 + src2 := sourceString(cfg)
181 +
182 + assert.Equal(t, src1, src2)
183 + assert.Contains(t, src1, "discoverer=http,url=https://example.com/path,hash=")
184 + assert.NotContains(t, src1, "user")
185 + assert.NotContains(t, src1, "pass")
186 + assert.NotContains(t, src1, "token")
187 + assert.NotContains(t, src1, "secret")
188 + assert.NotContains(t, src1, "fragment")
189 +}
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/source.go new
+41
@@ -0,0 +1,41 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package httpsd
4 +
5 +import (
6 + "encoding/json"
7 + "fmt"
8 + "net/url"
9 +)
10 +
11 +func sourceString(cfg Config) string {
12 + src := fmt.Sprintf("discoverer=%s,url=%s,hash=%x", shortName, sanitizedURL(cfg.URL), sourceHash(cfg))
13 + if cfg.Source != "" {
14 + src += fmt.Sprintf(",%s", cfg.Source)
15 + }
16 + return src
17 +}
18 +
19 +func sanitizedURL(rawURL string) string {
20 + u, err := url.Parse(rawURL)
21 + if err != nil {
22 + return rawURL
23 + }
24 + u.User = nil
25 + u.RawQuery = ""
26 + u.Fragment = ""
27 + return u.String()
28 +}
29 +
30 +func sourceHash(cfg Config) uint64 {
31 + type identity struct {
32 + HTTPConfig any `json:"http_config"`
33 + Format string `json:"format"`
34 + }
35 +
36 + bs, _ := json.Marshal(identity{
37 + HTTPConfig: cfg.HTTPConfig,
38 + Format: cfg.format(),
39 + })
40 + return hashBytes(bs)
41 +}
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/target.go new
+30
@@ -0,0 +1,30 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package httpsd
4 +
5 +import (
6 + "fmt"
7 +
8 + "github.com/netdata/netdata/go/plugins/plugin/agent/discovery/sd/model"
9 +)
10 +
11 +type targetGroup struct {
12 + source string
13 + targets []model.Target
14 +}
15 +
16 +func (g *targetGroup) Provider() string { return fullName }
17 +func (g *targetGroup) Source() string { return g.source }
18 +func (g *targetGroup) Targets() []model.Target { return g.targets }
19 +
20 +type target struct {
21 + model.Base `hash:"ignore"`
22 +
23 + hash uint64
24 + label string
25 +
26 + Item any
27 +}
28 +
29 +func (t *target) TUID() string { return fmt.Sprintf("http_%s_%x", t.label, t.hash) }
30 +func (t *target) Hash() uint64 { return t.hash }
src/go/plugin/go.d/discovery/sdext/registry.go
+17
@@ -9,6 +9,7 @@ import (
9 "github.com/netdata/netdata/go/plugins/plugin/agent/discovery/sd"
10 "github.com/netdata/netdata/go/plugins/plugin/agent/discovery/sd/model"
11 "github.com/netdata/netdata/go/plugins/plugin/go.d/discovery/sdext/discoverer/dockersd"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/discovery/sdext/discoverer/httpsd"
13 "github.com/netdata/netdata/go/plugins/plugin/go.d/discovery/sdext/discoverer/k8ssd"
14 "github.com/netdata/netdata/go/plugins/plugin/go.d/discovery/sdext/discoverer/netlistensd"
15 "github.com/netdata/netdata/go/plugins/plugin/go.d/discovery/sdext/discoverer/snmpsd"
@@ -17,6 +18,7 @@ import (
18 const (
19 discovererNetListeners = "net_listeners"
20 discovererDocker = "docker"
21 + discovererHTTP = "http"
22 discovererK8s = "k8s"
23 discovererSNMP = "snmp"
24 )
@@ -35,6 +37,12 @@ func Registry(includeDocker bool) sd.Registry {
37 parseJSONConfig[[]k8ssd.Config],
38 newK8sDiscoverers,
39 ),
40 + sd.NewDescriptor(
41 + discovererHTTP,
42 + schemaHTTP,
43 + parseJSONConfig[httpsd.Config],
44 + newHTTPDiscoverers,
45 + ),
46 sd.NewDescriptor(
47 discovererSNMP,
48 schemaSNMP,
@@ -79,6 +87,15 @@ func newDockerDiscoverers(cfg dockersd.Config, source string) ([]model.Discovere
87 return []model.Discoverer{d}, nil
88 }
89
90 +func newHTTPDiscoverers(cfg httpsd.Config, source string) ([]model.Discoverer, error) {
91 + cfg.Source = source
92 + d, err := httpsd.NewDiscoverer(cfg)
93 + if err != nil {
94 + return nil, err
95 + }
96 + return []model.Discoverer{d}, nil
97 +}
98 +
99 func newK8sDiscoverers(cfgs []k8ssd.Config, source string) ([]model.Discoverer, error) {
100 if len(cfgs) == 0 {
101 return nil, fmt.Errorf("empty %q discoverer config", discovererK8s)
src/go/plugin/go.d/discovery/sdext/registry_test.go
+2
@@ -10,8 +10,10 @@ import (
10
11 func TestRegistry_DockerInclusion(t *testing.T) {
12 withDocker := Registry(true)
13 + assert.Contains(t, withDocker.Types(), discovererHTTP)
14 assert.Contains(t, withDocker.Types(), discovererDocker)
15
16 withoutDocker := Registry(false)
17 + assert.Contains(t, withoutDocker.Types(), discovererHTTP)
18 assert.NotContains(t, withoutDocker.Types(), discovererDocker)
19 }
src/go/plugin/go.d/discovery/sdext/schemas.go
+3
@@ -13,5 +13,8 @@ var schemaDocker string
13 //go:embed "config_schema_k8s.json"
14 var schemaK8s string
15
16 +//go:embed "config_schema_http.json"
17 +var schemaHTTP string
18 +
19 //go:embed "config_schema_snmp.json"
20 var schemaSNMP string