@cryptotaxi247 / netdata-1 / commits / d3747d3c4

docs(go.d/sd): add metadata-driven docs pipeline for service discovery (#22345)

Ilya Mashchenko committed May 4, 2026 at 15:32 UTC d3747d3c4b939395522dc5a4d9ef1276aa06d50e
27 files changed +4570 -10
integrations/gen_doc_service_discovery_page.py new
+391
@@ -0,0 +1,391 @@
1 +"""
2 +Generate src/collectors/SERVICE-DISCOVERY.md from integrations/integrations.js.
3 +
4 +Mirrors gen_doc_secrets_page.py:
5 +- reads discovered service-discovery integrations from integrations.js;
6 +- renders the umbrella Service Discovery page;
7 +- carries the canonical SD model, helper-function reference, and rule
8 + semantics that per-discoverer pages link back to.
9 +"""
10 +
11 +from __future__ import annotations
12 +
13 +import json
14 +import pathlib
15 +import re
16 +from typing import Any, Dict, List
17 +
18 +GITHUB_BLOB_PREFIX = "https://github.com/netdata/netdata/blob/master"
19 +TEMPLATE_PATH = pathlib.Path(__file__).resolve().parent / "templates"
20 +
21 +SD_PAGE = {
22 + "title": "# Service Discovery",
23 + "intro": [
24 + "Stop hand-writing collector jobs for every host, container, or device. Netdata's Service Discovery (SD) finds monitorable targets in your environment and turns them into collector jobs automatically.",
25 + "Each SD pipeline is a `discoverer:` (where to look) plus a list of `services:` rules (how to turn what was found into collector jobs). The discoverer emits **targets**; the rules render **collector job YAML** from those targets using Go templates.",
26 + ],
27 + "jump_to": [
28 + {"label": "How it works", "anchor": "how-it-works"},
29 + {"label": "Configuration file structure", "anchor": "configuration-file-structure"},
30 + {"label": "Rule evaluation semantics", "anchor": "rule-evaluation-semantics"},
31 + {"label": "Template helper reference", "anchor": "template-helper-reference"},
32 + {"label": "config_template rendering", "anchor": "config_template-rendering"},
33 + {"label": "Supported discoverers", "anchor": "supported-discoverers"},
34 + {"label": "Mixing discoverers", "anchor": "mixing-discoverers"},
35 + {"label": "Troubleshooting", "anchor": "troubleshooting"},
36 + ],
37 + "how_it_works": {
38 + "heading": "## How it works",
39 + "intro": [
40 + "Each discovery pipeline runs in five stages:",
41 + ],
42 + "stages": [
43 + {
44 + "name": "Discover",
45 + "description": "The discoverer probes its environment for monitorable things — running containers, listening sockets, kubernetes resources, SNMP devices on a subnet, items returned by an HTTP endpoint. What gets probed and how often is controlled by `discoverer:` options.",
46 + },
47 + {
48 + "name": "Build a target",
49 + "description": "Each discovered thing becomes a target. The target carries discoverer-specific fields (variables) — for example, net_listeners targets have `.Port` / `.Comm` / `.Cmdline`; docker targets have `.Image` / `.Name` / `.Labels`; snmp targets have `.IPAddress` / `.SysInfo.*` / `.Credential.*`. Per-discoverer pages list the full variable set.",
50 + },
51 + {
52 + "name": "Match against services rules",
53 + "description": "The rule engine evaluates each `services:` rule against each target, top to bottom. A rule's `match` expression is a Go template that must render to the literal string `\"true\"` for the rule to apply.",
54 + },
55 + {
56 + "name": "Render the collector job",
57 + "description": "When a rule matches, its `config_template` is executed with the target as context, producing collector job YAML. The rendered YAML can be a single job map or a sequence of jobs.",
58 + },
59 + {
60 + "name": "Hand off to the collector",
61 + "description": "The agent registers the rendered jobs with the matching collector module. From this point on, the collector runs the job — the discoverer is no longer involved.",
62 + },
63 + ],
64 + },
65 + "config_file": {
66 + "heading": "## Configuration file structure",
67 + "intro": [
68 + "Each discoverer has its own file under `/etc/netdata/go.d/sd/`. The filename determines the discoverer kind (`net_listeners.conf`, `docker.conf`, `http.conf`, `snmp.conf`, `k8s.conf`).",
69 + "Every SD file has the same shape:",
70 + ],
71 + "skeleton": """```yaml
72 +disabled: no # set to yes to disable this pipeline
73 +
74 +discoverer:
75 + <kind>: # discoverer kind (must match filename)
76 + # discoverer-specific options — see the per-discoverer page
77 +
78 +services:
79 + - id: <rule-id>
80 + match: <go-template> # must render to the literal string "true"
81 + config_template: | # optional — omit to make this a "skip rule"
82 + # collector job YAML, rendered with target as context
83 +```""",
84 + "notes": [
85 + "`disabled: yes` keeps the file on disk but turns the pipeline off.",
86 + "Editing a stock file requires restarting the agent. UI-managed pipelines apply live.",
87 + "Where each discoverer's stock conf ships (with the Netdata package, with the Helm chart, or not at all) is documented on its per-discoverer page.",
88 + ],
89 + },
90 + "rule_eval": {
91 + "heading": "## Rule evaluation semantics",
92 + "intro": [
93 + "For each discovered target, the engine walks the `services:` array top-to-bottom. The two rule shapes behave differently:",
94 + ],
95 + "shapes": [
96 + {
97 + "name": "Skip rule (no `config_template`)",
98 + "description": "When a skip rule matches, the target is dropped immediately — no jobs are produced and **no further rules run for that target**. Use skip rules to exclude targets the catch-all would otherwise pick up. Place them **before** any template rule.",
99 + },
100 + {
101 + "name": "Template rule (with `config_template`)",
102 + "description": "When a template rule matches, its `config_template` is rendered into one or more collector jobs and rule evaluation **continues** with the next rule. A single target can therefore produce jobs from multiple matching template rules.",
103 + },
104 + ],
105 + "ordering": [
106 + "Recommended ordering for a multi-rule pipeline:",
107 + "",
108 + "1. Skip rules — drop targets you don't want monitored.",
109 + "2. Specific template rules — vendor-specific, label-specific, port-specific.",
110 + "3. A catch-all template rule (`match: '{{ true }}'`) — the default fallback.",
111 + "",
112 + "If a specific template rule already produced the right job, follow it with a skip rule keyed on the same condition to suppress the catch-all for those targets.",
113 + ],
114 + },
115 + "helpers": {
116 + "heading": "## Template helper reference",
117 + "intro": [
118 + "Match expressions and config templates are [Go `text/template`](https://pkg.go.dev/text/template) bodies with three additional helper sets: the standard Go template builtins, the [sprig](https://masterminds.github.io/sprig/) function library, and a small set of Netdata-specific helpers.",
119 + "All templates run with `missingkey=error`. Referencing a field that does not exist on the target type (e.g. typo `.SysInfoo.Name` instead of `.SysInfo.Name`) makes the template execution fail; the agent logs the error and skips that rule for that target.",
120 + ],
121 + "go_builtins": {
122 + "heading": "### Go template builtins",
123 + "intro": "Standard Go `text/template` syntax: pipelines, conditionals, loops, variable assignment, whitespace control.",
124 + "table": [
125 + {"name": "`if`/`else if`/`else`/`end`", "description": "Conditional rendering."},
126 + {"name": "`range`/`end`", "description": "Iterate over a slice or map."},
127 + {"name": "`with`/`end`", "description": "Set the dot to a value if it is non-empty."},
128 + {"name": "`{{- ... -}}`", "description": "Whitespace-trim left/right around the action."},
129 + {"name": "`{{ $var := ... }}`", "description": "Variable assignment, scoped to the enclosing block."},
130 + {"name": "`eq`, `ne`, `lt`, `le`, `gt`, `ge`", "description": "Comparison. `eq A B C ...` is true if `A` equals **any** of the following arguments."},
131 + {"name": "`and`, `or`, `not`", "description": "Boolean composition (variadic)."},
132 + {"name": "`index`", "description": "Index a slice or map: `index .Labels \"app\"`."},
133 + {"name": "`printf`", "description": "Formatted string."},
134 + ],
135 + },
136 + "sprig": {
137 + "heading": "### Sprig functions",
138 + "intro": "The full [sprig library](https://masterminds.github.io/sprig/) is included. The functions most commonly used in stock SD configs:",
139 + "table": [
140 + {"name": "`default DEFAULT VALUE`", "description": "Return `VALUE` if non-empty; otherwise `DEFAULT`."},
141 + {"name": "`empty VALUE`", "description": "True if the value is the zero value for its type."},
142 + {"name": "`hasKey MAP KEY`", "description": "True if `MAP` contains `KEY`. Used heavily by the `http` discoverer."},
143 + {"name": "`kindIs KIND VALUE`", "description": "True if `VALUE`'s reflect kind matches: `string`, `map`, `slice`, `bool`, …"},
144 + {"name": "`lower S` / `upper S`", "description": "Lowercase / uppercase a string."},
145 + {"name": "`trim S` / `trimPrefix PREFIX S` / `trimSuffix SUFFIX S`", "description": "Whitespace and prefix/suffix trimming."},
146 + {"name": "`replace OLD NEW S`", "description": "String replace."},
147 + {"name": "`regexFind RE S`", "description": "Return the first regex match, or empty."},
148 + {"name": "`regexMatch RE S`", "description": "True if `S` matches `RE` (substring match unless anchored)."},
149 + {"name": "`printf FMT V...`", "description": "Same as Go's `fmt.Sprintf`."},
150 + ],
151 + "outro": "See the [sprig docs](https://masterminds.github.io/sprig/) for the full list (string, math, encoding, list, dict, date helpers).",
152 + },
153 + "netdata": {
154 + "heading": "### Netdata helpers",
155 + "intro": "Custom helpers added to the SD template engine:",
156 + "table": [
157 + {
158 + "name": "`match TYPE VALUE PATTERN [PATTERN...]`",
159 + "description": "Returns the string `\"true\"` if `VALUE` matches **any** of the patterns under the named matcher type. `TYPE` is one of:",
160 + "sub": [
161 + "`\"glob\"` — shell glob (`*`, `?`, `[abc]`).",
162 + "`\"sp\"` — Netdata simple patterns (space-separated, `*` wildcard, `!` for negation).",
163 + "`\"re\"` — RE2 regular expression. Matches if the regex matches anywhere in `VALUE` unless explicitly anchored with `^` / `$`.",
164 + "`\"dstar\"` — [doublestar](https://github.com/bmatcuk/doublestar) glob (supports `**` for path-style matching).",
165 + ],
166 + },
167 + {
168 + "name": "`glob VALUE PATTERN [PATTERN...]`",
169 + "description": "Shortcut for `match \"glob\" VALUE PATTERN...`.",
170 + },
171 + {
172 + "name": "`promPort PORT`",
173 + "description": "Returns the well-known Prometheus exporter module name registered for `PORT`, or the empty string. `net_listeners`-specific.",
174 + },
175 + {
176 + "name": "`toYaml VALUE`",
177 + "description": "Serialize `VALUE` as a YAML string. Used by `http` discoverer rules that pass through items as collector job configs.",
178 + },
179 + ],
180 + "notes": [
181 + "`match` and `glob` are case-sensitive.",
182 + "`match \"re\" ...` is unanchored. Use `^...$` to require a full-string match.",
183 + "There is **no** standalone `regexp` or `regex` helper. Use `match \"re\"` (or sprig's `regexFind` / `regexMatch`) for regex matching.",
184 + "`match` and `glob` return the **string** `\"true\"` or `\"false\"`, not a Go bool. The rule engine compares the trimmed template output against the literal `\"true\"`, so a top-level `match: '{{ glob .X \"foo*\" }}'` works directly.",
185 + "**Composing with `if` / `and` / `or` is a footgun**: in Go templates a non-empty string is truthy, so both `\"true\"` *and* `\"false\"` evaluate truthy under `if`. `{{ if glob .X \"foo*\" }}` is therefore **wrong**. Either wrap each result with `eq ... \"true\"` to get a real bool, or use an explicit `if-then-true` block:\n ```\n match: '{{ if and (eq (glob .Vendor \"Cisco*\") \"true\") (eq .Category \"router\") }}true{{ end }}'\n ```",
186 + ],
187 + },
188 + },
189 + "config_template": {
190 + "heading": "## config_template rendering",
191 + "intro": [
192 + "When a template rule matches, its `config_template` is executed with the target as the dot context. The rendered output is parsed as YAML to produce one or more collector jobs.",
193 + ],
194 + "rules": [
195 + {
196 + "name": "Single map → one job",
197 + "description": "If the rendered YAML is a map, a single collector job is created.",
198 + },
199 + {
200 + "name": "Sequence of maps → one job per element",
201 + "description": "If the rendered YAML is a sequence (top-level YAML `-` items), one job is created per element. Use this for multi-job rules. Example: a `net_listeners` rule that emits both a TCP and a Unix-socket MySQL job from the same target —\n\n ```yaml\n - id: mysql\n match: '{{ or (eq .Port \"3306\") (eq .Comm \"mysqld\") }}'\n config_template: |\n - name: local\n dsn: netdata@unix(/var/run/mysqld/mysqld.sock)/\n - name: local\n dsn: netdata@tcp({{.Address}})/\n ```",
202 + },
203 + {
204 + "name": "Module inference from rule `id`",
205 + "description": "If the rendered job map has no `module:` key, the rule's `id` is used as the module name. Set `id: snmp` (or `docker`, `http`, …) to omit `module:` from your template; otherwise include `module:` explicitly.",
206 + },
207 + {
208 + "name": "`id` uniqueness",
209 + "description": "Rule IDs are not required to be unique across rules — multiple rules can share an `id`. The `id` is used for module inference and shows up in agent logs to help you identify which rule produced a job. Pick descriptive IDs (`cisco`, `hp-printers`, `skip-vips`).",
210 + },
211 + {
212 + "name": "Failure handling",
213 + "description": "Render errors, YAML parse errors, and `missingkey=error` failures are logged at warn level. The agent skips the rule for that target and continues evaluation.",
214 + },
215 + ],
216 + },
217 + "supported": {
218 + "heading": "## Supported discoverers",
219 + "intro": "Each discoverer has its own page covering its options, target variables, evaluation specifics, and worked examples.",
220 + },
221 + "mixing": {
222 + "heading": "## Mixing discoverers",
223 + "body": [
224 + "All discoverers can run simultaneously. Each `/etc/netdata/go.d/sd/<kind>.conf` is independent. The same target can theoretically be discovered by more than one discoverer (for example, a containerised application appears in both `docker` and `net_listeners`); each discoverer's pipeline is independent and may produce its own job.",
225 + "Use `disabled: yes` at the top of a stock file to keep it on disk but turn the pipeline off.",
226 + "UI-managed pipelines and file-based pipelines coexist. UI-managed pipelines apply live; file-based pipelines require an agent restart to reload.",
227 + ],
228 + },
229 + "troubleshooting": {
230 + "heading": "## Troubleshooting",
231 + "intro": [
232 + "Common cross-discoverer problems. For discoverer-specific issues, see the per-discoverer page.",
233 + ],
234 + "problems": [
235 + {
236 + "name": "No targets discovered",
237 + "description": "Check the agent log for `discoverer=<kind>` lines. Confirm `disabled: no` and that the discoverer's prerequisites are met (network reachability, credentials, API access).",
238 + },
239 + {
240 + "name": "Targets discovered but no jobs created",
241 + "description": "Check that at least one `services:` rule has both a matching `match` expression and a `config_template`. A rule with no `config_template` is a skip rule — it drops the target instead of producing a job.",
242 + },
243 + {
244 + "name": "`match` always evaluates false",
245 + "description": "Match expressions must render to the literal string `\"true\"`. A bare `{{ if ... }}` block that does not output anything renders to the empty string, which is treated as false. Use `{{ true }}` for catch-all, `{{ glob .X \"...\" }}` for pattern matches, or `{{ if ... }}true{{ end }}` for ad-hoc conditions.",
246 + },
247 + {
248 + "name": "Template render error",
249 + "description": "Look for `failed to execute services[N]->config_template on target` in the log. The most common cause is a typo in a variable reference (e.g. `.SysInfoo.Name`) — `missingkey=error` rejects unknown fields. Use the per-discoverer page's variable table to verify spelling.",
250 + },
251 + {
252 + "name": "YAML parse error after rendering",
253 + "description": "`failed to parse services[N] template data` means the rendered output is not valid YAML. Common cause: a discovered string field contains a colon, hash, or other YAML special character. YAML-quote dynamic values (`name: \"{{ .X }}\"`) when they may be irregular.",
254 + },
255 + ],
256 + },
257 +}
258 +
259 +
260 +def _extract_integrations_json(js_text: str) -> str:
261 + after_categories = js_text.split("export const categories = ", 1)[1]
262 + _, after_integrations = after_categories.split("export const integrations = ", 1)
263 + return re.split(r"\n\s*export const|\Z", after_integrations, maxsplit=1)[0].strip().rstrip(';').strip()
264 +
265 +
266 +def load_integrations(js_path: str = "integrations/integrations.js") -> Any:
267 + with open(js_path, "r", encoding="utf-8") as f:
268 + js_data = f.read()
269 + return json.loads(_extract_integrations_json(js_data))
270 +
271 +
272 +def iterate_integrations(integrations: Any):
273 + if isinstance(integrations, dict):
274 + for integ in integrations.values():
275 + if isinstance(integ, dict):
276 + yield integ
277 + elif isinstance(integrations, list):
278 + for integ in integrations:
279 + if isinstance(integ, dict):
280 + yield integ
281 +
282 +
283 +def collect_sd_integrations(integrations: Any) -> List[Dict[str, Any]]:
284 + items = []
285 + for integ in iterate_integrations(integrations):
286 + if integ.get("integration_type") != "service_discovery":
287 + continue
288 + meta = integ.get("meta", {})
289 + if not isinstance(meta, dict):
290 + continue
291 + if not isinstance(meta.get("name"), str) or not isinstance(meta.get("kind"), str):
292 + continue
293 + items.append(integ)
294 + items.sort(key=lambda item: item["meta"]["name"].lower())
295 + return items
296 +
297 +
298 +def get_repo_path_from_blob_url(url: str) -> str:
299 + if url.startswith(GITHUB_BLOB_PREFIX):
300 + return url[len(GITHUB_BLOB_PREFIX):]
301 + return url
302 +
303 +
304 +def get_sd_readme_link(integ: Dict[str, Any]) -> str:
305 + edit_link = integ.get("edit_link", "") if isinstance(integ, dict) else ""
306 + repo_path = get_repo_path_from_blob_url(edit_link)
307 + if repo_path.endswith("/metadata.yaml"):
308 + return repo_path[: -len("metadata.yaml")] + "README.md"
309 + return ""
310 +
311 +
312 +_jinja_env = None
313 +
314 +
315 +def get_jinja_env():
316 + global _jinja_env
317 + if _jinja_env is None:
318 + from jinja2 import Environment, FileSystemLoader, select_autoescape
319 + _jinja_env = Environment(
320 + loader=FileSystemLoader(TEMPLATE_PATH),
321 + autoescape=select_autoescape(),
322 + block_start_string='[%',
323 + block_end_string='%]',
324 + variable_start_string='[[',
325 + variable_end_string=']]',
326 + comment_start_string='[#',
327 + comment_end_string='#]',
328 + trim_blocks=True,
329 + lstrip_blocks=True,
330 + )
331 + return _jinja_env
332 +
333 +
334 +def build_discoverers_context(integrations: Any) -> List[Dict[str, str]]:
335 + items = []
336 + for integ in collect_sd_integrations(integrations):
337 + meta = integ.get("meta", {})
338 + kind = meta.get("kind", "")
339 + name = meta.get("name", "")
340 + tagline = meta.get("tagline", "")
341 + readme_link = get_sd_readme_link(integ)
342 + items.append({
343 + "name": name,
344 + "kind": kind,
345 + "config_file": f"/etc/netdata/go.d/sd/{kind}.conf",
346 + "name_link": f'[{name}]({readme_link})' if readme_link else name,
347 + "tagline": tagline,
348 + })
349 + return items
350 +
351 +
352 +def build_page_context() -> Dict[str, Any]:
353 + return {
354 + "title": SD_PAGE["title"],
355 + "intro": SD_PAGE["intro"],
356 + "jump_to_line": " • ".join(
357 + f'[{jump["label"]}](#{jump["anchor"]})' for jump in SD_PAGE["jump_to"]
358 + ),
359 + "how_it_works": SD_PAGE["how_it_works"],
360 + "config_file": SD_PAGE["config_file"],
361 + "rule_eval": SD_PAGE["rule_eval"],
362 + "helpers": SD_PAGE["helpers"],
363 + "config_template": SD_PAGE["config_template"],
364 + "supported": SD_PAGE["supported"],
365 + "mixing": SD_PAGE["mixing"],
366 + "troubleshooting": SD_PAGE["troubleshooting"],
367 + }
368 +
369 +
370 +def render_sd_md(integrations: Any) -> str:
371 + template = get_jinja_env().get_template("service_discovery.md")
372 + return template.render(
373 + page=build_page_context(),
374 + discoverers=build_discoverers_context(integrations),
375 + )
376 +
377 +
378 +def generate_sd_md() -> None:
379 + integrations = load_integrations()
380 + content = render_sd_md(integrations)
381 +
382 + outfile = pathlib.Path("./src/collectors/SERVICE-DISCOVERY.md")
383 + outfile.parent.mkdir(parents=True, exist_ok=True)
384 +
385 + tmp = outfile.with_suffix(outfile.suffix + ".tmp")
386 + tmp.write_text(content.rstrip("\n") + "\n", encoding="utf-8")
387 + tmp.replace(outfile)
388 +
389 +
390 +if __name__ == "__main__":
391 + generate_sd_md()
integrations/gen_docs_integrations.py
+46
@@ -33,6 +33,7 @@ def cleanup(only_base_paths=None):
33 "integrations/logs",
34 "integrations/cloud-authentication",
35 "src/go/plugin/agent/secrets/secretstore/backends",
36 + "src/go/plugin/go.d/discovery/sdext/discoverer",
37 ]
38 bases = only_base_paths if only_base_paths else targets
39 for base in bases:
@@ -385,6 +386,36 @@ endmeta-->
386 if integration.get("troubleshooting"):
387 md += f"\n{integration['troubleshooting']}\n"
388
389 + elif mode == "service_discovery":
390 + meta_yaml = integration["edit_link"].replace("blob", "edit")
391 + sidebar_label = integration["meta"]["name"]
392 + learn_rel_path = "Collecting Metrics/Service Discovery"
393 + keywords = integration["keywords"] if "keywords" in integration else None
394 +
395 + md = f"""<!--startmeta
396 +meta_yaml: "{meta_yaml}"
397 +sidebar_label: "{sidebar_label}"
398 +learn_status: "Published"
399 +learn_rel_path: "{learn_rel_path}"
400 +"""
401 + if keywords:
402 + md += f"keywords: {keywords}\n"
403 +
404 + md += """message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE SERVICE DISCOVERY DISCOVERER'S metadata.yaml FILE"
405 +endmeta-->
406 +
407 +"""
408 + md += create_overview(integration, integration['meta']['icon_filename'])
409 +
410 + if integration.get("setup"):
411 + md += f"\n{integration['setup']}\n"
412 + if integration.get("services"):
413 + md += f"\n{integration['services']}\n"
414 + if integration.get("verify"):
415 + md += f"\n{integration['verify']}\n"
416 + if integration.get("troubleshooting"):
417 + md += f"\n{integration['troubleshooting']}\n"
418 +
419 except Exception as e:
420 print("Exception building md", e, integration.get("id"))
421
@@ -609,6 +640,21 @@ def main():
640 output_slug=clean_string(integration["meta"]["kind"]),
641 )
642
643 + elif itype == "service_discovery" and not args.collector:
644 + meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
645 + integration, categories, mode="service_discovery"
646 + )
647 + path = build_path(meta_yaml)
648 + write_to_file(
649 + path,
650 + md,
651 + meta_yaml,
652 + sidebar_label,
653 + community,
654 + integration_id=iid,
655 + output_slug=clean_string(integration["meta"]["kind"]),
656 + )
657 +
658 elif itype == "agent_notification" and not args.collector:
659 meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
660 integration, categories, mode="agent-notification"
integrations/gen_integrations.py
+129 -10
@@ -64,6 +64,10 @@ SECRETSTORE_SOURCES = [
64 (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'agent' / 'secrets' / 'secretstore' / 'backends', True),
65 ]
66
67 +SERVICE_DISCOVERY_SOURCES = [
68 + (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'go.d' / 'discovery' / 'sdext' / 'discoverer', True),
69 +]
70 +
71 COLLECTOR_RENDER_KEYS = [
72 'alerts',
73 'metrics',
@@ -109,6 +113,14 @@ SECRETSTORE_RENDER_KEYS = [
113 'troubleshooting',
114 ]
115
116 +SERVICE_DISCOVERY_RENDER_KEYS = [
117 + 'overview',
118 + 'setup',
119 + 'services',
120 + 'verify',
121 + 'troubleshooting',
122 +]
123 +
124 CUSTOM_TAG_PATTERN = re.compile('\\{% if .*?%\\}.*?\\{% /if %\\}|\\{%.*?%\\}', flags=re.DOTALL)
125 FIXUP_BLANK_PATTERN = re.compile('\\\\\\n *\\n')
126
@@ -201,6 +213,11 @@ SECRETSTORE_VALIDATOR = Draft7Validator(
213 registry=registry,
214 )
215
216 +SERVICE_DISCOVERY_VALIDATOR = Draft7Validator(
217 + {'$ref': './service_discovery.json#'},
218 + registry=registry,
219 +)
220 +
221 _jinja_env = False
222
223
@@ -248,15 +265,24 @@ def anchorfy(value):
265
266
267 def get_section_template_name(item, key):
251 - if key != 'setup':
252 - return f'{key}.md'
253 -
268 integration_type = item.get('integration_type')
255 - if integration_type == 'secretstore':
256 - return 'setup-secretstore.md'
257 - if integration_type == 'logs':
258 - return 'setup-logs.md'
259 - return 'setup-generic.md'
269 +
270 + if key == 'setup':
271 + if integration_type == 'secretstore':
272 + return 'setup-secretstore.md'
273 + if integration_type == 'service_discovery':
274 + return 'setup-service_discovery.md'
275 + if integration_type == 'logs':
276 + return 'setup-logs.md'
277 + return 'setup-generic.md'
278 +
279 + if integration_type == 'service_discovery':
280 + if key == 'services':
281 + return 'sd-services.md'
282 + if key == 'verify':
283 + return 'sd-verify.md'
284 +
285 + return f'{key}.md'
286
287
288 def get_category_sets(categories):
@@ -689,6 +715,54 @@ def load_secretstores():
715 return ret
716
717
718 +def _load_service_discovery_file(file, repo):
719 + debug(f'Loading {file}.')
720 + data = load_yaml(file)
721 +
722 + if not data:
723 + return []
724 +
725 + try:
726 + SERVICE_DISCOVERY_VALIDATOR.validate(data)
727 + except ValidationError as e:
728 + warn(
729 + f'Failed to validate {file} against the schema: {e.message} (path: {"/".join(str(p) for p in e.absolute_path)})',
730 + file)
731 + return []
732 +
733 + if 'id' in data:
734 + data['integration_type'] = 'service_discovery'
735 + data['_src_path'] = file
736 + data['_repo'] = repo
737 + data['_index'] = 0
738 +
739 + return [data]
740 + else:
741 + ret = []
742 +
743 + for idx, item in enumerate(data):
744 + item['integration_type'] = 'service_discovery'
745 + item['_src_path'] = file
746 + item['_repo'] = repo
747 + item['_index'] = idx
748 + ret.append(item)
749 +
750 + return ret
751 +
752 +
753 +def load_service_discoveries():
754 + ret = []
755 +
756 + for repo, path, match in SERVICE_DISCOVERY_SOURCES:
757 + if match and path.exists() and path.is_dir():
758 + for file in path.glob(METADATA_PATTERN):
759 + ret.extend(_load_service_discovery_file(file, repo))
760 + elif not match and path.exists() and path.is_file():
761 + ret.extend(_load_service_discovery_file(path, repo))
762 +
763 + return ret
764 +
765 +
766 def make_id(meta):
767 if 'monitored_instance' in meta:
768 instance_name = meta['monitored_instance']['name'].replace(' ', '_')
@@ -1188,6 +1262,48 @@ def render_secretstores(categories, secretstores, ids):
1262 return secretstores, clean_secretstores, ids
1263
1264
1265 +def render_service_discoveries(categories, service_discoveries, ids):
1266 + debug('Sorting service discoveries.')
1267 +
1268 + sort_integrations(service_discoveries)
1269 +
1270 + debug('Checking service discovery ids.')
1271 +
1272 + service_discoveries, ids = dedupe_integrations(service_discoveries, ids)
1273 +
1274 + clean_service_discoveries = []
1275 +
1276 + for item in service_discoveries:
1277 + item['edit_link'] = make_edit_link(item)
1278 +
1279 + clean_item = deepcopy(item)
1280 +
1281 + for key in SERVICE_DISCOVERY_RENDER_KEYS:
1282 + if key in item.keys():
1283 + template = get_jinja_env().get_template(get_section_template_name(item, key))
1284 + data = template.render(entry=item, clean=False)
1285 + clean_data = template.render(entry=item, clean=True)
1286 +
1287 + if 'variables' in item['meta']:
1288 + template = get_jinja_env().from_string(data)
1289 + data = template.render(variables=item['meta']['variables'], clean=False)
1290 + template = get_jinja_env().from_string(clean_data)
1291 + clean_data = template.render(variables=item['meta']['variables'], clean=True)
1292 + else:
1293 + data = ''
1294 + clean_data = ''
1295 +
1296 + item[key] = data
1297 + clean_item[key] = clean_data
1298 +
1299 + for k in ['_src_path', '_repo', '_index']:
1300 + del item[k], clean_item[k]
1301 +
1302 + clean_service_discoveries.append(clean_item)
1303 +
1304 + return service_discoveries, clean_service_discoveries, ids
1305 +
1306 +
1307 def convert_local_links(text, prefix):
1308 return text.replace("](/", f"]({prefix}/")
1309
@@ -1220,6 +1336,7 @@ def main():
1336 logs = load_logs()
1337 authentications = load_authentications()
1338 secretstores = load_secretstores()
1339 + service_discoveries = load_service_discoveries()
1340
1341 collectors, clean_collectors, ids = render_collectors(categories, collectors, dict())
1342 deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids)
@@ -1231,11 +1348,13 @@ def main():
1348 logs, clean_logs, ids = render_logs(categories, logs, ids)
1349 authentications, clean_authentications, ids = render_authentications(categories, authentications, ids)
1350 secretstores, clean_secretstores, ids = render_secretstores(categories, secretstores, ids)
1351 + service_discoveries, clean_service_discoveries, ids = render_service_discoveries(categories, service_discoveries,
1352 + ids)
1353
1235 - integrations = collectors + deploy + exporters + agent_notifications + cloud_notifications + logs + authentications + secretstores
1354 + integrations = collectors + deploy + exporters + agent_notifications + cloud_notifications + logs + authentications + secretstores + service_discoveries
1355 render_integrations(categories, integrations)
1356
1238 - clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_agent_notifications + clean_cloud_notifications + clean_logs + clean_authentications + clean_secretstores
1357 + clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_agent_notifications + clean_cloud_notifications + clean_logs + clean_authentications + clean_secretstores + clean_service_discoveries
1358 render_json(categories, clean_integrations)
1359
1360 return fail_on_warnings()
integrations/schemas/service_discovery.json new
+248
@@ -0,0 +1,248 @@
1 +{
2 + "$schema": "http://json-schema.org/draft-07/schema#",
3 + "title": "Netdata service discovery (SD) discoverer metadata.",
4 + "oneOf": [
5 + {
6 + "$ref": "#/$defs/entry"
7 + },
8 + {
9 + "type": "array",
10 + "minItems": 1,
11 + "items": {
12 + "$ref": "#/$defs/entry"
13 + }
14 + }
15 + ],
16 + "$defs": {
17 + "meta": {
18 + "type": "object",
19 + "description": "Information about the service discovery discoverer.",
20 + "properties": {
21 + "kind": {
22 + "type": "string",
23 + "description": "Runtime discoverer kind, matching the registry name and stock conf filename (e.g. 'snmp', 'docker', 'k8s', 'http', 'net_listeners')."
24 + },
25 + "name": {
26 + "type": "string",
27 + "description": "Display name shown in docs, sidebar, and the SD hub page."
28 + },
29 + "tagline": {
30 + "type": "string",
31 + "description": "One-line summary of what this discoverer finds, used in the SD hub page table. Should fit on one line in a markdown table cell."
32 + },
33 + "link": {
34 + "type": "string",
35 + "description": "Reference link for the underlying technology or protocol (Wikipedia page, vendor page, RFC, etc.)."
36 + },
37 + "icon_filename": {
38 + "type": "string",
39 + "description": "Icon filename hosted at https://netdata.cloud/img/."
40 + }
41 + },
42 + "required": ["kind", "name", "tagline", "link", "icon_filename"]
43 + },
44 + "overview": {
45 + "type": "object",
46 + "description": "General information about the discoverer.",
47 + "properties": {
48 + "description": {
49 + "type": "string",
50 + "description": "What this discoverer does, what it monitors, and when to use it."
51 + },
52 + "how_it_works": {
53 + "type": "string",
54 + "description": "Optional beginner on-ramp: a short, concrete explanation of the discoverer's lifecycle (probe -> sysinfo/metadata read -> rule match -> job creation). Rendered as '### How it works' under Overview."
55 + },
56 + "limitations": {
57 + "type": "string",
58 + "description": "Optional explanation of notable limitations or behavior."
59 + }
60 + },
61 + "required": ["description"]
62 + },
63 + "verify": {
64 + "type": "object",
65 + "description": "Optional 'Verify discovery worked' section, rendered as a top-level h2 between Service Rules and Troubleshooting. Use it to tell users where discovered jobs appear in the UI, how to read discoverer logs, and what success looks like.",
66 + "properties": {
67 + "description": {
68 + "type": "string",
69 + "description": "Optional intro sentence(s)."
70 + },
71 + "checks": {
72 + "type": "object",
73 + "properties": {
74 + "list": {
75 + "type": "array",
76 + "minItems": 1,
77 + "description": "Concrete things the user can check to confirm discovery works.",
78 + "items": {
79 + "type": "object",
80 + "properties": {
81 + "name": {
82 + "type": "string",
83 + "description": "Check title (e.g. 'Confirm targets are appearing', 'Read the discoverer log')."
84 + },
85 + "description": {
86 + "type": "string",
87 + "description": "Step-by-step explanation."
88 + }
89 + },
90 + "required": ["name", "description"]
91 + }
92 + }
93 + },
94 + "required": ["list"]
95 + }
96 + },
97 + "required": ["checks"]
98 + },
99 + "services_template_variable": {
100 + "type": "object",
101 + "properties": {
102 + "name": {
103 + "type": "string",
104 + "description": "Variable name as used in templates, e.g. '.Port' or '.SysInfo.Name'."
105 + },
106 + "type": {
107 + "type": "string",
108 + "description": "Variable type (typically 'string'). Document empty/zero-value semantics in description."
109 + },
110 + "description": {
111 + "type": "string",
112 + "description": "What the variable holds, where it comes from, and what to expect when the value is empty or unset."
113 + }
114 + },
115 + "required": ["name", "description"]
116 + },
117 + "services_evaluation_step": {
118 + "type": "object",
119 + "description": "One bullet describing rule evaluation behavior for this discoverer.",
120 + "properties": {
121 + "name": {
122 + "type": "string"
123 + },
124 + "description": {
125 + "type": "string"
126 + }
127 + },
128 + "required": ["name", "description"]
129 + },
130 + "services_example": {
131 + "type": "object",
132 + "properties": {
133 + "name": {
134 + "type": "string",
135 + "description": "Example name."
136 + },
137 + "description": {
138 + "type": "string",
139 + "description": "Example description."
140 + },
141 + "config": {
142 + "type": "string",
143 + "description": "YAML snippet showing one or more entries from the 'services:' array."
144 + }
145 + },
146 + "required": ["name", "description", "config"]
147 + },
148 + "services": {
149 + "type": "object",
150 + "description": "How 'services:' rules turn discovered targets into collector jobs for this discoverer. Note: shared rule semantics, the full template-function reference, and 'config_template' rendering rules live on the SD hub page; per-discoverer pages should focus on what is specific to this discoverer.",
151 + "properties": {
152 + "description": {
153 + "type": "string",
154 + "description": "Introductory text. Should explain what a rule is for THIS discoverer and link to the hub page for the shared model."
155 + },
156 + "evaluation": {
157 + "type": "object",
158 + "description": "Optional discoverer-specific notes on rule evaluation order, skip-rules, and multi-job behavior. The shared semantics belong on the hub page; use this only when this discoverer differs (or when a clarification is worth repeating in context).",
159 + "properties": {
160 + "description": {
161 + "type": "string"
162 + },
163 + "list": {
164 + "type": "array",
165 + "items": {
166 + "$ref": "#/$defs/services_evaluation_step"
167 + }
168 + }
169 + }
170 + },
171 + "template_variables": {
172 + "type": "object",
173 + "description": "Discoverer-specific variables available inside 'match' and 'config_template'. Do NOT list shared template helper functions (sprig, glob, match, etc.); those live on the hub page.",
174 + "properties": {
175 + "description": {
176 + "type": "string"
177 + },
178 + "list": {
179 + "type": "array",
180 + "minItems": 1,
181 + "items": {
182 + "$ref": "#/$defs/services_template_variable"
183 + }
184 + }
185 + },
186 + "required": ["list"]
187 + },
188 + "examples": {
189 + "type": "object",
190 + "description": "Worked examples of 'services:' rules for this discoverer.",
191 + "properties": {
192 + "description": {
193 + "type": "string"
194 + },
195 + "list": {
196 + "type": "array",
197 + "minItems": 1,
198 + "items": {
199 + "$ref": "#/$defs/services_example"
200 + }
201 + }
202 + },
203 + "required": ["list"]
204 + }
205 + },
206 + "required": ["description", "template_variables", "examples"]
207 + },
208 + "entry": {
209 + "type": "object",
210 + "description": "Metadata for a single service discovery discoverer.",
211 + "properties": {
212 + "id": {
213 + "$ref": "./shared.json#/$defs/id"
214 + },
215 + "meta": {
216 + "$ref": "#/$defs/meta"
217 + },
218 + "keywords": {
219 + "$ref": "./shared.json#/$defs/keywords"
220 + },
221 + "overview": {
222 + "$ref": "#/$defs/overview"
223 + },
224 + "setup": {
225 + "$ref": "./shared.json#/$defs/full_setup"
226 + },
227 + "services": {
228 + "$ref": "#/$defs/services"
229 + },
230 + "verify": {
231 + "$ref": "#/$defs/verify"
232 + },
233 + "troubleshooting": {
234 + "$ref": "./shared.json#/$defs/troubleshooting"
235 + }
236 + },
237 + "required": [
238 + "id",
239 + "meta",
240 + "keywords",
241 + "overview",
242 + "setup",
243 + "services",
244 + "troubleshooting"
245 + ]
246 + }
247 + }
248 +}
integrations/templates/overview.md
+2
@@ -10,4 +10,6 @@
10 [% include 'overview/authentication.md' %]
11 [% elif entry.integration_type == 'logs' %]
12 [% include 'overview/logs.md' %]
13 +[% elif entry.integration_type == 'service_discovery' %]
14 +[% include 'overview/service_discovery.md' %]
15 [% endif %]
integrations/templates/overview/service_discovery.md new
+20
@@ -0,0 +1,20 @@
1 +[# Jinja template fragment: integrations/templates/overview/service_discovery.md #]
2 +# [[ entry.meta.name ]] discovery
3 +
4 +Kind: `[[ entry.meta.kind ]]`
5 +
6 +## Overview
7 +
8 +[[ entry.overview.description ]]
9 +
10 +[% if entry.overview.how_it_works %]
11 +### How it works
12 +
13 +[[ entry.overview.how_it_works ]]
14 +
15 +[% endif %]
16 +[% if entry.overview.limitations %]
17 +### Limitations
18 +
19 +[[ entry.overview.limitations ]]
20 +[% endif %]
integrations/templates/sd-services.md new
+65
@@ -0,0 +1,65 @@
1 +[# Jinja template: integrations/templates/services.md
2 + Renders the SD-specific 'services:' rules section.
3 + - Heading is h2 'Service Rules' (sibling of Setup/Troubleshooting).
4 + - Shared template-helper reference lives on the SD hub page; this section
5 + only documents discoverer-specific variables.
6 + - Optional `services.evaluation` h3 surfaces rule-evaluation gotchas where
7 + they differ between discoverers (e.g. skip-rules in net_listeners/docker). #]
8 +## Service Rules
9 +
10 +[[ entry.services.description ]]
11 +
12 +[% if entry.services.evaluation is defined and (entry.services.evaluation.description or entry.services.evaluation.list) %]
13 +### How rules are evaluated
14 +[% if entry.services.evaluation.description %]
15 +
16 +[[ entry.services.evaluation.description ]]
17 +
18 +[% endif %]
19 +[% if entry.services.evaluation.list %]
20 +
21 +[% for step in entry.services.evaluation.list %]
22 +- **[[ step.name ]]** — [[ strfy(step.description) ]]
23 +[% endfor %]
24 +
25 +[% endif %]
26 +[% endif %]
27 +### Template Variables
28 +[% if entry.services.template_variables.description %]
29 +
30 +[[ entry.services.template_variables.description ]]
31 +
32 +[% endif %]
33 +[% set has_types = entry.services.template_variables.list | selectattr("type","defined") | list | length > 0 %]
34 +[% if has_types %]
35 +
36 +| Variable | Type | Description |
37 +|:---------|:-----|:------------|
38 +[% for v in entry.services.template_variables.list %]
39 +| `[[ v.name ]]` | [[ v.type if v.type is defined else "string" ]] | [[ strfy(v.description) ]] |
40 +[% endfor %]
41 +[% else %]
42 +
43 +| Variable | Description |
44 +|:---------|:------------|
45 +[% for v in entry.services.template_variables.list %]
46 +| `[[ v.name ]]` | [[ strfy(v.description) ]] |
47 +[% endfor %]
48 +[% endif %]
49 +
50 +### Examples
51 +[% if entry.services.examples.description %]
52 +
53 +[[ entry.services.examples.description ]]
54 +
55 +[% endif %]
56 +[% for example in entry.services.examples.list %]
57 +#### [[ example.name ]]
58 +
59 +[[ example.description ]]
60 +
61 +```yaml
62 +[[ example.config ]]
63 +```
64 +
65 +[% endfor %]
integrations/templates/sd-verify.md new
+17
@@ -0,0 +1,17 @@
1 +[# Jinja template: integrations/templates/verify.md
2 + Renders the optional '## Verify discovery worked' h2.
3 + Skip the section entirely if the metadata has no `verify:` block. #]
4 +[% if entry.verify is defined and entry.verify.checks and entry.verify.checks.list %]
5 +## Verify discovery worked
6 +[% if entry.verify.description %]
7 +
8 +[[ entry.verify.description ]]
9 +
10 +[% endif %]
11 +[% for check in entry.verify.checks.list %]
12 +### [[ check.name ]]
13 +
14 +[[ check.description ]]
15 +
16 +[% endfor %]
17 +[% endif %]
integrations/templates/service_discovery.md new
+147
@@ -0,0 +1,147 @@
1 +[# Jinja template: integrations/templates/service_discovery.md
2 + Renders the umbrella Service Discovery hub page (analog of templates/secrets.md).
3 + #]
4 +[[ page.title ]]
5 +
6 +[% for paragraph in page.intro %]
7 +[[ paragraph ]]
8 +
9 +[% endfor %]
10 +### Jump To
11 +
12 +[[ page.jump_to_line ]]
13 +
14 +
15 +[[ page.how_it_works.heading ]]
16 +
17 +[% for paragraph in page.how_it_works.intro %]
18 +[[ paragraph ]]
19 +
20 +[% endfor %]
21 +[% for stage in page.how_it_works.stages %]
22 +[[ loop.index ]]. **[[ stage.name ]]** — [[ stage.description ]]
23 +[% endfor %]
24 +
25 +
26 +[[ page.config_file.heading ]]
27 +
28 +[% for paragraph in page.config_file.intro %]
29 +[[ paragraph ]]
30 +
31 +[% endfor %]
32 +[[ page.config_file.skeleton ]]
33 +
34 +[% for note in page.config_file.notes %]
35 +- [[ note ]]
36 +[% endfor %]
37 +
38 +
39 +[[ page.rule_eval.heading ]]
40 +
41 +[% for paragraph in page.rule_eval.intro %]
42 +[[ paragraph ]]
43 +
44 +[% endfor %]
45 +[% for shape in page.rule_eval.shapes %]
46 +- **[[ shape.name ]]** — [[ shape.description ]]
47 +[% endfor %]
48 +
49 +### Order matters
50 +
51 +[% for line in page.rule_eval.ordering %]
52 +[[ line ]]
53 +[% endfor %]
54 +
55 +
56 +[[ page.helpers.heading ]]
57 +
58 +[% for paragraph in page.helpers.intro %]
59 +[[ paragraph ]]
60 +
61 +[% endfor %]
62 +
63 +[[ page.helpers.go_builtins.heading ]]
64 +
65 +[[ page.helpers.go_builtins.intro ]]
66 +
67 +| Construct | Description |
68 +|:----------|:------------|
69 +[% for row in page.helpers.go_builtins.table %]
70 +| [[ row.name ]] | [[ row.description ]] |
71 +[% endfor %]
72 +
73 +
74 +[[ page.helpers.sprig.heading ]]
75 +
76 +[[ page.helpers.sprig.intro ]]
77 +
78 +| Function | Description |
79 +|:---------|:------------|
80 +[% for row in page.helpers.sprig.table %]
81 +| [[ row.name ]] | [[ row.description ]] |
82 +[% endfor %]
83 +
84 +[[ page.helpers.sprig.outro ]]
85 +
86 +
87 +[[ page.helpers.netdata.heading ]]
88 +
89 +[[ page.helpers.netdata.intro ]]
90 +
91 +[% for row in page.helpers.netdata.table %]
92 +- [[ row.name ]] — [[ row.description ]]
93 +[% if row.sub is defined %]
94 +[% for sub in row.sub %]
95 + - [[ sub ]]
96 +[% endfor %]
97 +[% endif %]
98 +[% endfor %]
99 +
100 +**Notes:**
101 +
102 +[% for note in page.helpers.netdata.notes %]
103 +- [[ note ]]
104 +[% endfor %]
105 +
106 +
107 +[[ page.config_template.heading ]]
108 +
109 +[% for paragraph in page.config_template.intro %]
110 +[[ paragraph ]]
111 +
112 +[% endfor %]
113 +[% for rule in page.config_template.rules %]
114 +- **[[ rule.name ]]** — [[ rule.description ]]
115 +[% endfor %]
116 +
117 +
118 +[[ page.supported.heading ]]
119 +
120 +[[ page.supported.intro ]]
121 +
122 +| Discoverer | Kind | Stock conf | Discovers |
123 +|:-----------|:-----|:-----------|:----------|
124 +[% for d in discoverers %]
125 +| [[ d.name_link ]] | `[[ d.kind ]]` | `[[ d.config_file ]]` | [[ d.tagline ]] |
126 +[% endfor %]
127 +
128 +
129 +[[ page.mixing.heading ]]
130 +
131 +[% for paragraph in page.mixing.body %]
132 +[[ paragraph ]]
133 +
134 +[% endfor %]
135 +
136 +[[ page.troubleshooting.heading ]]
137 +
138 +[% for paragraph in page.troubleshooting.intro %]
139 +[[ paragraph ]]
140 +
141 +[% endfor %]
142 +[% for item in page.troubleshooting.problems %]
143 +### [[ item.name ]]
144 +
145 +[[ item.description ]]
146 +
147 +[% endfor %]
integrations/templates/setup-service_discovery.md new
+101
@@ -0,0 +1,101 @@
1 +[# Jinja template: integrations/templates/setup-service_discovery.md
2 + Closely mirrors setup-secretstore.md but for SD discoverers.
3 + The 'via File' section explicitly notes the dual-block (`discoverer:` + `services:`)
4 + structure and points readers forward to the Service Rules section. #]
5 +## Setup
6 +
7 +You can configure the `[[ entry.meta.kind ]]` discoverer in two ways:
8 +
9 +| Method | Best for | How to |
10 +|:--|:--|:--|
11 +| [**UI**](#via-ui) | Fast setup without editing files | Go to `Collectors -> go.d -> ServiceDiscovery -> [[ entry.meta.kind ]]`, then add a discovery pipeline. |
12 +| [**File**](#via-file) | File-based configuration or automation | Edit `/etc/netdata/[[ entry.setup.configuration.file.name ]]` and define the `discoverer:` and `services:` blocks. |
13 +
14 +### Prerequisites
15 +[% if entry.setup.prerequisites.list %]
16 +
17 +[% for prereq in entry.setup.prerequisites.list %]
18 +#### [[ prereq.title ]]
19 +
20 +[[ prereq.description ]]
21 +
22 +[% endfor %]
23 +[% else %]
24 +
25 +No action required.
26 +
27 +[% endif %]
28 +### Configuration
29 +
30 +#### Options
31 +
32 +[[ entry.setup.configuration.options.description ]]
33 +
34 +[% if entry.setup.configuration.options.list %]
35 +[% if entry.setup.configuration.options.folding.enabled and not clean %]
36 +{% details open=true summary="[[ entry.setup.configuration.options.folding.title or 'Discoverer options' ]]" %}
37 +[% endif %]
38 +
39 +| Option | Description | Default | Required |
40 +|:-----|:------------|:--------|:---------:|
41 +[% for item in entry.setup.configuration.options.list %]
42 +[% set item_anchor = "option-" ~ anchorfy(item.name) %]
43 +| [[ ("[" ~ strfy(item.name) ~ "](#" ~ item_anchor ~ ")") if ('detailed_description' in item) else strfy(item.name) ]] | [[ strfy(item.description) ]] | [[ strfy(item.default_value) ]] | [[ strfy(item.required) ]] |
44 +[% endfor %]
45 +
46 +[% for item in entry.setup.configuration.options.list %]
47 +[% if 'detailed_description' in item %]
48 +<a id="[[ "option-" ~ anchorfy(item.name) ]]"></a>
49 +##### [[ item.name ]]
50 +
51 +[[ item.detailed_description ]]
52 +
53 +[% endif %]
54 +[% endfor %]
55 +
56 +[% if entry.setup.configuration.options.folding.enabled and not clean %]
57 +{% /details %}
58 +[% endif %]
59 +[% else %]
60 +There are no configuration options.
61 +
62 +[% endif %]
63 +
64 +#### via UI
65 +
66 +1. Open the Netdata Dynamic Configuration UI.
67 +2. Go to `Collectors -> go.d -> ServiceDiscovery -> [[ entry.meta.kind ]]`.
68 +3. Add a new discovery pipeline and give it a name.
69 +4. Fill in the discoverer-specific settings and the service rules.
70 +5. Save the discovery pipeline.
71 +
72 +#### via File
73 +
74 +Define the discovery pipeline in `/etc/netdata/[[ entry.setup.configuration.file.name ]]`.
75 +
76 +The file has two top-level blocks: `discoverer:` (the options above) and `services:` (rules that turn discovered targets into collector jobs — see [Service Rules](#service-rules)).
77 +
78 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
79 +
80 +##### Examples
81 +[% if entry.setup.configuration.examples.list %]
82 +
83 +[% for example in entry.setup.configuration.examples.list %]
84 +###### [[ example.name ]]
85 +
86 +[[ example.description ]]
87 +
88 +[% if example.folding is defined and example.folding.enabled and not clean %]
89 +{% details open=true summary="[[ entry.setup.configuration.examples.folding.title or 'Example configuration' ]]" %}
90 +[% endif %]
91 +```yaml
92 +[[ example.config ]]
93 +```
94 +[% if example.folding is defined and example.folding.enabled and not clean %]
95 +{% /details %}
96 +[% endif %]
97 +[% endfor %]
98 +[% else %]
99 +There are no configuration examples.
100 +
101 +[% endif %]
integrations/templates/troubleshooting.md
+5
@@ -128,6 +128,11 @@ Note that this will test _all_ alert mechanisms for the selected role.
128 [% if entry.troubleshooting.problems.list %]
129 ## Troubleshooting
130
131 +[% endif %]
132 +[% elif entry.integration_type == 'service_discovery' %]
133 +[% if entry.troubleshooting.problems.list %]
134 +## Troubleshooting
135 +
136 [% endif %]
137 [% endif %]
138 [% for item in entry.troubleshooting.problems.list %]
src/collectors/SERVICE-DISCOVERY.md new
+200
@@ -0,0 +1,200 @@
1 +# Service Discovery
2 +
3 +Stop hand-writing collector jobs for every host, container, or device. Netdata's Service Discovery (SD) finds monitorable targets in your environment and turns them into collector jobs automatically.
4 +
5 +Each SD pipeline is a `discoverer:` (where to look) plus a list of `services:` rules (how to turn what was found into collector jobs). The discoverer emits **targets**; the rules render **collector job YAML** from those targets using Go templates.
6 +
7 +### Jump To
8 +
9 +[How it works](#how-it-works) • [Configuration file structure](#configuration-file-structure) • [Rule evaluation semantics](#rule-evaluation-semantics) • [Template helper reference](#template-helper-reference) • [config_template rendering](#config_template-rendering) • [Supported discoverers](#supported-discoverers) • [Mixing discoverers](#mixing-discoverers) • [Troubleshooting](#troubleshooting)
10 +
11 +
12 +## How it works
13 +
14 +Each discovery pipeline runs in five stages:
15 +
16 +1. **Discover** — The discoverer probes its environment for monitorable things — running containers, listening sockets, kubernetes resources, SNMP devices on a subnet, items returned by an HTTP endpoint. What gets probed and how often is controlled by `discoverer:` options.
17 +2. **Build a target** — Each discovered thing becomes a target. The target carries discoverer-specific fields (variables) — for example, net_listeners targets have `.Port` / `.Comm` / `.Cmdline`; docker targets have `.Image` / `.Name` / `.Labels`; snmp targets have `.IPAddress` / `.SysInfo.*` / `.Credential.*`. Per-discoverer pages list the full variable set.
18 +3. **Match against services rules** — The rule engine evaluates each `services:` rule against each target, top to bottom. A rule's `match` expression is a Go template that must render to the literal string `"true"` for the rule to apply.
19 +4. **Render the collector job** — When a rule matches, its `config_template` is executed with the target as context, producing collector job YAML. The rendered YAML can be a single job map or a sequence of jobs.
20 +5. **Hand off to the collector** — The agent registers the rendered jobs with the matching collector module. From this point on, the collector runs the job — the discoverer is no longer involved.
21 +
22 +
23 +## Configuration file structure
24 +
25 +Each discoverer has its own file under `/etc/netdata/go.d/sd/`. The filename determines the discoverer kind (`net_listeners.conf`, `docker.conf`, `http.conf`, `snmp.conf`, `k8s.conf`).
26 +
27 +Every SD file has the same shape:
28 +
29 +```yaml
30 +disabled: no # set to yes to disable this pipeline
31 +
32 +discoverer:
33 + <kind>: # discoverer kind (must match filename)
34 + # discoverer-specific options — see the per-discoverer page
35 +
36 +services:
37 + - id: <rule-id>
38 + match: <go-template> # must render to the literal string "true"
39 + config_template: | # optional — omit to make this a "skip rule"
40 + # collector job YAML, rendered with target as context
41 +```
42 +
43 +- `disabled: yes` keeps the file on disk but turns the pipeline off.
44 +- Editing a stock file requires restarting the agent. UI-managed pipelines apply live.
45 +- Where each discoverer's stock conf ships (with the Netdata package, with the Helm chart, or not at all) is documented on its per-discoverer page.
46 +
47 +
48 +## Rule evaluation semantics
49 +
50 +For each discovered target, the engine walks the `services:` array top-to-bottom. The two rule shapes behave differently:
51 +
52 +- **Skip rule (no `config_template`)** — When a skip rule matches, the target is dropped immediately — no jobs are produced and **no further rules run for that target**. Use skip rules to exclude targets the catch-all would otherwise pick up. Place them **before** any template rule.
53 +- **Template rule (with `config_template`)** — When a template rule matches, its `config_template` is rendered into one or more collector jobs and rule evaluation **continues** with the next rule. A single target can therefore produce jobs from multiple matching template rules.
54 +
55 +### Order matters
56 +
57 +Recommended ordering for a multi-rule pipeline:
58 +
59 +1. Skip rules — drop targets you don't want monitored.
60 +2. Specific template rules — vendor-specific, label-specific, port-specific.
61 +3. A catch-all template rule (`match: '{{ true }}'`) — the default fallback.
62 +
63 +If a specific template rule already produced the right job, follow it with a skip rule keyed on the same condition to suppress the catch-all for those targets.
64 +
65 +
66 +## Template helper reference
67 +
68 +Match expressions and config templates are [Go `text/template`](https://pkg.go.dev/text/template) bodies with three additional helper sets: the standard Go template builtins, the [sprig](https://masterminds.github.io/sprig/) function library, and a small set of Netdata-specific helpers.
69 +
70 +All templates run with `missingkey=error`. Referencing a field that does not exist on the target type (e.g. typo `.SysInfoo.Name` instead of `.SysInfo.Name`) makes the template execution fail; the agent logs the error and skips that rule for that target.
71 +
72 +
73 +### Go template builtins
74 +
75 +Standard Go `text/template` syntax: pipelines, conditionals, loops, variable assignment, whitespace control.
76 +
77 +| Construct | Description |
78 +|:----------|:------------|
79 +| `if`/`else if`/`else`/`end` | Conditional rendering. |
80 +| `range`/`end` | Iterate over a slice or map. |
81 +| `with`/`end` | Set the dot to a value if it is non-empty. |
82 +| `{{- ... -}}` | Whitespace-trim left/right around the action. |
83 +| `{{ $var := ... }}` | Variable assignment, scoped to the enclosing block. |
84 +| `eq`, `ne`, `lt`, `le`, `gt`, `ge` | Comparison. `eq A B C ...` is true if `A` equals **any** of the following arguments. |
85 +| `and`, `or`, `not` | Boolean composition (variadic). |
86 +| `index` | Index a slice or map: `index .Labels "app"`. |
87 +| `printf` | Formatted string. |
88 +
89 +
90 +### Sprig functions
91 +
92 +The full [sprig library](https://masterminds.github.io/sprig/) is included. The functions most commonly used in stock SD configs:
93 +
94 +| Function | Description |
95 +|:---------|:------------|
96 +| `default DEFAULT VALUE` | Return `VALUE` if non-empty; otherwise `DEFAULT`. |
97 +| `empty VALUE` | True if the value is the zero value for its type. |
98 +| `hasKey MAP KEY` | True if `MAP` contains `KEY`. Used heavily by the `http` discoverer. |
99 +| `kindIs KIND VALUE` | True if `VALUE`'s reflect kind matches: `string`, `map`, `slice`, `bool`, … |
100 +| `lower S` / `upper S` | Lowercase / uppercase a string. |
101 +| `trim S` / `trimPrefix PREFIX S` / `trimSuffix SUFFIX S` | Whitespace and prefix/suffix trimming. |
102 +| `replace OLD NEW S` | String replace. |
103 +| `regexFind RE S` | Return the first regex match, or empty. |
104 +| `regexMatch RE S` | True if `S` matches `RE` (substring match unless anchored). |
105 +| `printf FMT V...` | Same as Go's `fmt.Sprintf`. |
106 +
107 +See the [sprig docs](https://masterminds.github.io/sprig/) for the full list (string, math, encoding, list, dict, date helpers).
108 +
109 +
110 +### Netdata helpers
111 +
112 +Custom helpers added to the SD template engine:
113 +
114 +- `match TYPE VALUE PATTERN [PATTERN...]` — Returns the string `"true"` if `VALUE` matches **any** of the patterns under the named matcher type. `TYPE` is one of:
115 + - `"glob"` — shell glob (`*`, `?`, `[abc]`).
116 + - `"sp"` — Netdata simple patterns (space-separated, `*` wildcard, `!` for negation).
117 + - `"re"` — RE2 regular expression. Matches if the regex matches anywhere in `VALUE` unless explicitly anchored with `^` / `$`.
118 + - `"dstar"` — [doublestar](https://github.com/bmatcuk/doublestar) glob (supports `**` for path-style matching).
119 +- `glob VALUE PATTERN [PATTERN...]` — Shortcut for `match "glob" VALUE PATTERN...`.
120 +- `promPort PORT` — Returns the well-known Prometheus exporter module name registered for `PORT`, or the empty string. `net_listeners`-specific.
121 +- `toYaml VALUE` — Serialize `VALUE` as a YAML string. Used by `http` discoverer rules that pass through items as collector job configs.
122 +
123 +**Notes:**
124 +
125 +- `match` and `glob` are case-sensitive.
126 +- `match "re" ...` is unanchored. Use `^...$` to require a full-string match.
127 +- There is **no** standalone `regexp` or `regex` helper. Use `match "re"` (or sprig's `regexFind` / `regexMatch`) for regex matching.
128 +- `match` and `glob` return the **string** `"true"` or `"false"`, not a Go bool. The rule engine compares the trimmed template output against the literal `"true"`, so a top-level `match: '{{ glob .X "foo*" }}'` works directly.
129 +- **Composing with `if` / `and` / `or` is a footgun**: in Go templates a non-empty string is truthy, so both `"true"` *and* `"false"` evaluate truthy under `if`. `{{ if glob .X "foo*" }}` is therefore **wrong**. Either wrap each result with `eq ... "true"` to get a real bool, or use an explicit `if-then-true` block:
130 + ```
131 + match: '{{ if and (eq (glob .Vendor "Cisco*") "true") (eq .Category "router") }}true{{ end }}'
132 + ```
133 +
134 +
135 +## config_template rendering
136 +
137 +When a template rule matches, its `config_template` is executed with the target as the dot context. The rendered output is parsed as YAML to produce one or more collector jobs.
138 +
139 +- **Single map → one job** — If the rendered YAML is a map, a single collector job is created.
140 +- **Sequence of maps → one job per element** — If the rendered YAML is a sequence (top-level YAML `-` items), one job is created per element. Use this for multi-job rules. Example: a `net_listeners` rule that emits both a TCP and a Unix-socket MySQL job from the same target —
141 +
142 + ```yaml
143 + - id: mysql
144 + match: '{{ or (eq .Port "3306") (eq .Comm "mysqld") }}'
145 + config_template: |
146 + - name: local
147 + dsn: netdata@unix(/var/run/mysqld/mysqld.sock)/
148 + - name: local
149 + dsn: netdata@tcp({{.Address}})/
150 + ```
151 +- **Module inference from rule `id`** — If the rendered job map has no `module:` key, the rule's `id` is used as the module name. Set `id: snmp` (or `docker`, `http`, …) to omit `module:` from your template; otherwise include `module:` explicitly.
152 +- **`id` uniqueness** — Rule IDs are not required to be unique across rules — multiple rules can share an `id`. The `id` is used for module inference and shows up in agent logs to help you identify which rule produced a job. Pick descriptive IDs (`cisco`, `hp-printers`, `skip-vips`).
153 +- **Failure handling** — Render errors, YAML parse errors, and `missingkey=error` failures are logged at warn level. The agent skips the rule for that target and continues evaluation.
154 +
155 +
156 +## Supported discoverers
157 +
158 +Each discoverer has its own page covering its options, target variables, evaluation specifics, and worked examples.
159 +
160 +| Discoverer | Kind | Stock conf | Discovers |
161 +|:-----------|:-----|:-----------|:----------|
162 +| [Docker](/src/go/plugin/go.d/discovery/sdext/discoverer/dockersd/README.md) | `docker` | `/etc/netdata/go.d/sd/docker.conf` | Running containers on the local Docker daemon. |
163 +| [HTTP endpoint](/src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/README.md) | `http` | `/etc/netdata/go.d/sd/http.conf` | Items returned by an HTTP/HTTPS endpoint (JSON or YAML). |
164 +| [Kubernetes](/src/go/plugin/go.d/discovery/sdext/discoverer/k8ssd/README.md) | `k8s` | `/etc/netdata/go.d/sd/k8s.conf` | Pods and services in a Kubernetes cluster. |
165 +| [Local listening processes](/src/go/plugin/go.d/discovery/sdext/discoverer/netlistensd/README.md) | `net_listeners` | `/etc/netdata/go.d/sd/net_listeners.conf` | Local processes that listen on TCP/UDP ports. |
166 +| [SNMP](/src/go/plugin/go.d/discovery/sdext/discoverer/snmpsd/README.md) | `snmp` | `/etc/netdata/go.d/sd/snmp.conf` | SNMP-capable devices on configured network subnets. |
167 +
168 +
169 +## Mixing discoverers
170 +
171 +All discoverers can run simultaneously. Each `/etc/netdata/go.d/sd/<kind>.conf` is independent. The same target can theoretically be discovered by more than one discoverer (for example, a containerised application appears in both `docker` and `net_listeners`); each discoverer's pipeline is independent and may produce its own job.
172 +
173 +Use `disabled: yes` at the top of a stock file to keep it on disk but turn the pipeline off.
174 +
175 +UI-managed pipelines and file-based pipelines coexist. UI-managed pipelines apply live; file-based pipelines require an agent restart to reload.
176 +
177 +
178 +## Troubleshooting
179 +
180 +Common cross-discoverer problems. For discoverer-specific issues, see the per-discoverer page.
181 +
182 +### No targets discovered
183 +
184 +Check the agent log for `discoverer=<kind>` lines. Confirm `disabled: no` and that the discoverer's prerequisites are met (network reachability, credentials, API access).
185 +
186 +### Targets discovered but no jobs created
187 +
188 +Check that at least one `services:` rule has both a matching `match` expression and a `config_template`. A rule with no `config_template` is a skip rule — it drops the target instead of producing a job.
189 +
190 +### `match` always evaluates false
191 +
192 +Match expressions must render to the literal string `"true"`. A bare `{{ if ... }}` block that does not output anything renders to the empty string, which is treated as false. Use `{{ true }}` for catch-all, `{{ glob .X "..." }}` for pattern matches, or `{{ if ... }}true{{ end }}` for ad-hoc conditions.
193 +
194 +### Template render error
195 +
196 +Look for `failed to execute services[N]->config_template on target` in the log. The most common cause is a typo in a variable reference (e.g. `.SysInfoo.Name`) — `missingkey=error` rejects unknown fields. Use the per-discoverer page's variable table to verify spelling.
197 +
198 +### YAML parse error after rendering
199 +
200 +`failed to parse services[N] template data` means the rendered output is not valid YAML. Common cause: a discovered string field contains a colon, hash, or other YAML special character. YAML-quote dynamic values (`name: "{{ .X }}"`) when they may be irregular.
src/go/plugin/go.d/discovery/sdext/discoverer/dockersd/README.md new
+1
@@ -0,0 +1 @@
1 +integrations/docker.md
\ No newline at end of file
src/go/plugin/go.d/discovery/sdext/discoverer/dockersd/integrations/docker.md new
+320
@@ -0,0 +1,320 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/dockersd/README.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/dockersd/metadata.yaml"
4 +sidebar_label: "Docker"
5 +learn_status: "Published"
6 +learn_rel_path: "Collecting Metrics/Service Discovery"
7 +keywords: ['service discovery', 'sd', 'docker', 'containers', 'discovery']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE SERVICE DISCOVERY DISCOVERER'S metadata.yaml FILE"
9 +endmeta-->
10 +
11 +# Docker discovery
12 +
13 +
14 +<img src="https://netdata.cloud/img/docker.svg" width="150"/>
15 +
16 +
17 +Kind: `docker`
18 +
19 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
20 +
21 +## Overview
22 +
23 +Netdata can automatically discover running Docker containers on the local Docker daemon and generate collector jobs for the services running inside them. The discoverer queries the Docker API on a fixed interval, builds one target per container port, and applies your `services:` rules to render collector job YAML — typically picking the right go.d module from the container image (nginx, postgres, redis, …).
24 +
25 +This page covers Docker-specific setup. For the broader Service Discovery model and the shared template-helper reference, see [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md).
26 +
27 +
28 +### How it works
29 +
30 +Each discovery cycle, the discoverer:
31 +
32 +1. **Calls** `ContainerList` on the Docker API at the configured `address`.
33 +2. **Builds one target per `(container, network, port)` triple** for every container that has at least one network and at least one published port. Containers running in `network: host` mode are intentionally skipped — those are picked up by the [`net_listeners`](https://github.com/netdata/netdata/blob/master/src/go/plugin/go.d/discovery/sdext/discoverer/netlistensd/README.md) discoverer instead.
34 +3. **Exposes** target fields: `.Name`, `.Image`, `.Command`, `.Labels`, `.PrivatePort`, `.PublicPort`, `.PublicPortIP`, `.PortProtocol`, `.NetworkMode`, `.NetworkDriver`, `.IPAddress`, `.Address` (the convenience `IPAddress:PrivatePort`).
35 +4. **Runs the `services:` rules** against each target. The default stock conf carries curated rules for ~40 popular images (nginx, postgres, redis, rabbitmq, etc.) keyed on `.Image` patterns.
36 +5. **Reconciles** disappeared containers — when a container exits, its target is removed and the corresponding collector job stops on the next reconcile.
37 +
38 +
39 +### Limitations
40 +
41 +- Containers in **`network: host` mode** are not produced as Docker targets. Configure the `net_listeners` discoverer to pick them up via the host's process table.
42 +- Only **TCP** ports are typically useful; the stock conf's first rule explicitly skips non-TCP, missing-port, and IPv6-mapped entries.
43 +- Only **published ports** appear as targets. A container that exposes ports only inside a Docker network without `-p` mapping still produces a target via its private port and network IP.
44 +- The discoverer reads the live container list; it does not inspect image manifests, healthcheck output, or process tables inside the container. Anything beyond labels/image/ports must be inferred via service rules.
45 +- Only the **local Docker daemon** is supported (Unix socket or TCP). There is no docker-swarm or remote-cluster discovery mode.
46 +
47 +
48 +## Setup
49 +
50 +You can configure the `docker` discoverer in two ways:
51 +
52 +| Method | Best for | How to |
53 +|:--|:--|:--|
54 +| [**UI**](#via-ui) | Fast setup without editing files | Go to `Collectors -> go.d -> ServiceDiscovery -> docker`, then add a discovery pipeline. |
55 +| [**File**](#via-file) | File-based configuration or automation | Edit `/etc/netdata/go.d/sd/docker.conf` and define the `discoverer:` and `services:` blocks. |
56 +
57 +### Prerequisites
58 +
59 +#### Access to the Docker socket
60 +
61 +The Netdata Agent must be able to reach the Docker daemon. The default `address` is `unix:///var/run/docker.sock`. If you run Netdata in a container, mount the socket: `-v /var/run/docker.sock:/var/run/docker.sock:ro`. The Netdata user (or the container) must have read access to the socket.
62 +
63 +
64 +#### Discovery is enabled by default
65 +
66 +The stock conf at `/etc/netdata/go.d/sd/docker.conf` ships with `disabled: no` and a curated set of `services:` rules covering ~40 popular images. To turn discovery off, set `disabled: yes` at the top of the file.
67 +
68 +
69 +### Configuration
70 +
71 +#### Options
72 +
73 +The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn discovered containers into collector jobs — see [Service Rules](#service-rules)).
74 +
75 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
76 +
77 +
78 +
79 +| Option | Description | Default | Required |
80 +|:-----|:------------|:--------|:---------:|
81 +| [address](#option-address) | Docker daemon address. | unix:///var/run/docker.sock | no |
82 +| timeout | Maximum time to wait for a Docker API response (per request). | 2s | no |
83 +
84 +<a id="option-address"></a>
85 +##### address
86 +
87 +Supports both Unix-socket (`unix:///var/run/docker.sock`) and TCP (`tcp://hostname:2375`) endpoints.
88 +
89 +If unset, Netdata also honors the `DOCKER_HOST` environment variable when present.
90 +
91 +
92 +
93 +
94 +#### via UI
95 +
96 +1. Open the Netdata Dynamic Configuration UI.
97 +2. Go to `Collectors -> go.d -> ServiceDiscovery -> docker`.
98 +3. Add a new discovery pipeline and give it a name.
99 +4. Fill in the discoverer-specific settings and the service rules.
100 +5. Save the discovery pipeline.
101 +
102 +#### via File
103 +
104 +Define the discovery pipeline in `/etc/netdata/go.d/sd/docker.conf`.
105 +
106 +The file has two top-level blocks: `discoverer:` (the options above) and `services:` (rules that turn discovered targets into collector jobs — see [Service Rules](#service-rules)).
107 +
108 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
109 +
110 +##### Examples
111 +
112 +###### Default (Unix socket)
113 +
114 +Use the default local Docker socket and the stock services rules.
115 +
116 +```yaml
117 +disabled: no
118 +discoverer:
119 + docker:
120 + address: unix:///var/run/docker.sock
121 +services:
122 + # See the stock conf for the full curated rule set.
123 + - id: skip
124 + match: |
125 + {{ or (eq .NetworkMode "host") (not (eq .PortProtocol "tcp")) (empty .PrivatePort) }}
126 + - id: nginx
127 + match: '{{ match "sp" .Image "nginx nginx:*" }}'
128 + config_template: |
129 + name: docker_{{.Name}}
130 + url: http://{{.Address}}/stub_status
131 +
132 +```
133 +###### Remote daemon over TCP
134 +
135 +Point the discoverer at a remote Docker daemon. TLS is not yet wired into the discoverer; either expose the daemon on a trusted internal network or use a stunnel/socat proxy.
136 +
137 +```yaml
138 +disabled: no
139 +discoverer:
140 + docker:
141 + address: tcp://docker.internal:2375
142 + timeout: 5s
143 +services:
144 + - id: skip
145 + match: '{{ or (eq .NetworkMode "host") (not (eq .PortProtocol "tcp")) (empty .PrivatePort) }}'
146 + - id: redis
147 + match: '{{ match "sp" .Image "redis redis:* */redis */redis:*" }}'
148 + config_template: |
149 + name: docker_{{.Name}}
150 + address: redis://@{{.Address}}
151 +
152 +```
153 +
154 +
155 +## Service Rules
156 +
157 +A `services:` rule turns each discovered container target into one or more collector jobs. Most rules match on `.Image` (using the `match "sp"` simple-pattern helper for the typical `image image:* */image */image:*` family), some also gate on `.PrivatePort`, and a few use `.Labels` to honor user intent.
158 +
159 +The shared rule model — function reference (`match`, `glob`, sprig, `toYaml`), `config_template` rendering rules, and the `missingkey=error` failure semantics — lives on the [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are Docker-specific.
160 +
161 +
162 +### How rules are evaluated
163 +
164 +Quick reference — see [Rule evaluation semantics](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model.
165 +
166 +
167 +
168 +- **The first rule in the stock conf is a skip rule** — It drops targets that are unreachable or uninteresting (host networking, non-TCP, missing port, IPv6-mapped public IP). Keep it as the first rule — every subsequent rule assumes it has filtered out the noise.
169 +- **Match on .Image with `match "sp"`** — The simple-patterns matcher (`match "sp" .Image "nginx nginx:* */nginx */nginx:*"`) is the idiomatic way to handle the four-form image family (bare, tagged, namespaced, namespaced-tagged). Use `glob` if you only need shell-style globbing without the simple-patterns engine.
170 +- **Module inference from rule id** — For Docker, set `id: <module-name>` (e.g. `id: nginx`) so the rendered job inherits the module name automatically. Use a different `id` only when you also include `module:` explicitly in the template.
171 +
172 +### Template Variables
173 +
174 +Available inside both `match` expressions and `config_template` bodies for Docker targets.
175 +
176 +
177 +| Variable | Type | Description |
178 +|:---------|:-----|:------------|
179 +| `.Name` | string | Container name (without the leading slash). |
180 +| `.Image` | string | Container image as reported by Docker (e.g. `nginx:1.25`, `myorg/redis:6`). |
181 +| `.Command` | string | Container command line. |
182 +| `.Labels` | map | All container labels. Use `index .Labels "key"` or `hasKey .Labels "key"` to read individual entries. |
183 +| `.IPAddress` | string | IP of the container on the matched network. |
184 +| `.Address` | string | Convenience `IPAddress:PrivatePort` — the canonical address used in most stock rule templates. |
185 +| `.PrivatePort` | string | Container-side port. |
186 +| `.PublicPort` | string | Host-side port (empty when the container does not publish a host mapping for this port). |
187 +| `.PublicPortIP` | string | Host IP that the container port is bound to (empty when no public mapping). |
188 +| `.PortProtocol` | string | Port protocol — `tcp` or `udp`. Stock rules typically gate on `eq .PortProtocol "tcp"`. |
189 +| `.NetworkMode` | string | Container network mode (`bridge`, `host`, `overlay`, custom network names, …). The stock skip rule drops `host` mode. |
190 +| `.NetworkDriver` | string | Driver of the matched network. |
191 +| `.ID` | string | Container ID (full hex). |
192 +
193 +### Examples
194 +
195 +Each example shows one or more entries from the `services:` array. Order matters — see [How rules are evaluated](#how-rules-are-evaluated).
196 +
197 +#### Skip rule for unreachable / uninteresting targets
198 +
199 +The first rule in the stock conf. Drops `host` networking (those are local-listener targets), non-TCP ports, ports without a private side, and IPv6-mapped public IPs. Place it first.
200 +
201 +```yaml
202 +- id: skip
203 + match: |
204 + {{ $netNOK := eq .NetworkMode "host" -}}
205 + {{ $protoNOK := not (eq .PortProtocol "tcp") -}}
206 + {{ $portNOK := empty .PrivatePort -}}
207 + {{ $addrNOK := or (empty .IPAddress) (glob .PublicPortIP "*:*") -}}
208 + {{ or $netNOK $protoNOK $portNOK $addrNOK }}
209 +
210 +```
211 +
212 +#### Nginx — module inferred from rule id
213 +
214 +Match the four common image-name forms with `match "sp"`. `id: nginx` makes the module name infer to `nginx` automatically — no `module:` line needed in the template.
215 +
216 +
217 +```yaml
218 +- id: nginx
219 + match: '{{ match "sp" .Image "nginx nginx:*" }}'
220 + config_template: |
221 + - name: docker_{{.Name}}
222 + url: http://{{.Address}}/stub_status
223 + - name: docker_{{.Name}}
224 + url: http://{{.Address}}/basic_status
225 + - name: docker_{{.Name}}
226 + url: http://{{.Address}}/nginx_status
227 + - name: docker_{{.Name}}
228 + url: http://{{.Address}}/status
229 +
230 +```
231 +
232 +#### Postgres — explicit module override
233 +
234 +When the rule `id` is something other than the target module name, set `module:` explicitly inside the rendered job.
235 +
236 +
237 +```yaml
238 +- id: postgres
239 + match: '{{ or (eq .PrivatePort "5432") (match "sp" .Image "postgres postgres:* */postgres */postgres:* */postgresql */postgresql:*") }}'
240 + config_template: |
241 + module: postgres
242 + name: docker_{{.Name}}
243 + dsn: postgres://netdata:postgres@{{.Address}}/postgres
244 +
245 +```
246 +
247 +#### Label-driven custom matching
248 +
249 +Use container labels to override behaviour without changing rules — e.g. opt a container in or out of monitoring, or pick a non-default endpoint. The example below requires the operator to set the label `netdata.go.d/module=mymodule` on the container.
250 +
251 +
252 +```yaml
253 +- id: label-routed
254 + match: '{{ and (hasKey .Labels "netdata.go.d/module") (eq (index .Labels "netdata.go.d/module") "mymodule") }}'
255 + config_template: |
256 + module: mymodule
257 + name: docker_{{.Name}}
258 + url: http://{{.Address}}/metrics
259 +
260 +```
261 +
262 +
263 +
264 +## Verify discovery worked
265 +
266 +After enabling the discoverer, confirm it is finding containers and producing jobs.
267 +
268 +### Confirm containers are being listed
269 +
270 +Watch the agent log for Docker discoverer messages. With systemd:
271 +
272 +```bash
273 +journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep "discoverer=docker"
274 +```
275 +
276 +On a healthy daemon you should see the agent successfully calling `ContainerList`. If the log shows `error on creating docker client` or permission errors, the agent cannot reach `/var/run/docker.sock`.
277 +
278 +
279 +### Confirm jobs are being created
280 +
281 +In the Netdata UI go to `Collectors -> go.d -> <module>` for whatever modules your service rules target (nginx, redis, postgres, …) — each container that matched a rule should appear as a `docker_<container-name>` job.
282 +
283 +
284 +### Confirm metrics are being collected
285 +
286 +If a job was created but no charts appear, the rendered `config_template` produced a config the collector module rejected (wrong DSN, unreachable URL, missing credential). Check the collector's log.
287 +
288 +
289 +
290 +
291 +## Troubleshooting
292 +
293 +### Permission denied on docker.sock
294 +
295 +The Netdata user must be able to read the Docker socket. On a typical Linux host:
296 +
297 +```bash
298 +sudo usermod -aG docker netdata
299 +systemctl restart netdata
300 +```
301 +
302 +In containers, mount the socket read-only and verify the file is readable from inside.
303 +
304 +
305 +### No targets discovered for containers in `host` networking
306 +
307 +`host`-mode containers are intentionally skipped by the Docker discoverer. Enable the `net_listeners` discoverer instead — it picks up locally-listening processes, which includes `host`-mode containers.
308 +
309 +
310 +### Wrong module picked for an image
311 +
312 +Stock rules match on `.Image` patterns. Custom forks or in-house image names won't match. Add a rule above the stock catch-alls keyed on your own image name (`match "sp" .Image "myorg/nginx myorg/nginx:*"`) or use a `.Labels`-driven rule.
313 +
314 +
315 +### Generated jobs fail to start
316 +
317 +Common causes: the rendered URL is not reachable from the agent (different network, firewall); credentials baked into the template are wrong; the module's port is not the one Docker reported. Check the rendered job YAML in the agent's debug output.
318 +
319 +
320 +
src/go/plugin/go.d/discovery/sdext/discoverer/dockersd/metadata.yaml new
+252
@@ -0,0 +1,252 @@
1 +# yamllint disable rule:line-length
2 +---
3 +id: 'service-discovery-docker'
4 +meta:
5 + kind: 'docker'
6 + name: 'Docker'
7 + tagline: 'Running containers on the local Docker daemon.'
8 + link: 'https://www.docker.com/'
9 + icon_filename: 'docker.svg'
10 +keywords:
11 + - 'service discovery'
12 + - 'sd'
13 + - 'docker'
14 + - 'containers'
15 + - 'discovery'
16 +overview:
17 + description: |
18 + Netdata can automatically discover running Docker containers on the local Docker daemon and generate collector jobs for the services running inside them. The discoverer queries the Docker API on a fixed interval, builds one target per container port, and applies your `services:` rules to render collector job YAML — typically picking the right go.d module from the container image (nginx, postgres, redis, …).
19 +
20 + This page covers Docker-specific setup. For the broader Service Discovery model and the shared template-helper reference, see [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md).
21 + how_it_works: |
22 + Each discovery cycle, the discoverer:
23 +
24 + 1. **Calls** `ContainerList` on the Docker API at the configured `address`.
25 + 2. **Builds one target per `(container, network, port)` triple** for every container that has at least one network and at least one published port. Containers running in `network: host` mode are intentionally skipped — those are picked up by the [`net_listeners`](/src/go/plugin/go.d/discovery/sdext/discoverer/netlistensd/README.md) discoverer instead.
26 + 3. **Exposes** target fields: `.Name`, `.Image`, `.Command`, `.Labels`, `.PrivatePort`, `.PublicPort`, `.PublicPortIP`, `.PortProtocol`, `.NetworkMode`, `.NetworkDriver`, `.IPAddress`, `.Address` (the convenience `IPAddress:PrivatePort`).
27 + 4. **Runs the `services:` rules** against each target. The default stock conf carries curated rules for ~40 popular images (nginx, postgres, redis, rabbitmq, etc.) keyed on `.Image` patterns.
28 + 5. **Reconciles** disappeared containers — when a container exits, its target is removed and the corresponding collector job stops on the next reconcile.
29 + limitations: |
30 + - Containers in **`network: host` mode** are not produced as Docker targets. Configure the `net_listeners` discoverer to pick them up via the host's process table.
31 + - Only **TCP** ports are typically useful; the stock conf's first rule explicitly skips non-TCP, missing-port, and IPv6-mapped entries.
32 + - Only **published ports** appear as targets. A container that exposes ports only inside a Docker network without `-p` mapping still produces a target via its private port and network IP.
33 + - The discoverer reads the live container list; it does not inspect image manifests, healthcheck output, or process tables inside the container. Anything beyond labels/image/ports must be inferred via service rules.
34 + - Only the **local Docker daemon** is supported (Unix socket or TCP). There is no docker-swarm or remote-cluster discovery mode.
35 +setup:
36 + prerequisites:
37 + list:
38 + - title: 'Access to the Docker socket'
39 + description: |
40 + The Netdata Agent must be able to reach the Docker daemon. The default `address` is `unix:///var/run/docker.sock`. If you run Netdata in a container, mount the socket: `-v /var/run/docker.sock:/var/run/docker.sock:ro`. The Netdata user (or the container) must have read access to the socket.
41 + - title: 'Discovery is enabled by default'
42 + description: |
43 + The stock conf at `/etc/netdata/go.d/sd/docker.conf` ships with `disabled: no` and a curated set of `services:` rules covering ~40 popular images. To turn discovery off, set `disabled: yes` at the top of the file.
44 + configuration:
45 + file:
46 + name: 'go.d/sd/docker.conf'
47 + options:
48 + description: |
49 + The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn discovered containers into collector jobs — see [Service Rules](#service-rules)).
50 +
51 + After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
52 + folding:
53 + title: 'Discoverer options'
54 + enabled: false
55 + list:
56 + - name: 'address'
57 + description: 'Docker daemon address.'
58 + default_value: 'unix:///var/run/docker.sock'
59 + required: false
60 + detailed_description: |
61 + Supports both Unix-socket (`unix:///var/run/docker.sock`) and TCP (`tcp://hostname:2375`) endpoints.
62 +
63 + If unset, Netdata also honors the `DOCKER_HOST` environment variable when present.
64 + - name: 'timeout'
65 + description: 'Maximum time to wait for a Docker API response (per request).'
66 + default_value: '2s'
67 + required: false
68 + examples:
69 + folding:
70 + title: 'Configuration examples'
71 + enabled: true
72 + list:
73 + - name: 'Default (Unix socket)'
74 + description: 'Use the default local Docker socket and the stock services rules.'
75 + config: |
76 + disabled: no
77 + discoverer:
78 + docker:
79 + address: unix:///var/run/docker.sock
80 + services:
81 + # See the stock conf for the full curated rule set.
82 + - id: skip
83 + match: |
84 + {{ or (eq .NetworkMode "host") (not (eq .PortProtocol "tcp")) (empty .PrivatePort) }}
85 + - id: nginx
86 + match: '{{ match "sp" .Image "nginx nginx:*" }}'
87 + config_template: |
88 + name: docker_{{.Name}}
89 + url: http://{{.Address}}/stub_status
90 + - name: 'Remote daemon over TCP'
91 + description: 'Point the discoverer at a remote Docker daemon. TLS is not yet wired into the discoverer; either expose the daemon on a trusted internal network or use a stunnel/socat proxy.'
92 + config: |
93 + disabled: no
94 + discoverer:
95 + docker:
96 + address: tcp://docker.internal:2375
97 + timeout: 5s
98 + services:
99 + - id: skip
100 + match: '{{ or (eq .NetworkMode "host") (not (eq .PortProtocol "tcp")) (empty .PrivatePort) }}'
101 + - id: redis
102 + match: '{{ match "sp" .Image "redis redis:* */redis */redis:*" }}'
103 + config_template: |
104 + name: docker_{{.Name}}
105 + address: redis://@{{.Address}}
106 +services:
107 + description: |
108 + A `services:` rule turns each discovered container target into one or more collector jobs. Most rules match on `.Image` (using the `match "sp"` simple-pattern helper for the typical `image image:* */image */image:*` family), some also gate on `.PrivatePort`, and a few use `.Labels` to honor user intent.
109 +
110 + The shared rule model — function reference (`match`, `glob`, sprig, `toYaml`), `config_template` rendering rules, and the `missingkey=error` failure semantics — lives on the [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are Docker-specific.
111 + evaluation:
112 + description: |
113 + Quick reference — see [Rule evaluation semantics](/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model.
114 + list:
115 + - name: 'The first rule in the stock conf is a skip rule'
116 + description: 'It drops targets that are unreachable or uninteresting (host networking, non-TCP, missing port, IPv6-mapped public IP). Keep it as the first rule — every subsequent rule assumes it has filtered out the noise.'
117 + - name: 'Match on .Image with `match "sp"`'
118 + description: |
119 + The simple-patterns matcher (`match "sp" .Image "nginx nginx:* */nginx */nginx:*"`) is the idiomatic way to handle the four-form image family (bare, tagged, namespaced, namespaced-tagged). Use `glob` if you only need shell-style globbing without the simple-patterns engine.
120 + - name: 'Module inference from rule id'
121 + description: 'For Docker, set `id: <module-name>` (e.g. `id: nginx`) so the rendered job inherits the module name automatically. Use a different `id` only when you also include `module:` explicitly in the template.'
122 + template_variables:
123 + description: 'Available inside both `match` expressions and `config_template` bodies for Docker targets.'
124 + list:
125 + - name: '.Name'
126 + type: 'string'
127 + description: 'Container name (without the leading slash).'
128 + - name: '.Image'
129 + type: 'string'
130 + description: 'Container image as reported by Docker (e.g. `nginx:1.25`, `myorg/redis:6`).'
131 + - name: '.Command'
132 + type: 'string'
133 + description: 'Container command line.'
134 + - name: '.Labels'
135 + type: 'map'
136 + description: 'All container labels. Use `index .Labels "key"` or `hasKey .Labels "key"` to read individual entries.'
137 + - name: '.IPAddress'
138 + type: 'string'
139 + description: 'IP of the container on the matched network.'
140 + - name: '.Address'
141 + type: 'string'
142 + description: 'Convenience `IPAddress:PrivatePort` — the canonical address used in most stock rule templates.'
143 + - name: '.PrivatePort'
144 + type: 'string'
145 + description: 'Container-side port.'
146 + - name: '.PublicPort'
147 + type: 'string'
148 + description: 'Host-side port (empty when the container does not publish a host mapping for this port).'
149 + - name: '.PublicPortIP'
150 + type: 'string'
151 + description: 'Host IP that the container port is bound to (empty when no public mapping).'
152 + - name: '.PortProtocol'
153 + type: 'string'
154 + description: 'Port protocol — `tcp` or `udp`. Stock rules typically gate on `eq .PortProtocol "tcp"`.'
155 + - name: '.NetworkMode'
156 + type: 'string'
157 + description: 'Container network mode (`bridge`, `host`, `overlay`, custom network names, …). The stock skip rule drops `host` mode.'
158 + - name: '.NetworkDriver'
159 + type: 'string'
160 + description: 'Driver of the matched network.'
161 + - name: '.ID'
162 + type: 'string'
163 + description: 'Container ID (full hex).'
164 + examples:
165 + description: 'Each example shows one or more entries from the `services:` array. Order matters — see [How rules are evaluated](#how-rules-are-evaluated).'
166 + list:
167 + - name: 'Skip rule for unreachable / uninteresting targets'
168 + description: 'The first rule in the stock conf. Drops `host` networking (those are local-listener targets), non-TCP ports, ports without a private side, and IPv6-mapped public IPs. Place it first.'
169 + config: |
170 + - id: skip
171 + match: |
172 + {{ $netNOK := eq .NetworkMode "host" -}}
173 + {{ $protoNOK := not (eq .PortProtocol "tcp") -}}
174 + {{ $portNOK := empty .PrivatePort -}}
175 + {{ $addrNOK := or (empty .IPAddress) (glob .PublicPortIP "*:*") -}}
176 + {{ or $netNOK $protoNOK $portNOK $addrNOK }}
177 + - name: 'Nginx — module inferred from rule id'
178 + description: |
179 + Match the four common image-name forms with `match "sp"`. `id: nginx` makes the module name infer to `nginx` automatically — no `module:` line needed in the template.
180 + config: |
181 + - id: nginx
182 + match: '{{ match "sp" .Image "nginx nginx:*" }}'
183 + config_template: |
184 + - name: docker_{{.Name}}
185 + url: http://{{.Address}}/stub_status
186 + - name: docker_{{.Name}}
187 + url: http://{{.Address}}/basic_status
188 + - name: docker_{{.Name}}
189 + url: http://{{.Address}}/nginx_status
190 + - name: docker_{{.Name}}
191 + url: http://{{.Address}}/status
192 + - name: 'Postgres — explicit module override'
193 + description: |
194 + When the rule `id` is something other than the target module name, set `module:` explicitly inside the rendered job.
195 + config: |
196 + - id: postgres
197 + match: '{{ or (eq .PrivatePort "5432") (match "sp" .Image "postgres postgres:* */postgres */postgres:* */postgresql */postgresql:*") }}'
198 + config_template: |
199 + module: postgres
200 + name: docker_{{.Name}}
201 + dsn: postgres://netdata:postgres@{{.Address}}/postgres
202 + - name: 'Label-driven custom matching'
203 + description: |
204 + Use container labels to override behaviour without changing rules — e.g. opt a container in or out of monitoring, or pick a non-default endpoint. The example below requires the operator to set the label `netdata.go.d/module=mymodule` on the container.
205 + config: |
206 + - id: label-routed
207 + match: '{{ and (hasKey .Labels "netdata.go.d/module") (eq (index .Labels "netdata.go.d/module") "mymodule") }}'
208 + config_template: |
209 + module: mymodule
210 + name: docker_{{.Name}}
211 + url: http://{{.Address}}/metrics
212 +verify:
213 + description: 'After enabling the discoverer, confirm it is finding containers and producing jobs.'
214 + checks:
215 + list:
216 + - name: 'Confirm containers are being listed'
217 + description: |
218 + Watch the agent log for Docker discoverer messages. With systemd:
219 +
220 + ```bash
221 + journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep "discoverer=docker"
222 + ```
223 +
224 + On a healthy daemon you should see the agent successfully calling `ContainerList`. If the log shows `error on creating docker client` or permission errors, the agent cannot reach `/var/run/docker.sock`.
225 + - name: 'Confirm jobs are being created'
226 + description: |
227 + In the Netdata UI go to `Collectors -> go.d -> <module>` for whatever modules your service rules target (nginx, redis, postgres, …) — each container that matched a rule should appear as a `docker_<container-name>` job.
228 + - name: 'Confirm metrics are being collected'
229 + description: |
230 + If a job was created but no charts appear, the rendered `config_template` produced a config the collector module rejected (wrong DSN, unreachable URL, missing credential). Check the collector's log.
231 +troubleshooting:
232 + problems:
233 + list:
234 + - name: 'Permission denied on docker.sock'
235 + description: |
236 + The Netdata user must be able to read the Docker socket. On a typical Linux host:
237 +
238 + ```bash
239 + sudo usermod -aG docker netdata
240 + systemctl restart netdata
241 + ```
242 +
243 + In containers, mount the socket read-only and verify the file is readable from inside.
244 + - name: 'No targets discovered for containers in `host` networking'
245 + description: |
246 + `host`-mode containers are intentionally skipped by the Docker discoverer. Enable the `net_listeners` discoverer instead — it picks up locally-listening processes, which includes `host`-mode containers.
247 + - name: 'Wrong module picked for an image'
248 + description: |
249 + Stock rules match on `.Image` patterns. Custom forks or in-house image names won't match. Add a rule above the stock catch-alls keyed on your own image name (`match "sp" .Image "myorg/nginx myorg/nginx:*"`) or use a `.Labels`-driven rule.
250 + - name: 'Generated jobs fail to start'
251 + description: |
252 + Common causes: the rendered URL is not reachable from the agent (different network, firewall); credentials baked into the template are wrong; the module's port is not the one Docker reported. Check the rendered job YAML in the agent's debug output.
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/README.md new
+1
@@ -0,0 +1 @@
1 +integrations/http.md
\ No newline at end of file
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/integrations/http.md new
+360
@@ -0,0 +1,360 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/README.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/metadata.yaml"
4 +sidebar_label: "HTTP endpoint"
5 +learn_status: "Published"
6 +learn_rel_path: "Collecting Metrics/Service Discovery"
7 +keywords: ['service discovery', 'sd', 'http', 'rest', 'discovery', 'cmdb']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE SERVICE DISCOVERY DISCOVERER'S metadata.yaml FILE"
9 +endmeta-->
10 +
11 +# HTTP endpoint discovery
12 +
13 +
14 +<img src="https://netdata.cloud/img/http.svg" width="150"/>
15 +
16 +
17 +Kind: `http`
18 +
19 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
20 +
21 +## Overview
22 +
23 +Netdata can pull a list of monitorable targets from any HTTP endpoint you control — a CMDB API, an internal asset registry, a static file served by nginx, or a Prometheus-style file_sd export. The discoverer fetches the endpoint, decodes JSON or YAML, and feeds each item into the `services:` rule engine. This is the "bring your own source-of-truth" discoverer.
24 +
25 +This page covers HTTP-specific setup. For the broader Service Discovery model and the shared template-helper reference, see [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md).
26 +
27 +
28 +### How it works
29 +
30 +Each discovery cycle, the discoverer:
31 +
32 +1. **Fetches** the configured `url` over HTTP/HTTPS, honouring all standard go.d collector HTTP options (auth, headers, TLS, proxy, timeout).
33 +2. **Decodes** the response as either JSON or YAML according to `format` (auto / json / yaml). With `format: auto`, the decoder uses `Content-Type` if it is unambiguous, otherwise tries JSON first then YAML.
34 +3. **Accepts two shapes** at the top level: a bare array (`[ item, item, … ]`) **or** an envelope (`{ "items": [ … ] }`). Anything else is rejected.
35 +4. **Builds one target per array element**, exposing `.Item` (the decoded element — could be a string, a map, a number, …), `.TUID`, and `.Hash`.
36 +5. **Runs the `services:` rules** against each target. The default stock rule passes the item through unchanged via the `toYaml` helper, so an endpoint that already serves go.d job configurations works with zero rule authoring.
37 +6. **Reconciles** disappeared items — when a target is no longer in the response, the corresponding job stops on the next reconcile.
38 +
39 +
40 +### Limitations
41 +
42 +- Only **one URL per pipeline**. To pull from multiple sources, configure multiple HTTP discovery pipelines (each as its own UI entry, or split the file into one job per source).
43 +- **Response size** is capped at 10 MiB.
44 +- **One-shot mode** (`interval: 0`) fetches a single time when the pipeline starts. It does **not** refetch on SD reload — recreate the pipeline to refresh.
45 +- **`bearer_token_file`** under `/var/run/secrets/` is treated as optional when Netdata is **not** running in Kubernetes (so the same config can be used in a Helm deployment without erroring out on dev hosts).
46 +- The discoverer does not introspect the items it received — anything beyond what the upstream endpoint provides must be inferred via service rules.
47 +
48 +
49 +## Setup
50 +
51 +You can configure the `http` discoverer in two ways:
52 +
53 +| Method | Best for | How to |
54 +|:--|:--|:--|
55 +| [**UI**](#via-ui) | Fast setup without editing files | Go to `Collectors -> go.d -> ServiceDiscovery -> http`, then add a discovery pipeline. |
56 +| [**File**](#via-file) | File-based configuration or automation | Edit `/etc/netdata/go.d/sd/http.conf` and define the `discoverer:` and `services:` blocks. |
57 +
58 +### Prerequisites
59 +
60 +#### Endpoint that returns JSON or YAML
61 +
62 +Stand up an HTTP endpoint that returns either a top-level array (`[ "https://a/health", "https://b/health" ]`) or an envelope (`{ "items": [...] }`). Items can be primitives (strings, numbers), maps, or any nestable value the rule engine knows how to consume.
63 +
64 +
65 +#### Choose a pass-through vs. curated approach
66 +
67 +- **Pass-through**: have your endpoint emit ready-made go.d job configurations and use the stock rule, which renders each item directly via `toYaml`. Zero rule authoring on the Netdata side.
68 +- **Curated**: have your endpoint emit raw data (URLs, hostnames, tags) and write `services:` rules that map the data to the right collector module. More work, more flexibility.
69 +
70 +
71 +### Configuration
72 +
73 +#### Options
74 +
75 +The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn fetched items into collector jobs — see [Service Rules](#service-rules)).
76 +
77 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
78 +
79 +
80 +
81 +| Option | Description | Default | Required |
82 +|:-----|:------------|:--------|:---------:|
83 +| [url](#option-url) | HTTP/HTTPS endpoint that returns the items. | | yes |
84 +| [interval](#option-interval) | How often to refetch the endpoint. | 1m | no |
85 +| [format](#option-format) | Response format. One of `auto`, `json`, `yaml`. | auto | no |
86 +| timeout | Per-request HTTP timeout. | 2s | no |
87 +| [headers / username / password / bearer_token_file / proxy_url / tls_skip_verify / etc.](#option-headers-username-password-bearer-token-file-proxy-url-tls-skip-verify-etc) | All standard go.d HTTP options are accepted (basic auth, bearer tokens, custom headers, HTTP proxy, TLS options). | | no |
88 +
89 +<a id="option-url"></a>
90 +##### url
91 +
92 +Must be a fully-qualified `http://` or `https://` URL. The endpoint is expected to return either a bare array or an `{"items": [...]}` envelope (see [Service Rules](#service-rules) for the input model).
93 +
94 +
95 +<a id="option-interval"></a>
96 +##### interval
97 +
98 +Set to `0` for one-shot mode — the endpoint is fetched once when the pipeline starts and never again. SD reload does not retrigger; recreate the pipeline to refresh.
99 +
100 +
101 +<a id="option-format"></a>
102 +##### format
103 +
104 +With `auto`, the decoder uses `Content-Type` when it is unambiguous (`application/json`, `application/yaml`, `*+json`, `*+yaml`), otherwise tries JSON first then YAML.
105 +
106 +
107 +<a id="option-headers-username-password-bearer-token-file-proxy-url-tls-skip-verify-etc"></a>
108 +##### headers / username / password / bearer_token_file / proxy_url / tls_skip_verify / etc.
109 +
110 +See any go.d HTTP-based collector (`httpcheck`, `prometheus`, `nginx`, …) for the full set. Notable: when `bearer_token_file` points under `/var/run/secrets/` and Netdata is **not** running inside Kubernetes, missing token files are silently ignored.
111 +
112 +
113 +
114 +
115 +#### via UI
116 +
117 +1. Open the Netdata Dynamic Configuration UI.
118 +2. Go to `Collectors -> go.d -> ServiceDiscovery -> http`.
119 +3. Add a new discovery pipeline and give it a name.
120 +4. Fill in the discoverer-specific settings and the service rules.
121 +5. Save the discovery pipeline.
122 +
123 +#### via File
124 +
125 +Define the discovery pipeline in `/etc/netdata/go.d/sd/http.conf`.
126 +
127 +The file has two top-level blocks: `discoverer:` (the options above) and `services:` (rules that turn discovered targets into collector jobs — see [Service Rules](#service-rules)).
128 +
129 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
130 +
131 +##### Examples
132 +
133 +###### Pass-through go.d jobs (stock rule)
134 +
135 +The endpoint serves go.d job configurations directly. Each item must include a `module` field. The stock rule pipes the item through `toYaml` unchanged.
136 +
137 +
138 +```yaml
139 +disabled: no
140 +discoverer:
141 + http:
142 + url: https://cmdb.example.com/netdata/jobs.yaml
143 + interval: 5m
144 + format: auto
145 +services:
146 + - id: passthrough
147 + match: '{{ true }}'
148 + config_template: |
149 + {{ .Item | toYaml }}
150 +
151 +```
152 +###### Array of bare URLs → httpcheck
153 +
154 +The endpoint returns `[ "https://a/health", "https://b/health" ]`. Map each URL to an `httpcheck` job.
155 +
156 +
157 +```yaml
158 +disabled: no
159 +discoverer:
160 + http:
161 + url: https://cmdb.example.com/netdata/health-urls.json
162 + interval: 1m
163 +services:
164 + - id: httpcheck
165 + match: '{{ kindIs "string" .Item }}'
166 + config_template: |
167 + name: {{ .TUID }}
168 + url: {{ .Item }}
169 +
170 +```
171 +###### Array of objects with custom shape
172 +
173 +The endpoint returns `[ { "name": "api", "url": "https://api.example.com/health" }, … ]`.
174 +
175 +
176 +```yaml
177 +disabled: no
178 +discoverer:
179 + http:
180 + url: https://cmdb.example.com/netdata/services.json
181 +services:
182 + - id: httpcheck
183 + match: '{{ and (kindIs "map" .Item) (hasKey .Item "url") }}'
184 + config_template: |
185 + name: {{ .Item.name }}
186 + url: {{ .Item.url }}
187 +
188 +```
189 +###### Bearer-token authentication
190 +
191 +Authenticate against the source-of-truth endpoint using a bearer token from a file.
192 +
193 +```yaml
194 +disabled: no
195 +discoverer:
196 + http:
197 + url: https://cmdb.example.com/api/v1/netdata/jobs
198 + bearer_token_file: /etc/netdata/secrets/cmdb-token
199 + headers:
200 + Accept: application/yaml
201 +services:
202 + - id: passthrough
203 + match: '{{ true }}'
204 + config_template: |
205 + {{ .Item | toYaml }}
206 +
207 +```
208 +
209 +
210 +## Service Rules
211 +
212 +A `services:` rule turns each fetched item into one or more collector jobs. The HTTP discoverer is unique among SD discoverers in that the target's data shape is **defined by the upstream endpoint**, not by this discoverer — `.Item` is whatever JSON/YAML element the endpoint returned.
213 +
214 +The shared rule model — function reference (`match`, sprig including `kindIs`/`hasKey`/`toYaml`), `config_template` rendering rules, and the `missingkey=error` failure semantics — lives on the [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are HTTP-specific.
215 +
216 +
217 +### How rules are evaluated
218 +
219 +Quick reference — see [Rule evaluation semantics](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model.
220 +
221 +
222 +
223 +- **Type-check `.Item` first** — Because `.Item` is whatever the endpoint serves, write defensive rules that check the type before reading sub-fields. `kindIs "string" .Item`, `kindIs "map" .Item`, and `hasKey .Item "<key>"` are the workhorses. A rule that does `{{ .Item.url }}` on a non-map item will fail at template-render time (`missingkey=error`) and the rule will be skipped.
224 +- **Pass-through requires `module:` in the upstream payload** — The pass-through rule (`{{ .Item / toYaml }}`) forwards the item unchanged to the collector subsystem. The collector subsystem requires every job to have `name:` and `module:`. If your endpoint omits `module:`, the resulting job has no module and the agent rejects it. Either include `module:` upstream or wrap with a curated rule that adds it.
225 +- **Module inference from rule id** — When you write a curated rule and the rendered job omits `module:`, the rule `id` is used as the module name. So `id: httpcheck` is enough to produce httpcheck jobs without writing `module: httpcheck` in every template.
226 +
227 +### Template Variables
228 +
229 +Available inside both `match` expressions and `config_template` bodies for HTTP targets.
230 +
231 +
232 +| Variable | Type | Description |
233 +|:---------|:-----|:------------|
234 +| `.Item` | any | The decoded array element. Type depends on the upstream endpoint — could be a string, a number, a bool, a map, or a nested structure. Always type-check before reading sub-fields. |
235 +| `.TUID` | string | Stable per-target ID (`http_<endpoint-label>_<hash>`). Useful as a job `name:` when the upstream payload does not provide one. |
236 +| `.Hash` | uint64 | Hash of the item content. Used internally for change detection. |
237 +
238 +### Examples
239 +
240 +Each example shows one or more entries from the `services:` array. Order matters — see [How rules are evaluated](#how-rules-are-evaluated).
241 +
242 +#### Pass-through (default stock rule)
243 +
244 +The endpoint already returns valid go.d job configurations. Forward each item unchanged via `toYaml`. Each item **must** include a `module` field (and `name`).
245 +
246 +
247 +```yaml
248 +- id: passthrough
249 + match: '{{ true }}'
250 + config_template: |
251 + {{ .Item | toYaml }}
252 +
253 +```
254 +
255 +#### Array of strings → httpcheck (curated)
256 +
257 +Endpoint serves `[ "https://a/health", "https://b/health" ]`. Use `kindIs "string"` to gate the rule, then map each string to an `httpcheck` job. `id: httpcheck` makes the module infer automatically.
258 +
259 +
260 +```yaml
261 +- id: httpcheck
262 + match: '{{ kindIs "string" .Item }}'
263 + config_template: |
264 + name: {{ .TUID }}
265 + url: {{ .Item }}
266 +
267 +```
268 +
269 +#### Array of objects → httpcheck (curated)
270 +
271 +Endpoint serves `[ { "name": "api", "url": "https://api/health" }, … ]`. Type-check that the item is a map and has a `url` key, then map fields into the rendered job.
272 +
273 +
274 +```yaml
275 +- id: httpcheck
276 + match: '{{ and (kindIs "map" .Item) (hasKey .Item "url") }}'
277 + config_template: |
278 + name: {{ .Item.name }}
279 + url: {{ .Item.url }}
280 +
281 +```
282 +
283 +#### Multiple modules from one endpoint
284 +
285 +Your endpoint mixes shapes — some items target `httpcheck`, some target `prometheus`. Use `hasKey` to discriminate, with each rule producing its own module's jobs.
286 +
287 +
288 +```yaml
289 +- id: prometheus
290 + match: '{{ and (kindIs "map" .Item) (hasKey .Item "metrics_url") }}'
291 + config_template: |
292 + name: {{ .Item.name }}
293 + url: {{ .Item.metrics_url }}
294 +
295 +- id: httpcheck
296 + match: '{{ and (kindIs "map" .Item) (hasKey .Item "health_url") }}'
297 + config_template: |
298 + name: {{ .Item.name }}
299 + url: {{ .Item.health_url }}
300 +
301 +```
302 +
303 +
304 +
305 +## Verify discovery worked
306 +
307 +After enabling the discoverer, confirm the endpoint is reachable and items are being parsed.
308 +
309 +### Confirm the endpoint is being fetched
310 +
311 +Watch the agent log for `discoverer=http` messages. With systemd:
312 +
313 +```bash
314 +journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep "discoverer=http"
315 +```
316 +
317 +A successful fetch logs the number of items decoded. Failures (DNS, TLS, auth, parse) appear at warn level.
318 +
319 +
320 +### Reproduce the fetch with curl
321 +
322 +When the discoverer log shows a parse error, hit the endpoint with `curl` to inspect what it returned:
323 +
324 +```bash
325 +curl -sS -H "Accept: application/yaml" https://cmdb.example.com/netdata/jobs.yaml | head -40
326 +```
327 +
328 +The response **must** be a top-level array or `{"items": [...]}` envelope.
329 +
330 +
331 +### Confirm jobs are being created
332 +
333 +In the Netdata UI go to `Collectors -> go.d -> <module>`. Pass-through jobs use the `name` your endpoint provided; curated rules use whatever you set in the `config_template`.
334 +
335 +
336 +
337 +
338 +## Troubleshooting
339 +
340 +### parse response as json: ...; parse response as yaml: ...
341 +
342 +The response is neither valid JSON nor valid YAML. Common causes: the endpoint returned an HTML error page (check status code and `Content-Type`), the JSON has trailing garbage, or YAML indentation is wrong. Reproduce with `curl -i` to see the headers + body.
343 +
344 +
345 +### Items decoded but no jobs created
346 +
347 +Your `services:` rules are not matching, or they match but the rendered template is empty. With pass-through (`{{ .Item | toYaml }}`), make sure each upstream item includes `module:` and `name:`. With curated rules, double-check the type checks (`kindIs`, `hasKey`).
348 +
349 +
350 +### TLS/certificate errors against an internal endpoint
351 +
352 +Use `tls_skip_verify: yes` to bypass for testing, then mount the issuing CA and set `tls_ca: /path/to/ca.crt` for production.
353 +
354 +
355 +### Bearer token file not found
356 +
357 +When Netdata runs **outside** Kubernetes and the configured `bearer_token_file` points under `/var/run/secrets/`, missing tokens are silently ignored — this is intentional so the same config works in dev and in Helm. If you are inside k8s, the file must exist.
358 +
359 +
360 +
src/go/plugin/go.d/discovery/sdext/discoverer/httpsd/metadata.yaml new
+265
@@ -0,0 +1,265 @@
1 +# yamllint disable rule:line-length
2 +---
3 +id: 'service-discovery-http'
4 +meta:
5 + kind: 'http'
6 + name: 'HTTP endpoint'
7 + tagline: 'Items returned by an HTTP/HTTPS endpoint (JSON or YAML).'
8 + link: 'https://datatracker.ietf.org/doc/html/rfc9110'
9 + icon_filename: 'http.svg'
10 +keywords:
11 + - 'service discovery'
12 + - 'sd'
13 + - 'http'
14 + - 'rest'
15 + - 'discovery'
16 + - 'cmdb'
17 +overview:
18 + description: |
19 + Netdata can pull a list of monitorable targets from any HTTP endpoint you control — a CMDB API, an internal asset registry, a static file served by nginx, or a Prometheus-style file_sd export. The discoverer fetches the endpoint, decodes JSON or YAML, and feeds each item into the `services:` rule engine. This is the "bring your own source-of-truth" discoverer.
20 +
21 + This page covers HTTP-specific setup. For the broader Service Discovery model and the shared template-helper reference, see [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md).
22 + how_it_works: |
23 + Each discovery cycle, the discoverer:
24 +
25 + 1. **Fetches** the configured `url` over HTTP/HTTPS, honouring all standard go.d collector HTTP options (auth, headers, TLS, proxy, timeout).
26 + 2. **Decodes** the response as either JSON or YAML according to `format` (auto / json / yaml). With `format: auto`, the decoder uses `Content-Type` if it is unambiguous, otherwise tries JSON first then YAML.
27 + 3. **Accepts two shapes** at the top level: a bare array (`[ item, item, … ]`) **or** an envelope (`{ "items": [ … ] }`). Anything else is rejected.
28 + 4. **Builds one target per array element**, exposing `.Item` (the decoded element — could be a string, a map, a number, …), `.TUID`, and `.Hash`.
29 + 5. **Runs the `services:` rules** against each target. The default stock rule passes the item through unchanged via the `toYaml` helper, so an endpoint that already serves go.d job configurations works with zero rule authoring.
30 + 6. **Reconciles** disappeared items — when a target is no longer in the response, the corresponding job stops on the next reconcile.
31 + limitations: |
32 + - Only **one URL per pipeline**. To pull from multiple sources, configure multiple HTTP discovery pipelines (each as its own UI entry, or split the file into one job per source).
33 + - **Response size** is capped at 10 MiB.
34 + - **One-shot mode** (`interval: 0`) fetches a single time when the pipeline starts. It does **not** refetch on SD reload — recreate the pipeline to refresh.
35 + - **`bearer_token_file`** under `/var/run/secrets/` is treated as optional when Netdata is **not** running in Kubernetes (so the same config can be used in a Helm deployment without erroring out on dev hosts).
36 + - The discoverer does not introspect the items it received — anything beyond what the upstream endpoint provides must be inferred via service rules.
37 +setup:
38 + prerequisites:
39 + list:
40 + - title: 'Endpoint that returns JSON or YAML'
41 + description: |
42 + Stand up an HTTP endpoint that returns either a top-level array (`[ "https://a/health", "https://b/health" ]`) or an envelope (`{ "items": [...] }`). Items can be primitives (strings, numbers), maps, or any nestable value the rule engine knows how to consume.
43 + - title: 'Choose a pass-through vs. curated approach'
44 + description: |
45 + - **Pass-through**: have your endpoint emit ready-made go.d job configurations and use the stock rule, which renders each item directly via `toYaml`. Zero rule authoring on the Netdata side.
46 + - **Curated**: have your endpoint emit raw data (URLs, hostnames, tags) and write `services:` rules that map the data to the right collector module. More work, more flexibility.
47 + configuration:
48 + file:
49 + name: 'go.d/sd/http.conf'
50 + options:
51 + description: |
52 + The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn fetched items into collector jobs — see [Service Rules](#service-rules)).
53 +
54 + After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
55 + folding:
56 + title: 'Discoverer options'
57 + enabled: false
58 + list:
59 + - name: 'url'
60 + description: 'HTTP/HTTPS endpoint that returns the items.'
61 + default_value: ''
62 + required: true
63 + detailed_description: |
64 + Must be a fully-qualified `http://` or `https://` URL. The endpoint is expected to return either a bare array or an `{"items": [...]}` envelope (see [Service Rules](#service-rules) for the input model).
65 + - name: 'interval'
66 + description: 'How often to refetch the endpoint.'
67 + default_value: '1m'
68 + required: false
69 + detailed_description: |
70 + Set to `0` for one-shot mode — the endpoint is fetched once when the pipeline starts and never again. SD reload does not retrigger; recreate the pipeline to refresh.
71 + - name: 'format'
72 + description: 'Response format. One of `auto`, `json`, `yaml`.'
73 + default_value: 'auto'
74 + required: false
75 + detailed_description: |
76 + With `auto`, the decoder uses `Content-Type` when it is unambiguous (`application/json`, `application/yaml`, `*+json`, `*+yaml`), otherwise tries JSON first then YAML.
77 + - name: 'timeout'
78 + description: 'Per-request HTTP timeout.'
79 + default_value: '2s'
80 + required: false
81 + - name: 'headers / username / password / bearer_token_file / proxy_url / tls_skip_verify / etc.'
82 + description: 'All standard go.d HTTP options are accepted (basic auth, bearer tokens, custom headers, HTTP proxy, TLS options).'
83 + default_value: ''
84 + required: false
85 + detailed_description: |
86 + See any go.d HTTP-based collector (`httpcheck`, `prometheus`, `nginx`, …) for the full set. Notable: when `bearer_token_file` points under `/var/run/secrets/` and Netdata is **not** running inside Kubernetes, missing token files are silently ignored.
87 + examples:
88 + folding:
89 + title: 'Configuration examples'
90 + enabled: true
91 + list:
92 + - name: 'Pass-through go.d jobs (stock rule)'
93 + description: |
94 + The endpoint serves go.d job configurations directly. Each item must include a `module` field. The stock rule pipes the item through `toYaml` unchanged.
95 + config: |
96 + disabled: no
97 + discoverer:
98 + http:
99 + url: https://cmdb.example.com/netdata/jobs.yaml
100 + interval: 5m
101 + format: auto
102 + services:
103 + - id: passthrough
104 + match: '{{ true }}'
105 + config_template: |
106 + {{ .Item | toYaml }}
107 + - name: 'Array of bare URLs → httpcheck'
108 + description: |
109 + The endpoint returns `[ "https://a/health", "https://b/health" ]`. Map each URL to an `httpcheck` job.
110 + config: |
111 + disabled: no
112 + discoverer:
113 + http:
114 + url: https://cmdb.example.com/netdata/health-urls.json
115 + interval: 1m
116 + services:
117 + - id: httpcheck
118 + match: '{{ kindIs "string" .Item }}'
119 + config_template: |
120 + name: {{ .TUID }}
121 + url: {{ .Item }}
122 + - name: 'Array of objects with custom shape'
123 + description: |
124 + The endpoint returns `[ { "name": "api", "url": "https://api.example.com/health" }, … ]`.
125 + config: |
126 + disabled: no
127 + discoverer:
128 + http:
129 + url: https://cmdb.example.com/netdata/services.json
130 + services:
131 + - id: httpcheck
132 + match: '{{ and (kindIs "map" .Item) (hasKey .Item "url") }}'
133 + config_template: |
134 + name: {{ .Item.name }}
135 + url: {{ .Item.url }}
136 + - name: 'Bearer-token authentication'
137 + description: 'Authenticate against the source-of-truth endpoint using a bearer token from a file.'
138 + config: |
139 + disabled: no
140 + discoverer:
141 + http:
142 + url: https://cmdb.example.com/api/v1/netdata/jobs
143 + bearer_token_file: /etc/netdata/secrets/cmdb-token
144 + headers:
145 + Accept: application/yaml
146 + services:
147 + - id: passthrough
148 + match: '{{ true }}'
149 + config_template: |
150 + {{ .Item | toYaml }}
151 +services:
152 + description: |
153 + A `services:` rule turns each fetched item into one or more collector jobs. The HTTP discoverer is unique among SD discoverers in that the target's data shape is **defined by the upstream endpoint**, not by this discoverer — `.Item` is whatever JSON/YAML element the endpoint returned.
154 +
155 + The shared rule model — function reference (`match`, sprig including `kindIs`/`hasKey`/`toYaml`), `config_template` rendering rules, and the `missingkey=error` failure semantics — lives on the [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are HTTP-specific.
156 + evaluation:
157 + description: |
158 + Quick reference — see [Rule evaluation semantics](/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model.
159 + list:
160 + - name: 'Type-check `.Item` first'
161 + description: |
162 + Because `.Item` is whatever the endpoint serves, write defensive rules that check the type before reading sub-fields. `kindIs "string" .Item`, `kindIs "map" .Item`, and `hasKey .Item "<key>"` are the workhorses. A rule that does `{{ .Item.url }}` on a non-map item will fail at template-render time (`missingkey=error`) and the rule will be skipped.
163 + - name: 'Pass-through requires `module:` in the upstream payload'
164 + description: |
165 + The pass-through rule (`{{ .Item | toYaml }}`) forwards the item unchanged to the collector subsystem. The collector subsystem requires every job to have `name:` and `module:`. If your endpoint omits `module:`, the resulting job has no module and the agent rejects it. Either include `module:` upstream or wrap with a curated rule that adds it.
166 + - name: 'Module inference from rule id'
167 + description: |
168 + When you write a curated rule and the rendered job omits `module:`, the rule `id` is used as the module name. So `id: httpcheck` is enough to produce httpcheck jobs without writing `module: httpcheck` in every template.
169 + template_variables:
170 + description: 'Available inside both `match` expressions and `config_template` bodies for HTTP targets.'
171 + list:
172 + - name: '.Item'
173 + type: 'any'
174 + description: 'The decoded array element. Type depends on the upstream endpoint — could be a string, a number, a bool, a map, or a nested structure. Always type-check before reading sub-fields.'
175 + - name: '.TUID'
176 + type: 'string'
177 + description: |
178 + Stable per-target ID (`http_<endpoint-label>_<hash>`). Useful as a job `name:` when the upstream payload does not provide one.
179 + - name: '.Hash'
180 + type: 'uint64'
181 + description: 'Hash of the item content. Used internally for change detection.'
182 + examples:
183 + description: 'Each example shows one or more entries from the `services:` array. Order matters — see [How rules are evaluated](#how-rules-are-evaluated).'
184 + list:
185 + - name: 'Pass-through (default stock rule)'
186 + description: |
187 + The endpoint already returns valid go.d job configurations. Forward each item unchanged via `toYaml`. Each item **must** include a `module` field (and `name`).
188 + config: |
189 + - id: passthrough
190 + match: '{{ true }}'
191 + config_template: |
192 + {{ .Item | toYaml }}
193 + - name: 'Array of strings → httpcheck (curated)'
194 + description: |
195 + Endpoint serves `[ "https://a/health", "https://b/health" ]`. Use `kindIs "string"` to gate the rule, then map each string to an `httpcheck` job. `id: httpcheck` makes the module infer automatically.
196 + config: |
197 + - id: httpcheck
198 + match: '{{ kindIs "string" .Item }}'
199 + config_template: |
200 + name: {{ .TUID }}
201 + url: {{ .Item }}
202 + - name: 'Array of objects → httpcheck (curated)'
203 + description: |
204 + Endpoint serves `[ { "name": "api", "url": "https://api/health" }, … ]`. Type-check that the item is a map and has a `url` key, then map fields into the rendered job.
205 + config: |
206 + - id: httpcheck
207 + match: '{{ and (kindIs "map" .Item) (hasKey .Item "url") }}'
208 + config_template: |
209 + name: {{ .Item.name }}
210 + url: {{ .Item.url }}
211 + - name: 'Multiple modules from one endpoint'
212 + description: |
213 + Your endpoint mixes shapes — some items target `httpcheck`, some target `prometheus`. Use `hasKey` to discriminate, with each rule producing its own module's jobs.
214 + config: |
215 + - id: prometheus
216 + match: '{{ and (kindIs "map" .Item) (hasKey .Item "metrics_url") }}'
217 + config_template: |
218 + name: {{ .Item.name }}
219 + url: {{ .Item.metrics_url }}
220 +
221 + - id: httpcheck
222 + match: '{{ and (kindIs "map" .Item) (hasKey .Item "health_url") }}'
223 + config_template: |
224 + name: {{ .Item.name }}
225 + url: {{ .Item.health_url }}
226 +verify:
227 + description: 'After enabling the discoverer, confirm the endpoint is reachable and items are being parsed.'
228 + checks:
229 + list:
230 + - name: 'Confirm the endpoint is being fetched'
231 + description: |
232 + Watch the agent log for `discoverer=http` messages. With systemd:
233 +
234 + ```bash
235 + journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep "discoverer=http"
236 + ```
237 +
238 + A successful fetch logs the number of items decoded. Failures (DNS, TLS, auth, parse) appear at warn level.
239 + - name: 'Reproduce the fetch with curl'
240 + description: |
241 + When the discoverer log shows a parse error, hit the endpoint with `curl` to inspect what it returned:
242 +
243 + ```bash
244 + curl -sS -H "Accept: application/yaml" https://cmdb.example.com/netdata/jobs.yaml | head -40
245 + ```
246 +
247 + The response **must** be a top-level array or `{"items": [...]}` envelope.
248 + - name: 'Confirm jobs are being created'
249 + description: |
250 + In the Netdata UI go to `Collectors -> go.d -> <module>`. Pass-through jobs use the `name` your endpoint provided; curated rules use whatever you set in the `config_template`.
251 +troubleshooting:
252 + problems:
253 + list:
254 + - name: 'parse response as json: ...; parse response as yaml: ...'
255 + description: |
256 + The response is neither valid JSON nor valid YAML. Common causes: the endpoint returned an HTML error page (check status code and `Content-Type`), the JSON has trailing garbage, or YAML indentation is wrong. Reproduce with `curl -i` to see the headers + body.
257 + - name: 'Items decoded but no jobs created'
258 + description: |
259 + Your `services:` rules are not matching, or they match but the rendered template is empty. With pass-through (`{{ .Item | toYaml }}`), make sure each upstream item includes `module:` and `name:`. With curated rules, double-check the type checks (`kindIs`, `hasKey`).
260 + - name: 'TLS/certificate errors against an internal endpoint'
261 + description: |
262 + Use `tls_skip_verify: yes` to bypass for testing, then mount the issuing CA and set `tls_ca: /path/to/ca.crt` for production.
263 + - name: 'Bearer token file not found'
264 + description: |
265 + When Netdata runs **outside** Kubernetes and the configured `bearer_token_file` points under `/var/run/secrets/`, missing tokens are silently ignored — this is intentional so the same config works in dev and in Helm. If you are inside k8s, the file must exist.
src/go/plugin/go.d/discovery/sdext/discoverer/k8ssd/README.md new
+1
@@ -0,0 +1 @@
1 +integrations/k8s.md
\ No newline at end of file
src/go/plugin/go.d/discovery/sdext/discoverer/k8ssd/integrations/k8s.md new
+338
@@ -0,0 +1,338 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/k8ssd/README.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/k8ssd/metadata.yaml"
4 +sidebar_label: "Kubernetes"
5 +learn_status: "Published"
6 +learn_rel_path: "Collecting Metrics/Service Discovery"
7 +keywords: ['service discovery', 'sd', 'k8s', 'kubernetes', 'pods', 'services', 'discovery']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE SERVICE DISCOVERY DISCOVERER'S metadata.yaml FILE"
9 +endmeta-->
10 +
11 +# Kubernetes discovery
12 +
13 +
14 +<img src="https://netdata.cloud/img/kubernetes.svg" width="150"/>
15 +
16 +
17 +Kind: `k8s`
18 +
19 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
20 +
21 +## Overview
22 +
23 +Netdata can automatically discover monitorable workloads inside a Kubernetes cluster — pods (with their containers and ports) or Services. The discoverer watches the Kubernetes API in real time, exposes per-pod-container or per-service-port targets to the rule engine, and lets you generate collector jobs from labels, annotations, container images, and ports.
24 +
25 +This page covers Kubernetes-specific setup. For the broader Service Discovery model and the shared template-helper reference, see [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md).
26 +
27 +
28 +### How it works
29 +
30 +Each Kubernetes discovery pipeline runs as either a **pod** discoverer or a **service** discoverer (selected by the `role` option). It then:
31 +
32 +1. **Connects** to the Kubernetes API using the in-cluster service-account credentials (no `api_server` config — the discoverer uses the standard k8s client config-loader chain).
33 +2. **Watches** Pods (or Services) in the configured `namespaces[]`, optionally narrowed by label/field selectors.
34 +3. **Builds targets**:
35 + - `role: pod` → one target per `(pod, container, container-port)` triple. Container env, image, labels, annotations, and node name are all exposed.
36 + - `role: service` → one target per `(service, service-port)` pair, with the cluster-internal DNS name (`name.ns.svc:port`) as `.Address`.
37 +4. **Runs the `services:` rules** against each target, producing collector jobs.
38 +5. **Reconciles** in real time — pod/service add/update/delete events update the target set without polling.
39 +
40 +
41 +### Limitations
42 +
43 +- **Stock conf ships in the Helm chart, not this repo**: a stock `/etc/netdata/go.d/sd/k8s.conf` is not packaged with the agent. On Kubernetes deployments you should install Netdata via the [Helm chart](https://github.com/netdata/helmchart) — the chart renders both the discoverer config and a curated rule set tailored to your cluster's Netdata setup.
44 +- **Outside Kubernetes**: this discoverer requires kube-API access (in-cluster service-account or kubeconfig). Running it on a workstation requires a kubeconfig and is not a typical use case.
45 +- **Two roles per pipeline, never both**: `role` is a single-valued option. If you want both pod and service discovery, configure two pipelines.
46 +- **`local_mode` for pods is opt-in**: by default the pod discoverer watches **all** pods in the configured namespaces. Set `pod.local_mode: true` to restrict to pods on the **same node** as the Netdata Agent (intended for the parent-on-every-node Helm topology). When `local_mode` is enabled, the env var `MY_NODE_NAME` must be set on the Netdata pod (the Helm chart sets this via the downward API).
47 +- **TLS to the API server is mTLS via the in-cluster CA bundle** — there is no per-pipeline TLS configuration to override.
48 +
49 +
50 +## Setup
51 +
52 +You can configure the `k8s` discoverer in two ways:
53 +
54 +| Method | Best for | How to |
55 +|:--|:--|:--|
56 +| [**UI**](#via-ui) | Fast setup without editing files | Go to `Collectors -> go.d -> ServiceDiscovery -> k8s`, then add a discovery pipeline. |
57 +| [**File**](#via-file) | File-based configuration or automation | Edit `/etc/netdata/go.d/sd/k8s.conf` and define the `discoverer:` and `services:` blocks. |
58 +
59 +### Prerequisites
60 +
61 +#### Run on Kubernetes via the Netdata Helm chart
62 +
63 +The supported way to run the k8s discoverer is via the [Netdata Helm chart](https://github.com/netdata/helmchart). The chart provisions the right RBAC (`get/list/watch` on `pods`, `services`, `configmaps`, `secrets`), wires `MY_NODE_NAME` for `local_mode`, and ships a stock `services:` rule set tuned to its parent/child topology.
64 +
65 +
66 +#### RBAC permissions
67 +
68 +The discoverer needs the following verbs from its service account:
69 +
70 +- `pods`: `get`, `list`, `watch` (cluster-wide or per-namespace, matching `namespaces[]`)
71 +- `services`: `get`, `list`, `watch` (only when `role: service`)
72 +- `configmaps`, `secrets`: `get`, `list`, `watch` (only when `role: pod` — used to enrich pod targets with referenced env values)
73 +
74 +The Helm chart's default RBAC role covers all of these.
75 +
76 +
77 +#### For `pod.local_mode: true`, set MY_NODE_NAME
78 +
79 +When `local_mode` is enabled, the Netdata Agent reads its node name from `MY_NODE_NAME`. The Helm chart sets this via the downward API:
80 +
81 +```yaml
82 +env:
83 + - name: MY_NODE_NAME
84 + valueFrom:
85 + fieldRef:
86 + fieldPath: spec.nodeName
87 +```
88 +
89 +
90 +### Configuration
91 +
92 +#### Options
93 +
94 +The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn discovered pods/services into collector jobs — see [Service Rules](#service-rules)).
95 +
96 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline. The default and recommended deployment path on Kubernetes is the [Netdata Helm chart](https://github.com/netdata/helmchart) — the chart renders this file and the rules for you.
97 +
98 +
99 +
100 +| Option | Description | Default | Required |
101 +|:-----|:------------|:--------|:---------:|
102 +| [role](#option-role) | What to discover. One of `pod` or `service`. | | yes |
103 +| namespaces | Namespaces to watch. Empty means all namespaces. | [] (all namespaces) | no |
104 +| [selector.label](#option-selector-label) | Label selector applied at watch time (server-side filtering). | | no |
105 +| [selector.field](#option-selector-field) | Field selector applied at watch time. | | no |
106 +| [pod.local_mode](#option-pod-local-mode) | Restrict pod discovery to pods on the same node as the Netdata Agent. | false | no |
107 +
108 +<a id="option-role"></a>
109 +##### role
110 +
111 +- `pod` — produces one target per `(pod, container, port)` triple. Use this for the bulk of in-cluster monitoring (databases, exporters, applications).
112 +- `service` — produces one target per `(service, port)` pair. Use this for cluster-internal endpoints monitored at the service-name DNS level.
113 +
114 +To watch both, configure two pipelines.
115 +
116 +
117 +<a id="option-selector-label"></a>
118 +##### selector.label
119 +
120 +Standard Kubernetes label-selector syntax: `app=foo`, `environment in (prod, staging)`, etc. Reduces watch traffic when only a subset of pods/services is interesting.
121 +
122 +
123 +<a id="option-selector-field"></a>
124 +##### selector.field
125 +
126 +Useful field selectors: `status.phase=Running`, `spec.nodeName=node-1`. When `pod.local_mode: true`, the discoverer automatically appends `spec.nodeName=$MY_NODE_NAME`.
127 +
128 +
129 +<a id="option-pod-local-mode"></a>
130 +##### pod.local_mode
131 +
132 +Only applies when `role: pod`. Requires `MY_NODE_NAME` to be set on the Netdata container. Used by the Helm chart's parent-on-every-node topology to keep watch traffic local.
133 +
134 +
135 +
136 +
137 +#### via UI
138 +
139 +1. Open the Netdata Dynamic Configuration UI.
140 +2. Go to `Collectors -> go.d -> ServiceDiscovery -> k8s`.
141 +3. Add a new discovery pipeline and give it a name.
142 +4. Fill in the discoverer-specific settings and the service rules.
143 +5. Save the discovery pipeline.
144 +
145 +#### via File
146 +
147 +Define the discovery pipeline in `/etc/netdata/go.d/sd/k8s.conf`.
148 +
149 +The file has two top-level blocks: `discoverer:` (the options above) and `services:` (rules that turn discovered targets into collector jobs — see [Service Rules](#service-rules)).
150 +
151 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
152 +
153 +##### Examples
154 +
155 +###### Pod discovery, local mode (Helm-style)
156 +
157 +The configuration the Helm chart renders by default for the parent-on-every-node topology.
158 +
159 +```yaml
160 +disabled: no
161 +discoverer:
162 + k8s:
163 + role: pod
164 + pod:
165 + local_mode: true
166 +services: [ ]
167 +
168 +```
169 +###### Service discovery in a specific namespace
170 +
171 +Watch only Services in the `monitoring` namespace, scoped by a label selector.
172 +
173 +```yaml
174 +disabled: no
175 +discoverer:
176 + k8s:
177 + role: service
178 + namespaces:
179 + - monitoring
180 + selector:
181 + label: app.kubernetes.io/component=metrics-endpoint
182 +services: [ ]
183 +
184 +```
185 +
186 +
187 +## Service Rules
188 +
189 +A `services:` rule turns each discovered pod-container target (`role: pod`) or service-port target (`role: service`) into one or more collector jobs. The two target shapes have different fields — annotations and labels are common to both, but pod targets additionally expose container-level info (image, env, controller).
190 +
191 +The shared rule model — function reference (`match`, `glob`, `hasKey`, `index`, sprig), `config_template` rendering rules, and the `missingkey=error` failure semantics — lives on the [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are k8s-specific.
192 +
193 +
194 +### How rules are evaluated
195 +
196 +Quick reference — see [Rule evaluation semantics](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model.
197 +
198 +
199 +
200 +- **Different target shape per role** — `role: pod` and `role: service` produce different target structs. Rules in a pipeline must assume one shape — design your pipeline to match the discoverer's `role`. To handle both, run two pipelines.
201 +- **Annotation-driven matching is idiomatic** — Standard Kubernetes practice is to opt pods/services into monitoring via annotations (e.g. `prometheus.io/scrape: "true"`, `netdata.cloud/scrape: "true"`). Use `hasKey .Annotations "key"` and `index .Annotations "key"` to read them.
202 +- **Container ports vs. service ports** — Pod targets expose `.Port` / `.PortName` / `.PortProtocol` from the container's `ports[]`. Service targets expose them from the service's `ports[]`. Container ports may not be advertised through a Service — when you want both granularities, run two pipelines.
203 +- **Module inference from rule id** — For Kubernetes, set `id: <module-name>` so the rendered job inherits the module name automatically — same as the other discoverers.
204 +
205 +### Template Variables
206 +
207 +Two distinct target shapes — `PodTarget` for `role: pod` and `ServiceTarget` for `role: service`.
208 +
209 +
210 +| Variable | Type | Description |
211 +|:---------|:-----|:------------|
212 +| `.Address` | string | For pods: `<pod-IP>:<port>` (or just `<pod-IP>` when no container port is exposed). For services: `<svc-name>.<namespace>.svc:<port>`. |
213 +| `.Namespace` | string | Pod/Service namespace. |
214 +| `.Name` | string | Pod or Service name. |
215 +| `.Annotations` | map | Pod/Service annotations. Read with `index .Annotations "key"`. |
216 +| `.Labels` | map | Pod/Service labels. Read with `index .Labels "key"`. |
217 +| `.Port` | string | Container port (pod target) or service port (service target). |
218 +| `.PortName` | string | Port name as declared in the spec (`http`, `metrics`, …). |
219 +| `.PortProtocol` | string | Port protocol (`TCP`, `UDP`). |
220 +| `.PodIP` | string | **Pod targets only.** IP address of the pod. |
221 +| `.NodeName` | string | **Pod targets only.** Name of the node hosting the pod. |
222 +| `.ContName` | string | **Pod targets only.** Container name (within the pod). |
223 +| `.Image` | string | **Pod targets only.** Container image. |
224 +| `.Env` | map | **Pod targets only.** Container environment, with values from referenced ConfigMaps and Secrets resolved. |
225 +| `.ControllerName` | string | **Pod targets only.** Owning controller name (e.g. ReplicaSet name). |
226 +| `.ControllerKind` | string | **Pod targets only.** Owning controller kind (`ReplicaSet`, `StatefulSet`, `DaemonSet`, `Job`, …). |
227 +| `.ClusterIP` | string | **Service targets only.** Cluster IP. |
228 +| `.ExternalName` | string | **Service targets only.** External name (for `type: ExternalName` services). |
229 +| `.Type` | string | **Service targets only.** Service type (`ClusterIP`, `NodePort`, `LoadBalancer`, `ExternalName`). |
230 +
231 +### Examples
232 +
233 +Each example shows one entry from the `services:` array. Order matters — see [How rules are evaluated](#how-rules-are-evaluated).
234 +
235 +#### Pod with prometheus.io/scrape annotation
236 +
237 +The de-facto standard "scrape me" annotation. Match pods that opt in, route to the `prometheus` module.
238 +
239 +
240 +```yaml
241 +- id: prometheus
242 + match: '{{ and (hasKey .Annotations "prometheus.io/scrape") (eq (index .Annotations "prometheus.io/scrape") "true") }}'
243 + config_template: |
244 + name: {{ .Namespace }}_{{ .Name }}_{{ .ContName }}
245 + url: http://{{ .Address }}{{ index .Annotations "prometheus.io/path" | default "/metrics" }}
246 +
247 +```
248 +
249 +#### Service-role: monitor each metrics-endpoint Service
250 +
251 +Run with `role: service`. Match Services that carry a `metrics-endpoint` component label.
252 +
253 +
254 +```yaml
255 +- id: prometheus
256 + match: '{{ and (hasKey .Labels "app.kubernetes.io/component") (eq (index .Labels "app.kubernetes.io/component") "metrics-endpoint") }}'
257 + config_template: |
258 + name: {{ .Namespace }}_{{ .Name }}
259 + url: http://{{ .Address }}/metrics
260 +
261 +```
262 +
263 +#### Image-driven: nginx pods
264 +
265 +Match nginx-image pods on a known port. Use `match "sp"` for the four-form image family.
266 +
267 +
268 +```yaml
269 +- id: nginx
270 + match: '{{ and (eq .Port "80") (match "sp" .Image "nginx nginx:* */nginx */nginx:*") }}'
271 + config_template: |
272 + name: {{ .Namespace }}_{{ .Name }}
273 + url: http://{{ .Address }}/stub_status
274 +
275 +```
276 +
277 +
278 +
279 +## Verify discovery worked
280 +
281 +After enabling the discoverer, confirm it is watching the API and producing targets.
282 +
283 +### Confirm the discoverer registered
284 +
285 +Watch the Netdata Agent log inside the pod for `discoverer=kubernetes` messages:
286 +
287 +```bash
288 +kubectl logs -n netdata <netdata-pod> | grep "discoverer=kubernetes"
289 +```
290 +
291 +On startup you should see "instance is started", role information, and which namespaces are being watched. RBAC failures appear as `forbidden` errors from the watch.
292 +
293 +
294 +### Confirm the API is reachable
295 +
296 +From the pod:
297 +
298 +```bash
299 +kubectl exec -n netdata <netdata-pod> -- curl -sSk \
300 + -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
301 + https://kubernetes.default.svc/api/v1/namespaces
302 +```
303 +
304 +A 401 / 403 indicates the service account lacks the right RBAC. The Helm chart provisions the correct role.
305 +
306 +
307 +### Confirm jobs are being created
308 +
309 +In the Netdata UI go to `Collectors -> go.d -> <module>`. Job names follow your `config_template` — the examples above use `<namespace>_<name>` patterns.
310 +
311 +
312 +
313 +
314 +## Troubleshooting
315 +
316 +### Permission denied (RBAC)
317 +
318 +The service account needs `get`, `list`, `watch` on `pods` (or `services`), and on `configmaps` + `secrets` for pod-role env enrichment. The Helm chart provisions this; out-of-Helm deployments must bind the equivalent role.
319 +
320 +
321 +### `local_mode` enabled but env "MY_NODE_NAME" not set
322 +
323 +When `pod.local_mode: true` is set but `MY_NODE_NAME` is missing, the discoverer fails at startup with `local_mode is enabled, but env 'MY_NODE_NAME' not set`. Set the env via the downward API on the Netdata pod (the Helm chart does this).
324 +
325 +
326 +### No targets discovered
327 +
328 +- Confirm pods/services exist in the configured `namespaces[]`.
329 +- If `selector.label` or `selector.field` is set, verify the targets actually carry the matching labels/fields.
330 +- With `local_mode`, only pods on the same node as the Netdata pod are visible.
331 +
332 +
333 +### Generated jobs fail to start
334 +
335 +The Address resolves to the pod's CNI IP — the Netdata Agent must be able to reach pod IPs. Most CNIs allow this from a pod running in the same cluster, but flat-network requirements differ. For service-role targets, the cluster-internal DNS name (`<svc>.<ns>.svc`) is used and should always resolve from inside the cluster.
336 +
337 +
338 +
src/go/plugin/go.d/discovery/sdext/discoverer/k8ssd/metadata.yaml new
+286
@@ -0,0 +1,286 @@
1 +# yamllint disable rule:line-length
2 +---
3 +id: 'service-discovery-k8s'
4 +meta:
5 + kind: 'k8s'
6 + name: 'Kubernetes'
7 + tagline: 'Pods and services in a Kubernetes cluster.'
8 + link: 'https://kubernetes.io/'
9 + icon_filename: 'kubernetes.svg'
10 +keywords:
11 + - 'service discovery'
12 + - 'sd'
13 + - 'k8s'
14 + - 'kubernetes'
15 + - 'pods'
16 + - 'services'
17 + - 'discovery'
18 +overview:
19 + description: |
20 + Netdata can automatically discover monitorable workloads inside a Kubernetes cluster — pods (with their containers and ports) or Services. The discoverer watches the Kubernetes API in real time, exposes per-pod-container or per-service-port targets to the rule engine, and lets you generate collector jobs from labels, annotations, container images, and ports.
21 +
22 + This page covers Kubernetes-specific setup. For the broader Service Discovery model and the shared template-helper reference, see [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md).
23 + how_it_works: |
24 + Each Kubernetes discovery pipeline runs as either a **pod** discoverer or a **service** discoverer (selected by the `role` option). It then:
25 +
26 + 1. **Connects** to the Kubernetes API using the in-cluster service-account credentials (no `api_server` config — the discoverer uses the standard k8s client config-loader chain).
27 + 2. **Watches** Pods (or Services) in the configured `namespaces[]`, optionally narrowed by label/field selectors.
28 + 3. **Builds targets**:
29 + - `role: pod` → one target per `(pod, container, container-port)` triple. Container env, image, labels, annotations, and node name are all exposed.
30 + - `role: service` → one target per `(service, service-port)` pair, with the cluster-internal DNS name (`name.ns.svc:port`) as `.Address`.
31 + 4. **Runs the `services:` rules** against each target, producing collector jobs.
32 + 5. **Reconciles** in real time — pod/service add/update/delete events update the target set without polling.
33 + limitations: |
34 + - **Stock conf ships in the Helm chart, not this repo**: a stock `/etc/netdata/go.d/sd/k8s.conf` is not packaged with the agent. On Kubernetes deployments you should install Netdata via the [Helm chart](https://github.com/netdata/helmchart) — the chart renders both the discoverer config and a curated rule set tailored to your cluster's Netdata setup.
35 + - **Outside Kubernetes**: this discoverer requires kube-API access (in-cluster service-account or kubeconfig). Running it on a workstation requires a kubeconfig and is not a typical use case.
36 + - **Two roles per pipeline, never both**: `role` is a single-valued option. If you want both pod and service discovery, configure two pipelines.
37 + - **`local_mode` for pods is opt-in**: by default the pod discoverer watches **all** pods in the configured namespaces. Set `pod.local_mode: true` to restrict to pods on the **same node** as the Netdata Agent (intended for the parent-on-every-node Helm topology). When `local_mode` is enabled, the env var `MY_NODE_NAME` must be set on the Netdata pod (the Helm chart sets this via the downward API).
38 + - **TLS to the API server is mTLS via the in-cluster CA bundle** — there is no per-pipeline TLS configuration to override.
39 +setup:
40 + prerequisites:
41 + list:
42 + - title: 'Run on Kubernetes via the Netdata Helm chart'
43 + description: |
44 + The supported way to run the k8s discoverer is via the [Netdata Helm chart](https://github.com/netdata/helmchart). The chart provisions the right RBAC (`get/list/watch` on `pods`, `services`, `configmaps`, `secrets`), wires `MY_NODE_NAME` for `local_mode`, and ships a stock `services:` rule set tuned to its parent/child topology.
45 + - title: 'RBAC permissions'
46 + description: |
47 + The discoverer needs the following verbs from its service account:
48 +
49 + - `pods`: `get`, `list`, `watch` (cluster-wide or per-namespace, matching `namespaces[]`)
50 + - `services`: `get`, `list`, `watch` (only when `role: service`)
51 + - `configmaps`, `secrets`: `get`, `list`, `watch` (only when `role: pod` — used to enrich pod targets with referenced env values)
52 +
53 + The Helm chart's default RBAC role covers all of these.
54 + - title: 'For `pod.local_mode: true`, set MY_NODE_NAME'
55 + description: |
56 + When `local_mode` is enabled, the Netdata Agent reads its node name from `MY_NODE_NAME`. The Helm chart sets this via the downward API:
57 +
58 + ```yaml
59 + env:
60 + - name: MY_NODE_NAME
61 + valueFrom:
62 + fieldRef:
63 + fieldPath: spec.nodeName
64 + ```
65 + configuration:
66 + file:
67 + name: 'go.d/sd/k8s.conf'
68 + options:
69 + description: |
70 + The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn discovered pods/services into collector jobs — see [Service Rules](#service-rules)).
71 +
72 + After editing the file, restart the Netdata Agent to load the updated discovery pipeline. The default and recommended deployment path on Kubernetes is the [Netdata Helm chart](https://github.com/netdata/helmchart) — the chart renders this file and the rules for you.
73 + folding:
74 + title: 'Discoverer options'
75 + enabled: false
76 + list:
77 + - name: 'role'
78 + description: 'What to discover. One of `pod` or `service`.'
79 + default_value: ''
80 + required: true
81 + detailed_description: |
82 + - `pod` — produces one target per `(pod, container, port)` triple. Use this for the bulk of in-cluster monitoring (databases, exporters, applications).
83 + - `service` — produces one target per `(service, port)` pair. Use this for cluster-internal endpoints monitored at the service-name DNS level.
84 +
85 + To watch both, configure two pipelines.
86 + - name: 'namespaces'
87 + description: 'Namespaces to watch. Empty means all namespaces.'
88 + default_value: '[] (all namespaces)'
89 + required: false
90 + - name: 'selector.label'
91 + description: 'Label selector applied at watch time (server-side filtering).'
92 + default_value: ''
93 + required: false
94 + detailed_description: |
95 + Standard Kubernetes label-selector syntax: `app=foo`, `environment in (prod, staging)`, etc. Reduces watch traffic when only a subset of pods/services is interesting.
96 + - name: 'selector.field'
97 + description: 'Field selector applied at watch time.'
98 + default_value: ''
99 + required: false
100 + detailed_description: |
101 + Useful field selectors: `status.phase=Running`, `spec.nodeName=node-1`. When `pod.local_mode: true`, the discoverer automatically appends `spec.nodeName=$MY_NODE_NAME`.
102 + - name: 'pod.local_mode'
103 + description: 'Restrict pod discovery to pods on the same node as the Netdata Agent.'
104 + default_value: 'false'
105 + required: false
106 + detailed_description: |
107 + Only applies when `role: pod`. Requires `MY_NODE_NAME` to be set on the Netdata container. Used by the Helm chart's parent-on-every-node topology to keep watch traffic local.
108 + examples:
109 + folding:
110 + title: 'Configuration examples'
111 + enabled: true
112 + list:
113 + - name: 'Pod discovery, local mode (Helm-style)'
114 + description: 'The configuration the Helm chart renders by default for the parent-on-every-node topology.'
115 + config: |
116 + disabled: no
117 + discoverer:
118 + k8s:
119 + role: pod
120 + pod:
121 + local_mode: true
122 + services: [ ]
123 + - name: 'Service discovery in a specific namespace'
124 + description: 'Watch only Services in the `monitoring` namespace, scoped by a label selector.'
125 + config: |
126 + disabled: no
127 + discoverer:
128 + k8s:
129 + role: service
130 + namespaces:
131 + - monitoring
132 + selector:
133 + label: app.kubernetes.io/component=metrics-endpoint
134 + services: [ ]
135 +services:
136 + description: |
137 + A `services:` rule turns each discovered pod-container target (`role: pod`) or service-port target (`role: service`) into one or more collector jobs. The two target shapes have different fields — annotations and labels are common to both, but pod targets additionally expose container-level info (image, env, controller).
138 +
139 + The shared rule model — function reference (`match`, `glob`, `hasKey`, `index`, sprig), `config_template` rendering rules, and the `missingkey=error` failure semantics — lives on the [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are k8s-specific.
140 + evaluation:
141 + description: |
142 + Quick reference — see [Rule evaluation semantics](/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model.
143 + list:
144 + - name: 'Different target shape per role'
145 + description: |
146 + `role: pod` and `role: service` produce different target structs. Rules in a pipeline must assume one shape — design your pipeline to match the discoverer's `role`. To handle both, run two pipelines.
147 + - name: 'Annotation-driven matching is idiomatic'
148 + description: |
149 + Standard Kubernetes practice is to opt pods/services into monitoring via annotations (e.g. `prometheus.io/scrape: "true"`, `netdata.cloud/scrape: "true"`). Use `hasKey .Annotations "key"` and `index .Annotations "key"` to read them.
150 + - name: 'Container ports vs. service ports'
151 + description: |
152 + Pod targets expose `.Port` / `.PortName` / `.PortProtocol` from the container's `ports[]`. Service targets expose them from the service's `ports[]`. Container ports may not be advertised through a Service — when you want both granularities, run two pipelines.
153 + - name: 'Module inference from rule id'
154 + description: |
155 + For Kubernetes, set `id: <module-name>` so the rendered job inherits the module name automatically — same as the other discoverers.
156 + template_variables:
157 + description: 'Two distinct target shapes — `PodTarget` for `role: pod` and `ServiceTarget` for `role: service`.'
158 + list:
159 + - name: '.Address'
160 + type: 'string'
161 + description: 'For pods: `<pod-IP>:<port>` (or just `<pod-IP>` when no container port is exposed). For services: `<svc-name>.<namespace>.svc:<port>`.'
162 + - name: '.Namespace'
163 + type: 'string'
164 + description: 'Pod/Service namespace.'
165 + - name: '.Name'
166 + type: 'string'
167 + description: 'Pod or Service name.'
168 + - name: '.Annotations'
169 + type: 'map'
170 + description: 'Pod/Service annotations. Read with `index .Annotations "key"`.'
171 + - name: '.Labels'
172 + type: 'map'
173 + description: 'Pod/Service labels. Read with `index .Labels "key"`.'
174 + - name: '.Port'
175 + type: 'string'
176 + description: 'Container port (pod target) or service port (service target).'
177 + - name: '.PortName'
178 + type: 'string'
179 + description: 'Port name as declared in the spec (`http`, `metrics`, …).'
180 + - name: '.PortProtocol'
181 + type: 'string'
182 + description: 'Port protocol (`TCP`, `UDP`).'
183 + - name: '.PodIP'
184 + type: 'string'
185 + description: '**Pod targets only.** IP address of the pod.'
186 + - name: '.NodeName'
187 + type: 'string'
188 + description: '**Pod targets only.** Name of the node hosting the pod.'
189 + - name: '.ContName'
190 + type: 'string'
191 + description: '**Pod targets only.** Container name (within the pod).'
192 + - name: '.Image'
193 + type: 'string'
194 + description: '**Pod targets only.** Container image.'
195 + - name: '.Env'
196 + type: 'map'
197 + description: '**Pod targets only.** Container environment, with values from referenced ConfigMaps and Secrets resolved.'
198 + - name: '.ControllerName'
199 + type: 'string'
200 + description: '**Pod targets only.** Owning controller name (e.g. ReplicaSet name).'
201 + - name: '.ControllerKind'
202 + type: 'string'
203 + description: '**Pod targets only.** Owning controller kind (`ReplicaSet`, `StatefulSet`, `DaemonSet`, `Job`, …).'
204 + - name: '.ClusterIP'
205 + type: 'string'
206 + description: '**Service targets only.** Cluster IP.'
207 + - name: '.ExternalName'
208 + type: 'string'
209 + description: '**Service targets only.** External name (for `type: ExternalName` services).'
210 + - name: '.Type'
211 + type: 'string'
212 + description: '**Service targets only.** Service type (`ClusterIP`, `NodePort`, `LoadBalancer`, `ExternalName`).'
213 + examples:
214 + description: 'Each example shows one entry from the `services:` array. Order matters — see [How rules are evaluated](#how-rules-are-evaluated).'
215 + list:
216 + - name: 'Pod with prometheus.io/scrape annotation'
217 + description: |
218 + The de-facto standard "scrape me" annotation. Match pods that opt in, route to the `prometheus` module.
219 + config: |
220 + - id: prometheus
221 + match: '{{ and (hasKey .Annotations "prometheus.io/scrape") (eq (index .Annotations "prometheus.io/scrape") "true") }}'
222 + config_template: |
223 + name: {{ .Namespace }}_{{ .Name }}_{{ .ContName }}
224 + url: http://{{ .Address }}{{ index .Annotations "prometheus.io/path" | default "/metrics" }}
225 + - name: 'Service-role: monitor each metrics-endpoint Service'
226 + description: |
227 + Run with `role: service`. Match Services that carry a `metrics-endpoint` component label.
228 + config: |
229 + - id: prometheus
230 + match: '{{ and (hasKey .Labels "app.kubernetes.io/component") (eq (index .Labels "app.kubernetes.io/component") "metrics-endpoint") }}'
231 + config_template: |
232 + name: {{ .Namespace }}_{{ .Name }}
233 + url: http://{{ .Address }}/metrics
234 + - name: 'Image-driven: nginx pods'
235 + description: |
236 + Match nginx-image pods on a known port. Use `match "sp"` for the four-form image family.
237 + config: |
238 + - id: nginx
239 + match: '{{ and (eq .Port "80") (match "sp" .Image "nginx nginx:* */nginx */nginx:*") }}'
240 + config_template: |
241 + name: {{ .Namespace }}_{{ .Name }}
242 + url: http://{{ .Address }}/stub_status
243 +verify:
244 + description: 'After enabling the discoverer, confirm it is watching the API and producing targets.'
245 + checks:
246 + list:
247 + - name: 'Confirm the discoverer registered'
248 + description: |
249 + Watch the Netdata Agent log inside the pod for `discoverer=kubernetes` messages:
250 +
251 + ```bash
252 + kubectl logs -n netdata <netdata-pod> | grep "discoverer=kubernetes"
253 + ```
254 +
255 + On startup you should see "instance is started", role information, and which namespaces are being watched. RBAC failures appear as `forbidden` errors from the watch.
256 + - name: 'Confirm the API is reachable'
257 + description: |
258 + From the pod:
259 +
260 + ```bash
261 + kubectl exec -n netdata <netdata-pod> -- curl -sSk \
262 + -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
263 + https://kubernetes.default.svc/api/v1/namespaces
264 + ```
265 +
266 + A 401 / 403 indicates the service account lacks the right RBAC. The Helm chart provisions the correct role.
267 + - name: 'Confirm jobs are being created'
268 + description: |
269 + In the Netdata UI go to `Collectors -> go.d -> <module>`. Job names follow your `config_template` — the examples above use `<namespace>_<name>` patterns.
270 +troubleshooting:
271 + problems:
272 + list:
273 + - name: 'Permission denied (RBAC)'
274 + description: |
275 + The service account needs `get`, `list`, `watch` on `pods` (or `services`), and on `configmaps` + `secrets` for pod-role env enrichment. The Helm chart provisions this; out-of-Helm deployments must bind the equivalent role.
276 + - name: '`local_mode` enabled but env "MY_NODE_NAME" not set'
277 + description: |
278 + When `pod.local_mode: true` is set but `MY_NODE_NAME` is missing, the discoverer fails at startup with `local_mode is enabled, but env 'MY_NODE_NAME' not set`. Set the env via the downward API on the Netdata pod (the Helm chart does this).
279 + - name: 'No targets discovered'
280 + description: |
281 + - Confirm pods/services exist in the configured `namespaces[]`.
282 + - If `selector.label` or `selector.field` is set, verify the targets actually carry the matching labels/fields.
283 + - With `local_mode`, only pods on the same node as the Netdata pod are visible.
284 + - name: 'Generated jobs fail to start'
285 + description: |
286 + The Address resolves to the pod's CNI IP — the Netdata Agent must be able to reach pod IPs. Most CNIs allow this from a pod running in the same cluster, but flat-network requirements differ. For service-role targets, the cluster-internal DNS name (`<svc>.<ns>.svc`) is used and should always resolve from inside the cluster.
src/go/plugin/go.d/discovery/sdext/discoverer/netlistensd/README.md new
+1
@@ -0,0 +1 @@
1 +integrations/net_listeners.md
\ No newline at end of file
src/go/plugin/go.d/discovery/sdext/discoverer/netlistensd/integrations/net_listeners.md new
+293
@@ -0,0 +1,293 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/netlistensd/README.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/netlistensd/metadata.yaml"
4 +sidebar_label: "Local listening processes"
5 +learn_status: "Published"
6 +learn_rel_path: "Collecting Metrics/Service Discovery"
7 +keywords: ['service discovery', 'sd', 'net_listeners', 'local processes', 'auto-discovery']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE SERVICE DISCOVERY DISCOVERER'S metadata.yaml FILE"
9 +endmeta-->
10 +
11 +# Local listening processes discovery
12 +
13 +
14 +<img src="https://netdata.cloud/img/netdata.png" width="150"/>
15 +
16 +
17 +Kind: `net_listeners`
18 +
19 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
20 +
21 +## Overview
22 +
23 +Netdata can automatically discover services running on the local host by inspecting the kernel's listening sockets. This is the discoverer that powers Netdata's "zero-config" experience for most database, cache, web-server, and exporter monitoring — if the service is listening on a known port and runs as a recognisable process, Netdata picks it up and starts collecting metrics without you writing any configuration.
24 +
25 +This page covers `net_listeners`-specific setup. For the broader Service Discovery model and the shared template-helper reference, see [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md).
26 +
27 +
28 +### How it works
29 +
30 +Each discovery cycle, the discoverer:
31 +
32 +1. **Reads the kernel's TCP/UDP listening-socket table** via the bundled `local-listeners` helper (which reads `/proc` on Linux, `netstat`-equivalents elsewhere).
33 +2. **Builds one target per `(protocol, IP, port, process)` tuple**, exposing `.Protocol`, `.IPAddress`, `.Port`, `.Comm` (process basename), `.Cmdline` (full command line), and `.Address` (the convenience `IPAddress:Port`).
34 +3. **Caches** each target for 10 minutes so a brief disappearance (process restart) does not churn collector jobs.
35 +4. **Runs the `services:` rules** against each target. The stock conf carries ~100 curated rules covering the bulk of go.d modules (databases, web servers, caches, message queues, exporters).
36 +5. **Reconciles** disappeared listeners — when a process stops listening, its target is removed and the corresponding collector job stops on the next reconcile.
37 +
38 +
39 +### Limitations
40 +
41 +- Only **local** listeners are visible. Discovering services on other hosts requires another discoverer (`http`, `snmp`, `k8s`, or a custom one).
42 +- The discoverer needs to **read kernel socket information**. On Linux this works for processes owned by other users only when Netdata can read the appropriate `/proc/<pid>/net` files; the Netdata installer configures this via the `local-listeners` setuid helper.
43 +- **Containerised services in `host` networking** appear as listeners and are picked up here, not by the Docker discoverer. Services in private container networks must be discovered by the Docker discoverer instead.
44 +- The discoverer does not introspect process runtime — anything beyond port/`comm`/`cmdline` (e.g. config-file path, version, runtime URL prefix) must be inferred via service rules or known by convention.
45 +
46 +
47 +## Setup
48 +
49 +You can configure the `net_listeners` discoverer in two ways:
50 +
51 +| Method | Best for | How to |
52 +|:--|:--|:--|
53 +| [**UI**](#via-ui) | Fast setup without editing files | Go to `Collectors -> go.d -> ServiceDiscovery -> net_listeners`, then add a discovery pipeline. |
54 +| [**File**](#via-file) | File-based configuration or automation | Edit `/etc/netdata/go.d/sd/net_listeners.conf` and define the `discoverer:` and `services:` blocks. |
55 +
56 +### Prerequisites
57 +
58 +#### Discovery is enabled by default
59 +
60 +The stock conf at `/etc/netdata/go.d/sd/net_listeners.conf` ships with `disabled: no` and a curated set of rules. To turn discovery off, set `disabled: yes` at the top of the file.
61 +
62 +
63 +#### Trust the curated stock rules
64 +
65 +The stock rule set covers most commonly-monitored services out of the box (Apache, Nginx, MySQL, PostgreSQL, Redis, RabbitMQ, MongoDB, Elasticsearch, etc.). Most users do not need to author their own rules — start by enabling the relevant collector module and let the stock rules find local instances.
66 +
67 +
68 +### Configuration
69 +
70 +#### Options
71 +
72 +The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn discovered listeners into collector jobs — see [Service Rules](#service-rules)).
73 +
74 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
75 +
76 +
77 +
78 +| Option | Description | Default | Required |
79 +|:-----|:------------|:--------|:---------:|
80 +| interval | How often to re-scan the listening-socket table. | 2m | no |
81 +| timeout | Maximum time to wait for the `local-listeners` helper to return. | 5s | no |
82 +
83 +
84 +
85 +#### via UI
86 +
87 +1. Open the Netdata Dynamic Configuration UI.
88 +2. Go to `Collectors -> go.d -> ServiceDiscovery -> net_listeners`.
89 +3. Add a new discovery pipeline and give it a name.
90 +4. Fill in the discoverer-specific settings and the service rules.
91 +5. Save the discovery pipeline.
92 +
93 +#### via File
94 +
95 +Define the discovery pipeline in `/etc/netdata/go.d/sd/net_listeners.conf`.
96 +
97 +The file has two top-level blocks: `discoverer:` (the options above) and `services:` (rules that turn discovered targets into collector jobs — see [Service Rules](#service-rules)).
98 +
99 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
100 +
101 +##### Examples
102 +
103 +###### Default — keep stock rules
104 +
105 +Most users should not need to touch this file. The stock conf carries ~100 curated rules. Set `disabled: yes` if you want to disable local-listener discovery entirely.
106 +
107 +```yaml
108 +disabled: no
109 +discoverer:
110 + net_listeners: { }
111 +
112 +# services rules ship in the stock conf — the snippet below is illustrative only
113 +services:
114 + - id: redis
115 + match: '{{ or (eq .Port "6379") (eq .Comm "redis-server") }}'
116 + config_template: |
117 + name: local
118 + address: redis://@{{.Address}}
119 +
120 +```
121 +###### Faster scan interval
122 +
123 +Bump the scan rate to once per minute. Useful for very dynamic environments where services come and go often (e.g. ephemeral test runners).
124 +
125 +```yaml
126 +disabled: no
127 +discoverer:
128 + net_listeners:
129 + interval: 1m
130 +services: [ ]
131 +
132 +```
133 +
134 +
135 +## Service Rules
136 +
137 +A `services:` rule turns each discovered listener into one or more collector jobs. The stock conf carries ~100 curated rules — most rules match on a `(.Port, .Comm)` pair (or just `.Comm` / `.Cmdline` for services with non-default ports), and most templates use `name: local` plus the canonical `.Address` to point the collector at the listener.
138 +
139 +The shared rule model — function reference (`match`, `glob`, sprig, `regexFind`, `trimPrefix`, `promPort`), `config_template` rendering rules, and the `missingkey=error` failure semantics — lives on the [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are `net_listeners`-specific.
140 +
141 +
142 +### How rules are evaluated
143 +
144 +Quick reference — see [Rule evaluation semantics](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model.
145 +
146 +
147 +
148 +- **Match by .Port AND .Comm where possible** — The idiomatic stock pattern is `{{ and (eq .Port "PORT") (eq .Comm "PROCESS") }}` — porty + processy. This avoids picking up the wrong process on the right port (e.g. an HTTP exporter listening on `:80` is not Apache).
149 +- **For services with non-standard ports, use .Cmdline + glob** — Some services (Logstash, RabbitMQ, ZooKeeper, Tomcat, …) listen on default ports but identify themselves through their command line, not their `.Comm` (which is a generic `java`, `python`, …). Stock rules for these use `glob .Cmdline "*tomcat*"` or `glob .Cmdline "*rabbitmq*"`.
150 +- **Use `match "sp"` for variant patterns** — When you want to match any of several short patterns, simple-patterns (`match "sp" .Comm "mysqld mariadbd"`) is shorter than nested `or (eq ...) (eq ...)`. See the hub page for matcher types.
151 +- **Module inference from rule id** — For `net_listeners`, set `id: <module-name>` so the rendered job inherits the module name automatically. The stock conf does this throughout (`id: nginx`, `id: postgres`, …).
152 +- **The `exporter` catch-all rule uses `promPort`** — The last rule in the stock conf catches Prometheus exporters by port using the `promPort` helper, which maps a port number to a known exporter module name (or empty if unknown). Keep this rule at the bottom — it overlaps with everything above it.
153 +
154 +### Template Variables
155 +
156 +Available inside both `match` expressions and `config_template` bodies for `net_listeners` targets.
157 +
158 +
159 +| Variable | Type | Description |
160 +|:---------|:-----|:------------|
161 +| `.Protocol` | string | Protocol — `TCP`, `UDP`, `TCP6`, `UDP6`. |
162 +| `.IPAddress` | string | Listening IP address (e.g. `127.0.0.1`, `0.0.0.0`, `::`). |
163 +| `.Port` | string | Listening port. Stock rules typically gate on this with `eq .Port "PORT"`. |
164 +| `.Comm` | string | Process basename (`comm` field — kernel-truncated to 15 chars). Examples: `nginx`, `mysqld`, `redis-server`. Best for native daemons. |
165 +| `.Cmdline` | string | Full process command line. Use `glob .Cmdline "*pattern*"` or `regexFind` (from sprig) to match interpreters that hide the real service name behind `java`/`python`/`node` (e.g. RabbitMQ, Logstash, ZooKeeper, Spigot). |
166 +| `.Address` | string | Convenience `IPAddress:Port` — used in nearly every stock rule template. |
167 +
168 +### Examples
169 +
170 +Each example shows one or more entries from the `services:` array. The full curated rule set lives in the stock conf; the snippets below illustrate the common patterns.
171 +
172 +#### Port + Comm (idiomatic)
173 +
174 +The most common stock-rule shape. Match the canonical port AND the canonical process name.
175 +
176 +```yaml
177 +- id: redis
178 + match: '{{ or (eq .Port "6379") (eq .Comm "redis-server") }}'
179 + config_template: |
180 + name: local
181 + address: redis://@{{.Address}}
182 +
183 +```
184 +
185 +#### Comm + Cmdline glob (for interpreted services)
186 +
187 +When the process is `java`/`python`/`node` and the real service name lives in the command line.
188 +
189 +```yaml
190 +- id: rabbitmq
191 + match: '{{ or (eq .Port "15672") (glob .Cmdline "*rabbitmq*") }}'
192 + config_template: |
193 + name: local
194 + url: http://{{.Address}}
195 + username: guest
196 + password: guest
197 + collect_queues_metrics: no
198 +
199 +```
200 +
201 +#### Multiple jobs from one target (sequence output)
202 +
203 +When one running service should produce multiple collector jobs — for example, MySQL exposing both a Unix-socket DSN and a TCP DSN. The rendered YAML is a top-level sequence; each item becomes a separate job.
204 +
205 +
206 +```yaml
207 +- id: mysql
208 + match: '{{ or (eq .Port "3306") (eq .Comm "mysqld" "mariadbd") }}'
209 + config_template: |
210 + - name: local
211 + dsn: netdata@unix(/var/run/mysqld/mysqld.sock)/
212 + - name: local
213 + dsn: netdata@tcp({{.Address}})/
214 +
215 +```
216 +
217 +#### Prometheus-exporter catch-all (`promPort`)
218 +
219 +The last rule in the stock conf catches generic Prometheus exporters by port. `promPort .Port` returns the well-known module name for that port, or empty. The rule is bottom-most because it overlaps with every preceding curated rule.
220 +
221 +
222 +```yaml
223 +- id: exporter
224 + match: '{{ or (and (not (empty (promPort .Port))) (not (eq .Comm "docker-proxy"))) (glob .Comm "*exporter*") }}'
225 + config_template: |
226 + {{ $name := promPort .Port -}}
227 + {{ if empty $name -}}
228 + {{ $name = printf "%s_%s" .Comm .Port -}}
229 + {{ end -}}
230 + module: prometheus
231 + name: {{$name}}_local
232 + url: http://{{.Address}}/metrics
233 +
234 +```
235 +
236 +
237 +
238 +## Verify discovery worked
239 +
240 +After enabling the discoverer, confirm it is finding listeners and producing jobs.
241 +
242 +### Confirm listeners are being scanned
243 +
244 +Watch the agent log for `discoverer=net_listeners` messages. With systemd:
245 +
246 +```bash
247 +journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep "discoverer=net_listeners"
248 +```
249 +
250 +On a healthy host you should see periodic scan activity. If the log shows `local-listeners` exec failures, the helper binary is missing or not executable.
251 +
252 +
253 +### Confirm the local-listeners helper sees your service
254 +
255 +Run the helper manually to see what targets it would emit:
256 +
257 +```bash
258 +sudo /usr/libexec/netdata/plugins.d/local-listeners
259 +```
260 +
261 +If your service does not appear in the helper output, the kernel's listening-socket table is the place to debug — `ss -tlnp` and `ss -ulnp` should show it.
262 +
263 +
264 +### Confirm jobs are being created
265 +
266 +In the Netdata UI go to `Collectors -> go.d -> <module>` for the module your listener should map to. Stock-rule jobs are typically named `local`.
267 +
268 +
269 +
270 +
271 +## Troubleshooting
272 +
273 +### A locally-running service is not picked up
274 +
275 +Check, in order:
276 +
277 +- Is the service actually listening (`ss -tlnp | grep <name>`)?
278 +- Does the stock conf have a rule for it? See `/etc/netdata/go.d/sd/net_listeners.conf`.
279 +- Is the service on a non-standard port? Stock rules typically gate on the canonical port. Add a rule keyed on `.Comm` or `.Cmdline` for non-default ports.
280 +- Is the process name truncated past 15 chars? `.Comm` is kernel-truncated; use `.Cmdline` instead.
281 +
282 +
283 +### Wrong module picked
284 +
285 +The stock `exporter` catch-all rule (last in the file) is greedy by design — anything on a known Prometheus port gets the `prometheus` module. Add a more specific rule above it if you want a different module to win.
286 +
287 +
288 +### Generated jobs fail to start
289 +
290 +The discoverer creates jobs but does not run them. Common causes: the rendered template assumes credentials the local service rejects (e.g. RabbitMQ default `guest:guest`); `0.0.0.0` listeners produce `0.0.0.0:port` addresses that the collector cannot connect to (use `127.0.0.1` in the template if appropriate); the service has TLS but the template uses HTTP.
291 +
292 +
293 +
src/go/plugin/go.d/discovery/sdext/discoverer/netlistensd/metadata.yaml new
+222
@@ -0,0 +1,222 @@
1 +# yamllint disable rule:line-length
2 +---
3 +id: 'service-discovery-net_listeners'
4 +meta:
5 + kind: 'net_listeners'
6 + name: 'Local listening processes'
7 + tagline: 'Local processes that listen on TCP/UDP ports.'
8 + link: 'https://en.wikipedia.org/wiki/Berkeley_sockets'
9 + icon_filename: 'netdata.png'
10 +keywords:
11 + - 'service discovery'
12 + - 'sd'
13 + - 'net_listeners'
14 + - 'local processes'
15 + - 'auto-discovery'
16 +overview:
17 + description: |
18 + Netdata can automatically discover services running on the local host by inspecting the kernel's listening sockets. This is the discoverer that powers Netdata's "zero-config" experience for most database, cache, web-server, and exporter monitoring — if the service is listening on a known port and runs as a recognisable process, Netdata picks it up and starts collecting metrics without you writing any configuration.
19 +
20 + This page covers `net_listeners`-specific setup. For the broader Service Discovery model and the shared template-helper reference, see [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md).
21 + how_it_works: |
22 + Each discovery cycle, the discoverer:
23 +
24 + 1. **Reads the kernel's TCP/UDP listening-socket table** via the bundled `local-listeners` helper (which reads `/proc` on Linux, `netstat`-equivalents elsewhere).
25 + 2. **Builds one target per `(protocol, IP, port, process)` tuple**, exposing `.Protocol`, `.IPAddress`, `.Port`, `.Comm` (process basename), `.Cmdline` (full command line), and `.Address` (the convenience `IPAddress:Port`).
26 + 3. **Caches** each target for 10 minutes so a brief disappearance (process restart) does not churn collector jobs.
27 + 4. **Runs the `services:` rules** against each target. The stock conf carries ~100 curated rules covering the bulk of go.d modules (databases, web servers, caches, message queues, exporters).
28 + 5. **Reconciles** disappeared listeners — when a process stops listening, its target is removed and the corresponding collector job stops on the next reconcile.
29 + limitations: |
30 + - Only **local** listeners are visible. Discovering services on other hosts requires another discoverer (`http`, `snmp`, `k8s`, or a custom one).
31 + - The discoverer needs to **read kernel socket information**. On Linux this works for processes owned by other users only when Netdata can read the appropriate `/proc/<pid>/net` files; the Netdata installer configures this via the `local-listeners` setuid helper.
32 + - **Containerised services in `host` networking** appear as listeners and are picked up here, not by the Docker discoverer. Services in private container networks must be discovered by the Docker discoverer instead.
33 + - The discoverer does not introspect process runtime — anything beyond port/`comm`/`cmdline` (e.g. config-file path, version, runtime URL prefix) must be inferred via service rules or known by convention.
34 +setup:
35 + prerequisites:
36 + list:
37 + - title: 'Discovery is enabled by default'
38 + description: |
39 + The stock conf at `/etc/netdata/go.d/sd/net_listeners.conf` ships with `disabled: no` and a curated set of rules. To turn discovery off, set `disabled: yes` at the top of the file.
40 + - title: 'Trust the curated stock rules'
41 + description: |
42 + The stock rule set covers most commonly-monitored services out of the box (Apache, Nginx, MySQL, PostgreSQL, Redis, RabbitMQ, MongoDB, Elasticsearch, etc.). Most users do not need to author their own rules — start by enabling the relevant collector module and let the stock rules find local instances.
43 + configuration:
44 + file:
45 + name: 'go.d/sd/net_listeners.conf'
46 + options:
47 + description: |
48 + The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn discovered listeners into collector jobs — see [Service Rules](#service-rules)).
49 +
50 + After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
51 + folding:
52 + title: 'Discoverer options'
53 + enabled: false
54 + list:
55 + - name: 'interval'
56 + description: 'How often to re-scan the listening-socket table.'
57 + default_value: '2m'
58 + required: false
59 + - name: 'timeout'
60 + description: 'Maximum time to wait for the `local-listeners` helper to return.'
61 + default_value: '5s'
62 + required: false
63 + examples:
64 + folding:
65 + title: 'Configuration examples'
66 + enabled: true
67 + list:
68 + - name: 'Default — keep stock rules'
69 + description: 'Most users should not need to touch this file. The stock conf carries ~100 curated rules. Set `disabled: yes` if you want to disable local-listener discovery entirely.'
70 + config: |
71 + disabled: no
72 + discoverer:
73 + net_listeners: { }
74 +
75 + # services rules ship in the stock conf — the snippet below is illustrative only
76 + services:
77 + - id: redis
78 + match: '{{ or (eq .Port "6379") (eq .Comm "redis-server") }}'
79 + config_template: |
80 + name: local
81 + address: redis://@{{.Address}}
82 + - name: 'Faster scan interval'
83 + description: 'Bump the scan rate to once per minute. Useful for very dynamic environments where services come and go often (e.g. ephemeral test runners).'
84 + config: |
85 + disabled: no
86 + discoverer:
87 + net_listeners:
88 + interval: 1m
89 + services: [ ]
90 +services:
91 + description: |
92 + A `services:` rule turns each discovered listener into one or more collector jobs. The stock conf carries ~100 curated rules — most rules match on a `(.Port, .Comm)` pair (or just `.Comm` / `.Cmdline` for services with non-default ports), and most templates use `name: local` plus the canonical `.Address` to point the collector at the listener.
93 +
94 + The shared rule model — function reference (`match`, `glob`, sprig, `regexFind`, `trimPrefix`, `promPort`), `config_template` rendering rules, and the `missingkey=error` failure semantics — lives on the [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are `net_listeners`-specific.
95 + evaluation:
96 + description: |
97 + Quick reference — see [Rule evaluation semantics](/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model.
98 + list:
99 + - name: 'Match by .Port AND .Comm where possible'
100 + description: |
101 + The idiomatic stock pattern is `{{ and (eq .Port "PORT") (eq .Comm "PROCESS") }}` — porty + processy. This avoids picking up the wrong process on the right port (e.g. an HTTP exporter listening on `:80` is not Apache).
102 + - name: 'For services with non-standard ports, use .Cmdline + glob'
103 + description: |
104 + Some services (Logstash, RabbitMQ, ZooKeeper, Tomcat, …) listen on default ports but identify themselves through their command line, not their `.Comm` (which is a generic `java`, `python`, …). Stock rules for these use `glob .Cmdline "*tomcat*"` or `glob .Cmdline "*rabbitmq*"`.
105 + - name: 'Use `match "sp"` for variant patterns'
106 + description: |
107 + When you want to match any of several short patterns, simple-patterns (`match "sp" .Comm "mysqld mariadbd"`) is shorter than nested `or (eq ...) (eq ...)`. See the hub page for matcher types.
108 + - name: 'Module inference from rule id'
109 + description: 'For `net_listeners`, set `id: <module-name>` so the rendered job inherits the module name automatically. The stock conf does this throughout (`id: nginx`, `id: postgres`, …).'
110 + - name: 'The `exporter` catch-all rule uses `promPort`'
111 + description: |
112 + The last rule in the stock conf catches Prometheus exporters by port using the `promPort` helper, which maps a port number to a known exporter module name (or empty if unknown). Keep this rule at the bottom — it overlaps with everything above it.
113 + template_variables:
114 + description: 'Available inside both `match` expressions and `config_template` bodies for `net_listeners` targets.'
115 + list:
116 + - name: '.Protocol'
117 + type: 'string'
118 + description: 'Protocol — `TCP`, `UDP`, `TCP6`, `UDP6`.'
119 + - name: '.IPAddress'
120 + type: 'string'
121 + description: 'Listening IP address (e.g. `127.0.0.1`, `0.0.0.0`, `::`).'
122 + - name: '.Port'
123 + type: 'string'
124 + description: 'Listening port. Stock rules typically gate on this with `eq .Port "PORT"`.'
125 + - name: '.Comm'
126 + type: 'string'
127 + description: 'Process basename (`comm` field — kernel-truncated to 15 chars). Examples: `nginx`, `mysqld`, `redis-server`. Best for native daemons.'
128 + - name: '.Cmdline'
129 + type: 'string'
130 + description: 'Full process command line. Use `glob .Cmdline "*pattern*"` or `regexFind` (from sprig) to match interpreters that hide the real service name behind `java`/`python`/`node` (e.g. RabbitMQ, Logstash, ZooKeeper, Spigot).'
131 + - name: '.Address'
132 + type: 'string'
133 + description: 'Convenience `IPAddress:Port` — used in nearly every stock rule template.'
134 + examples:
135 + description: 'Each example shows one or more entries from the `services:` array. The full curated rule set lives in the stock conf; the snippets below illustrate the common patterns.'
136 + list:
137 + - name: 'Port + Comm (idiomatic)'
138 + description: 'The most common stock-rule shape. Match the canonical port AND the canonical process name.'
139 + config: |
140 + - id: redis
141 + match: '{{ or (eq .Port "6379") (eq .Comm "redis-server") }}'
142 + config_template: |
143 + name: local
144 + address: redis://@{{.Address}}
145 + - name: 'Comm + Cmdline glob (for interpreted services)'
146 + description: 'When the process is `java`/`python`/`node` and the real service name lives in the command line.'
147 + config: |
148 + - id: rabbitmq
149 + match: '{{ or (eq .Port "15672") (glob .Cmdline "*rabbitmq*") }}'
150 + config_template: |
151 + name: local
152 + url: http://{{.Address}}
153 + username: guest
154 + password: guest
155 + collect_queues_metrics: no
156 + - name: 'Multiple jobs from one target (sequence output)'
157 + description: |
158 + When one running service should produce multiple collector jobs — for example, MySQL exposing both a Unix-socket DSN and a TCP DSN. The rendered YAML is a top-level sequence; each item becomes a separate job.
159 + config: |
160 + - id: mysql
161 + match: '{{ or (eq .Port "3306") (eq .Comm "mysqld" "mariadbd") }}'
162 + config_template: |
163 + - name: local
164 + dsn: netdata@unix(/var/run/mysqld/mysqld.sock)/
165 + - name: local
166 + dsn: netdata@tcp({{.Address}})/
167 + - name: 'Prometheus-exporter catch-all (`promPort`)'
168 + description: |
169 + The last rule in the stock conf catches generic Prometheus exporters by port. `promPort .Port` returns the well-known module name for that port, or empty. The rule is bottom-most because it overlaps with every preceding curated rule.
170 + config: |
171 + - id: exporter
172 + match: '{{ or (and (not (empty (promPort .Port))) (not (eq .Comm "docker-proxy"))) (glob .Comm "*exporter*") }}'
173 + config_template: |
174 + {{ $name := promPort .Port -}}
175 + {{ if empty $name -}}
176 + {{ $name = printf "%s_%s" .Comm .Port -}}
177 + {{ end -}}
178 + module: prometheus
179 + name: {{$name}}_local
180 + url: http://{{.Address}}/metrics
181 +verify:
182 + description: 'After enabling the discoverer, confirm it is finding listeners and producing jobs.'
183 + checks:
184 + list:
185 + - name: 'Confirm listeners are being scanned'
186 + description: |
187 + Watch the agent log for `discoverer=net_listeners` messages. With systemd:
188 +
189 + ```bash
190 + journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep "discoverer=net_listeners"
191 + ```
192 +
193 + On a healthy host you should see periodic scan activity. If the log shows `local-listeners` exec failures, the helper binary is missing or not executable.
194 + - name: 'Confirm the local-listeners helper sees your service'
195 + description: |
196 + Run the helper manually to see what targets it would emit:
197 +
198 + ```bash
199 + sudo /usr/libexec/netdata/plugins.d/local-listeners
200 + ```
201 +
202 + If your service does not appear in the helper output, the kernel's listening-socket table is the place to debug — `ss -tlnp` and `ss -ulnp` should show it.
203 + - name: 'Confirm jobs are being created'
204 + description: |
205 + In the Netdata UI go to `Collectors -> go.d -> <module>` for the module your listener should map to. Stock-rule jobs are typically named `local`.
206 +troubleshooting:
207 + problems:
208 + list:
209 + - name: 'A locally-running service is not picked up'
210 + description: |
211 + Check, in order:
212 +
213 + - Is the service actually listening (`ss -tlnp | grep <name>`)?
214 + - Does the stock conf have a rule for it? See `/etc/netdata/go.d/sd/net_listeners.conf`.
215 + - Is the service on a non-standard port? Stock rules typically gate on the canonical port. Add a rule keyed on `.Comm` or `.Cmdline` for non-default ports.
216 + - Is the process name truncated past 15 chars? `.Comm` is kernel-truncated; use `.Cmdline` instead.
217 + - name: 'Wrong module picked'
218 + description: |
219 + The stock `exporter` catch-all rule (last in the file) is greedy by design — anything on a known Prometheus port gets the `prometheus` module. Add a more specific rule above it if you want a different module to win.
220 + - name: 'Generated jobs fail to start'
221 + description: |
222 + The discoverer creates jobs but does not run them. Common causes: the rendered template assumes credentials the local service rejects (e.g. RabbitMQ default `guest:guest`); `0.0.0.0` listeners produce `0.0.0.0:port` addresses that the collector cannot connect to (use `127.0.0.1` in the template if appropriate); the service has TLS but the template uses HTTP.
src/go/plugin/go.d/discovery/sdext/discoverer/snmpsd/README.md new
+1
@@ -0,0 +1 @@
1 +integrations/snmp.md
\ No newline at end of file
src/go/plugin/go.d/discovery/sdext/discoverer/snmpsd/integrations/snmp.md new
+458
@@ -0,0 +1,458 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/snmpsd/README.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/discovery/sdext/discoverer/snmpsd/metadata.yaml"
4 +sidebar_label: "SNMP"
5 +learn_status: "Published"
6 +learn_rel_path: "Collecting Metrics/Service Discovery"
7 +keywords: ['service discovery', 'sd', 'snmp', 'snmpv3', 'usm', 'network', 'network devices', 'discovery']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE SERVICE DISCOVERY DISCOVERER'S metadata.yaml FILE"
9 +endmeta-->
10 +
11 +# SNMP discovery
12 +
13 +
14 +<img src="https://netdata.cloud/img/SNMP.png" width="150"/>
15 +
16 +
17 +Kind: `snmp`
18 +
19 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
20 +
21 +## Overview
22 +
23 +Netdata can automatically discover SNMP-capable devices on your network and generate `snmp` collector jobs for each one. Configure the IP ranges to scan and the SNMP credentials to try, and the discoverer probes each address, reads basic system information, and produces collector configurations from a set of customisable service rules.
24 +
25 +This page covers SNMP-specific setup. For the broader Service Discovery model (`discoverer:` and `services:` blocks, rule evaluation order, and the full template helper reference shared by all discoverers), see [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md).
26 +
27 +
28 +### How it works
29 +
30 +Each discovery cycle, the discoverer:
31 +
32 +1. **Iterates** every IP in the configured `networks[]` subnets, in parallel.
33 +2. **Probes** each IP over UDP/161 using the credential bound to that subnet, walking the standard `system` MIB (`sysDescr`, `sysName`, `sysContact`, `sysLocation`, `sysObjectID`).
34 +3. **Caches** the result in a status file so re-probing is skipped while `device_cache_ttl` has not expired.
35 +4. **Emits a target** per reachable device, exposing `.IPAddress`, `.SysInfo.*`, and `.Credential.*` to the rule engine.
36 +5. **Runs the `services:` rules** against each target. The rules render Go templates to produce one (or more) `snmp` collector job configurations.
37 +
38 +The discoverer never queries device-specific OIDs — those are queried later by the `snmp` collector once the job is created.
39 +
40 +
41 +### Limitations
42 +
43 +- Each subnet is capped at 512 IP addresses (a `/23` network or smaller). Split larger ranges into multiple `networks[]` entries.
44 +- Discovery uses **UDP/161**. The Netdata Agent host must be able to reach that port on every scanned IP, and any device-side ACLs must allow the Netdata host.
45 +- **One credential per subnet**: each `networks[]` entry is bound to exactly one credential. There is no automatic credential fallback. To probe the same subnet with multiple credentials, list it twice with different `credential` values; each device that responds to either credential will appear as a target (with its responding credential exposed via `.Credential.*`).
46 +- **Outbound interface**: probes use the host's default routing. There is no per-pipeline bind-address or VRF option — on multi-homed hosts, configure the OS routing table so the Netdata host reaches each target subnet via the correct interface.
47 +- The discoverer reads only the standard `system` MIB. Vendor-specific identification (`.SysInfo.Vendor`, `.Category`, `.Model`) is derived from `sysObjectID` and an enterprise-numbers table; values may be empty or `Unknown` for devices that are not in that table.
48 +- **SNMPv3 engine ID**: gosnmp negotiates the engine ID at the start of each probe (one extra round-trip per probe — usually irrelevant unless `parallel_scans_per_network` is high and the device is rate-limited). Engine IDs are not cached across probes, so devices that rotate engine IDs (rare; some HA pairs do this on failover) are handled transparently.
49 +- **Credential storage**: community strings and SNMPv3 passphrases are stored in plaintext both in `/etc/netdata/go.d/sd/snmp.conf` (file-based pipelines) and in the agent's dynamic-configuration store under `/var/lib/netdata/dyncfg/` (UI-managed pipelines). To avoid plaintext credentials on disk in either path, reference them via `${env:VAR}` or `${file:/path}` (see [Secrets Management](https://github.com/netdata/netdata/blob/master/src/collectors/SECRETS.md)).
50 +
51 +
52 +## Setup
53 +
54 +You can configure the `snmp` discoverer in two ways:
55 +
56 +| Method | Best for | How to |
57 +|:--|:--|:--|
58 +| [**UI**](#via-ui) | Fast setup without editing files | Go to `Collectors -> go.d -> ServiceDiscovery -> snmp`, then add a discovery pipeline. |
59 +| [**File**](#via-file) | File-based configuration or automation | Edit `/etc/netdata/go.d/sd/snmp.conf` and define the `discoverer:` and `services:` blocks. |
60 +
61 +### Prerequisites
62 +
63 +#### Plan your IP ranges and credentials
64 +
65 +Decide which subnets to scan and which SNMP credentials apply to each. SNMPv1 and SNMPv2c need a community string. SNMPv3 needs a USM username, security level, and (depending on the level) authentication and privacy passphrases.
66 +
67 +
68 +#### Allow UDP/161 reachability
69 +
70 +The Netdata Agent host must be able to reach UDP port 161 on every scanned IP. SNMP devices typically restrict which clients can query them — make sure the Netdata host is allowed by any device-side ACLs.
71 +
72 +
73 +### Configuration
74 +
75 +#### Options
76 +
77 +The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn discovered devices into `snmp` collector jobs — see [Service Rules](#service-rules)).
78 +
79 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
80 +
81 +
82 +
83 +| Option | Description | Default | Required |
84 +|:-----|:------------|:--------|:---------:|
85 +| [rescan_interval](#option-rescan-interval) | How often to rescan configured networks for devices. | 30m | no |
86 +| timeout | Maximum time to wait for an SNMP device response. | 1s | no |
87 +| [device_cache_ttl](#option-device-cache-ttl) | How long to trust cached discovery results before re-probing a device. | 12h | no |
88 +| parallel_scans_per_network | How many IPs to probe concurrently within each subnet. | 32 | no |
89 +| [credentials](#option-credentials) | List of SNMP credentials referenced by entries in `networks`. At least one credential is required. | | yes |
90 +| [networks](#option-networks) | List of subnets to scan, each tagged with the credential name to use. At least one network is required. | | yes |
91 +
92 +<a id="option-rescan-interval"></a>
93 +##### rescan_interval
94 +
95 +Set to `0` to perform a single discovery scan when the agent starts and never rescan. Negative values also disable rescanning.
96 +
97 +
98 +<a id="option-device-cache-ttl"></a>
99 +##### device_cache_ttl
100 +
101 +Set to `0` to never expire cached results — once a device is discovered it is never re-probed (until the agent restarts and the cache is invalidated by configuration changes).
102 +
103 +
104 +<a id="option-credentials"></a>
105 +##### credentials
106 +
107 +Each credential has a `name` (used by `networks[].credential`) and a `version`.
108 +
109 +**Accepted `version` values:** `1`, `2`, `2c`, `3`. (`2` is an alias for `2c`.)
110 +
111 +For SNMPv1 and SNMPv2c, set `community`.
112 +
113 +For SNMPv3, set:
114 +
115 +- `username` — USM user name.
116 +- `security_level` — one of `noAuthNoPriv`, `authNoPriv`, `authPriv`.
117 +- `auth_protocol` — one of `md5`, `sha` (HMAC-SHA-1, RFC 3414), `sha224`, `sha256`, `sha384`, `sha512` (HMAC-SHA-2, RFC 7860). Required for `authNoPriv` and `authPriv`.
118 +- `auth_password` — authentication passphrase. Required when `auth_protocol` is set.
119 +- `priv_protocol` — one of `des`, `aes` (AES-128), `aes192`, `aes256`, `aes192c`, `aes256c`. The `c` variants are the Cisco/Reeder draft; check your device's `show snmp user` output to pick the matching one. Required for `authPriv`.
120 +- `priv_password` — privacy passphrase. Required when `priv_protocol` is set.
121 +- `context_name` — only set this if your devices use a non-default SNMPv3 context.
122 +
123 +**Naming note:** the YAML keys are `auth_password` and `priv_password`. The same fields are exposed inside service rule templates as `.Credential.AuthPassphrase` and `.Credential.PrivacyPassphrase` (the Go struct names). Both refer to the same value.
124 +
125 +**Avoid plaintext on disk:** any of these fields can be sourced from environment variables or files using `${env:VAR_NAME}` or `${file:/absolute/path}` — see [Secrets Management](https://github.com/netdata/netdata/blob/master/src/collectors/SECRETS.md).
126 +
127 +
128 +<a id="option-networks"></a>
129 +##### networks
130 +
131 +Each entry needs `subnet` (an IP range) and `credential` (the name of an entry from `credentials`).
132 +
133 +**Supported subnet formats** (IPv4 and IPv6):
134 +
135 +- CIDR — `192.168.1.0/24`, `2001:db8::/120`
136 +- Range — `10.0.0.1-10.0.0.50`, `2001:db8::-2001:db8::ff`
137 +- Subnet mask — `192.168.1.0/255.255.255.0`
138 +- Single IP — `192.168.1.10`, `2001:db8::1`
139 +
140 +Maximum **512 IPs** per subnet entry. Split larger blocks across multiple entries.
141 +
142 +For CIDR notation, network and broadcast addresses are excluded (except `/31`, `/32`, `/127`, `/128`).
143 +
144 +
145 +
146 +
147 +#### via UI
148 +
149 +1. Open the Netdata Dynamic Configuration UI.
150 +2. Go to `Collectors -> go.d -> ServiceDiscovery -> snmp`.
151 +3. Add a new discovery pipeline and give it a name.
152 +4. Fill in the discoverer-specific settings and the service rules.
153 +5. Save the discovery pipeline.
154 +
155 +#### via File
156 +
157 +Define the discovery pipeline in `/etc/netdata/go.d/sd/snmp.conf`.
158 +
159 +The file has two top-level blocks: `discoverer:` (the options above) and `services:` (rules that turn discovered targets into collector jobs — see [Service Rules](#service-rules)).
160 +
161 +After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
162 +
163 +##### Examples
164 +
165 +###### Single subnet, SNMPv2c
166 +
167 +Scan a single /24 with the default `public` community.
168 +
169 +```yaml
170 +disabled: no
171 +discoverer:
172 + snmp:
173 + credentials:
174 + - name: public-v2c
175 + version: 2c
176 + community: public
177 + networks:
178 + - subnet: 192.168.1.0/24
179 + credential: public-v2c
180 +services:
181 + - id: snmp
182 + match: '{{ true }}'
183 +
184 +```
185 +###### Multiple subnets, mixed SNMPv2c and SNMPv3
186 +
187 +Mix SNMPv2c on one subnet with SNMPv3 (authPriv) on another. Credentials are referenced from environment variables to keep them out of plaintext on disk.
188 +
189 +```yaml
190 +disabled: no
191 +discoverer:
192 + snmp:
193 + rescan_interval: 1h
194 + credentials:
195 + - name: public-v2c
196 + version: 2c
197 + community: ${env:SNMP_V2C_COMMUNITY}
198 + - name: secure-v3
199 + version: 3
200 + security_level: authPriv
201 + username: netdata-monitor
202 + auth_protocol: sha256
203 + auth_password: ${env:SNMP_V3_AUTH}
204 + priv_protocol: aes256
205 + priv_password: ${env:SNMP_V3_PRIV}
206 + networks:
207 + - subnet: 192.168.10.0/24
208 + credential: public-v2c
209 + - subnet: 10.20.30.0/24
210 + credential: secure-v3
211 +services:
212 + - id: snmp
213 + match: '{{ true }}'
214 +
215 +```
216 +###### IPv6 subnet
217 +
218 +Scan a small IPv6 range with SNMPv2c.
219 +
220 +```yaml
221 +disabled: no
222 +discoverer:
223 + snmp:
224 + credentials:
225 + - name: public-v2c
226 + version: 2c
227 + community: public
228 + networks:
229 + - subnet: 2001:db8:0:1::/120
230 + credential: public-v2c
231 +services:
232 + - id: snmp
233 + match: '{{ true }}'
234 +
235 +```
236 +
237 +
238 +## Service Rules
239 +
240 +A `services:` rule turns each discovered SNMP device into one or more `snmp` collector jobs. Each rule has an `id`, a Go-template `match` expression that decides whether the rule applies to the device, and an optional `config_template` that renders the collector job YAML when the rule matches.
241 +
242 +The default rule shipped with Netdata (`{{ true }}`) creates one job per discovered device and handles both SNMPv2 and SNMPv3 — most users never change it. Customise rules when you want vendor-specific configs (Cisco vs. Juniper, printers vs. routers), per-VLAN overrides, or multiple jobs per device.
243 +
244 +The shared rule model — function reference (sprig + Netdata helpers `match`, `glob`, `promPort`, `toYaml`), `config_template` rendering rules, and the strict-missing-key error semantics — lives on the [Service Discovery](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are SNMP-specific.
245 +
246 +
247 +### How rules are evaluated
248 +
249 +Quick reference — see [Rule evaluation semantics](https://github.com/netdata/netdata/blob/master/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model (sequence-output multi-job rendering, module inference from `id`, `missingkey=error`, ordering recommendations).
250 +
251 +
252 +
253 +- **Skip rule (no config_template)** — A matching rule with no `config_template` drops the device immediately — no job, no further rule evaluation. Use it to exclude devices the catch-all would otherwise pick up. Place **before** any template rule.
254 +- **Template rule (with config_template)** — A matching rule with a `config_template` produces one or more jobs and rule evaluation **continues**. A single device can therefore produce jobs from several matching rules.
255 +- **For SNMP specifically** — Set `id: snmp` so the rendered job inherits the `snmp` module name automatically, or include `module: snmp` explicitly inside the `config_template` (required when `id` is anything else, e.g. `cisco`).
256 +
257 +### Template Variables
258 +
259 +Available inside both `match` expressions and `config_template` bodies. All variables are strings; empty values render as the empty string.
260 +
261 +
262 +
263 +| Variable | Type | Description |
264 +|:---------|:-----|:------------|
265 +| `.IPAddress` | string | IP address of the discovered device. Always set. |
266 +| `.SysInfo.Descr` | string | Value of `sysDescr.0` (vendor-supplied free-form description). May be empty. |
267 +| `.SysInfo.Contact` | string | Value of `sysContact.0`. May be empty. |
268 +| `.SysInfo.Name` | string | Value of `sysName.0` (typically the device hostname or FQDN). Defaults to the literal string `unknown` when the device does not return one. |
269 +| `.SysInfo.Location` | string | Value of `sysLocation.0`. May be empty. |
270 +| `.SysInfo.Organization` | string | Vendor or organization parsed from `sysObjectID` against the embedded enterprise-numbers table. Defaults to `Unknown` when the OID is not in the table. |
271 +| `.SysInfo.Vendor` | string | Vendor name inferred from `sysObjectID` and `sysDescr` via the bundled overrides. Empty when no override matches. |
272 +| `.SysInfo.Category` | string | Device category (e.g. `router`, `switch`, `printer`). Sourced from the bundled SNMP overrides; empty when no override matches the device. The set of category values is determined by the overrides, not a closed enum. |
273 +| `.SysInfo.Model` | string | Device model inferred from `sysObjectID` and `sysDescr` via the bundled overrides. Empty when no override matches. |
274 +| `.Credential.Name` | string | Name of the credential entry that successfully probed the device. |
275 +| `.Credential.Version` | string | Configured version string: `1`, `2`, `2c`, or `3`. Use `eq .Credential.Version "1" "2" "2c"` to branch v1/v2c vs v3. |
276 +| `.Credential.Community` | string | Community string (SNMPv1/v2c). Empty for SNMPv3. |
277 +| `.Credential.UserName` | string | SNMPv3 USM user name. Empty for v1/v2c. |
278 +| `.Credential.SecurityLevel` | string | SNMPv3 security level (`noAuthNoPriv`, `authNoPriv`, `authPriv`). |
279 +| `.Credential.AuthProtocol` | string | SNMPv3 auth protocol (`md5`, `sha`, `sha224`–`sha512`). |
280 +| `.Credential.AuthPassphrase` | string | SNMPv3 authentication passphrase. **YAML key for the same value: `auth_password`** (see [credentials option](#option-credentials)). |
281 +| `.Credential.PrivacyProtocol` | string | SNMPv3 privacy protocol (`des`, `aes`, `aes192`, `aes256`, `aes192c`, `aes256c`). |
282 +| `.Credential.PrivacyPassphrase` | string | SNMPv3 privacy passphrase. **YAML key for the same value: `priv_password`** (see [credentials option](#option-credentials)). |
283 +
284 +### Examples
285 +
286 +Each example shows one or more entries from the `services:` array. Order matters — see [How rules are evaluated](#how-rules-are-evaluated).
287 +
288 +#### Default catch-all rule
289 +
290 +Generate one `snmp` collector job per discovered device. This is the rule produced by the stock conf and is sufficient for most deployments — it handles both SNMPv2 and SNMPv3 by branching on `.Credential.Version`. The `id: snmp` makes the module name infer to `snmp` automatically.
291 +
292 +
293 +```yaml
294 +- id: snmp
295 + match: '{{ true }}'
296 + config_template: |
297 + {{- if .SysInfo.Name }}
298 + name: {{ .SysInfo.Name }}-ip-{{ .IPAddress }}
299 + {{- else }}
300 + name: ip-{{ .IPAddress }}
301 + {{- end }}
302 + hostname: {{ .IPAddress }}
303 + options:
304 + version: {{ .Credential.Version }}
305 + {{- if eq .Credential.Version "1" "2" "2c" }}
306 + community: {{ .Credential.Community }}
307 + {{- else }}
308 + user:
309 + name: {{ .Credential.UserName }}
310 + level: {{ .Credential.SecurityLevel }}
311 + auth_proto: {{ .Credential.AuthProtocol }}
312 + auth_key: {{ .Credential.AuthPassphrase }}
313 + priv_proto: {{ .Credential.PrivacyProtocol }}
314 + priv_key: {{ .Credential.PrivacyPassphrase }}
315 + {{- end }}
316 +
317 +```
318 +
319 +#### Skip rule for management VIPs
320 +
321 +Drop devices whose `sysName` starts with `vip-` so they are not monitored. A skip rule is a rule with **no** `config_template`. Place it before the catch-all so its match wins first. (The second rule below is the [Default catch-all rule](#default-catch-all-rule) — paste its full body in place of the placeholder comment.)
322 +
323 +
324 +```yaml
325 +- id: skip-vips
326 + match: '{{ glob .SysInfo.Name "vip-*" }}'
327 +- id: snmp
328 + match: '{{ true }}'
329 + config_template: |
330 + # paste the body from the "Default catch-all rule" example here
331 +
332 +```
333 +
334 +#### Vendor-specific override (Cisco) with catch-all suppressed
335 +
336 +Apply a Cisco-specific config to devices whose vendor matches `Cisco*`, then prevent the catch-all from producing a duplicate job for the same devices. This is the recommended three-rule pattern for any vendor-specific override:
337 +
338 +1. **Specific template rule** (`cisco`) — renders the Cisco-tuned job. `id: cisco` does **not** map to a real collector module, so `module: snmp` is set explicitly in the template.
339 +2. **Skip rule** (`skip-cisco-from-catchall`) — drops Cisco devices from the remaining pipeline so step 3 does not also fire for them.
340 +3. **Catch-all template rule** (`snmp`) — handles every non-Cisco device.
341 +
342 +
343 +```yaml
344 +- id: cisco
345 + match: '{{ glob .SysInfo.Vendor "Cisco*" }}'
346 + config_template: |
347 + module: snmp
348 + name: cisco-{{ .SysInfo.Name }}-{{ .IPAddress }}
349 + hostname: {{ .IPAddress }}
350 + options:
351 + version: {{ .Credential.Version }}
352 + {{- if eq .Credential.Version "1" "2" "2c" }}
353 + community: {{ .Credential.Community }}
354 + {{- else }}
355 + user:
356 + name: {{ .Credential.UserName }}
357 + level: {{ .Credential.SecurityLevel }}
358 + auth_proto: {{ .Credential.AuthProtocol }}
359 + auth_key: {{ .Credential.AuthPassphrase }}
360 + priv_proto: {{ .Credential.PrivacyProtocol }}
361 + priv_key: {{ .Credential.PrivacyPassphrase }}
362 + {{- end }}
363 +
364 +- id: skip-cisco-from-catchall
365 + match: '{{ glob .SysInfo.Vendor "Cisco*" }}'
366 +
367 +- id: snmp
368 + match: '{{ true }}'
369 + config_template: |
370 + # ... (same as the catch-all in the Default catch-all rule example above)
371 +
372 +```
373 +
374 +#### Category-based rule (HP printers)
375 +
376 +Match by `.SysInfo.Category` to apply a printer-specific config. Category values are populated from the bundled SNMP overrides — `printer` is one of the standard categories produced by the override file shipped with Netdata. Pair with a follow-up skip rule the same way as the Cisco example if you want to suppress the catch-all for printers.
377 +
378 +
379 +```yaml
380 +- id: hp-printer
381 + match: '{{ and (eq .SysInfo.Category "printer") (glob .SysInfo.Vendor "HP*" "Hewlett*") }}'
382 + config_template: |
383 + module: snmp
384 + name: printer-{{ .SysInfo.Name }}-{{ .IPAddress }}
385 + hostname: {{ .IPAddress }}
386 + update_every: 30
387 + options:
388 + version: {{ .Credential.Version }}
389 + community: {{ .Credential.Community }}
390 +
391 +```
392 +
393 +
394 +
395 +## Verify discovery worked
396 +
397 +After enabling the discoverer, confirm it is finding devices and producing jobs.
398 +
399 +### Confirm devices are being probed
400 +
401 +Watch the agent log for SNMP discoverer messages. On a successful probe you should see lines like:
402 +
403 +```text
404 +discoverer=snmp ... device '192.168.1.10': successfully discovered (sysName: 'sw01.example.com', network: '192.168.1.0/24')
405 +```
406 +
407 +With systemd:
408 +
409 +```bash
410 +journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep "discoverer=snmp"
411 +```
412 +
413 +Without systemd:
414 +
415 +```bash
416 +grep "discoverer=snmp" /var/log/netdata/collector.log
417 +```
418 +
419 +
420 +### Confirm jobs are being created
421 +
422 +Discovered devices should produce `snmp` collector jobs. In the Netdata UI go to `Collectors -> go.d -> snmp` — each discovered device appears as a job named according to your `config_template` (the default catch-all renders `<sysName>-ip-<address>`).
423 +
424 +
425 +### Confirm metrics are being collected
426 +
427 +Once a job exists, the `snmp` collector takes over and starts collecting metrics. Charts for each device appear under the SNMP integration on the dashboard. If a job is created but metrics never appear, the problem is in the `snmp` collector configuration (rendered by your `config_template`), not in the discoverer.
428 +
429 +
430 +
431 +
432 +## Troubleshooting
433 +
434 +### No devices are discovered
435 +
436 +Check the agent log for `discoverer=snmp` messages. Common causes:
437 +
438 +- The configured subnets do not match where your devices live. Verify with `ping` / `arp` from the Netdata host.
439 +- UDP port 161 is blocked between the Netdata host and the devices. Test with `nc -zu <ip> 161` or `snmpwalk -v2c -c <community> <ip> sysDescr.0`.
440 +- The credentials do not match what the devices accept. SNMPv3 mismatches commonly produce `authentication failure` or `decryption error` log lines.
441 +- A configured subnet exceeds the 512-IP cap and the discoverer rejected it at startup. Look for `subnet '...' exceeds maximum size of /23` in the log.
442 +
443 +
444 +### Wrong devices are matched by a rule
445 +
446 +Rule order matters — see [How rules are evaluated](#how-rules-are-evaluated). Place vendor-specific or device-specific rules **before** the catch-all. If you need to suppress the catch-all for a subset of devices, follow the specific rule with a skip rule (no `config_template`) keyed on the same condition.
447 +
448 +
449 +### Generated collector jobs fail to start
450 +
451 +The discoverer creates jobs but does not run them — the `snmp` collector does. Check the `snmp` collector log and the rendered job YAML in the agent's debug output. Common causes:
452 +
453 +- The rendered `config_template` produces invalid YAML for some discovered field values (for example, unescaped colons in `sysName`). YAML-quote dynamic values when in doubt.
454 +- Module name mismatch — the rule `id` (or explicit `module:` field) does not match `snmp`.
455 +- SNMPv3 credentials succeeded for `system` MIB during discovery but the collector cannot read other OIDs (different VACM view); confirm with `snmpwalk` against the device.
456 +
457 +
458 +
src/go/plugin/go.d/discovery/sdext/discoverer/snmpsd/metadata.yaml new
+400
@@ -0,0 +1,400 @@
1 +# yamllint disable rule:line-length
2 +---
3 +id: 'service-discovery-snmp'
4 +meta:
5 + kind: 'snmp'
6 + name: 'SNMP'
7 + tagline: 'SNMP-capable devices on configured network subnets.'
8 + link: 'https://en.wikipedia.org/wiki/Simple_Network_Management_Protocol'
9 + icon_filename: 'SNMP.png'
10 +keywords:
11 + - 'service discovery'
12 + - 'sd'
13 + - 'snmp'
14 + - 'snmpv3'
15 + - 'usm'
16 + - 'network'
17 + - 'network devices'
18 + - 'discovery'
19 +overview:
20 + description: |
21 + Netdata can automatically discover SNMP-capable devices on your network and generate `snmp` collector jobs for each one. Configure the IP ranges to scan and the SNMP credentials to try, and the discoverer probes each address, reads basic system information, and produces collector configurations from a set of customisable service rules.
22 +
23 + This page covers SNMP-specific setup. For the broader Service Discovery model (`discoverer:` and `services:` blocks, rule evaluation order, and the full template helper reference shared by all discoverers), see [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md).
24 + how_it_works: |
25 + Each discovery cycle, the discoverer:
26 +
27 + 1. **Iterates** every IP in the configured `networks[]` subnets, in parallel.
28 + 2. **Probes** each IP over UDP/161 using the credential bound to that subnet, walking the standard `system` MIB (`sysDescr`, `sysName`, `sysContact`, `sysLocation`, `sysObjectID`).
29 + 3. **Caches** the result in a status file so re-probing is skipped while `device_cache_ttl` has not expired.
30 + 4. **Emits a target** per reachable device, exposing `.IPAddress`, `.SysInfo.*`, and `.Credential.*` to the rule engine.
31 + 5. **Runs the `services:` rules** against each target. The rules render Go templates to produce one (or more) `snmp` collector job configurations.
32 +
33 + The discoverer never queries device-specific OIDs — those are queried later by the `snmp` collector once the job is created.
34 + limitations: |
35 + - Each subnet is capped at 512 IP addresses (a `/23` network or smaller). Split larger ranges into multiple `networks[]` entries.
36 + - Discovery uses **UDP/161**. The Netdata Agent host must be able to reach that port on every scanned IP, and any device-side ACLs must allow the Netdata host.
37 + - **One credential per subnet**: each `networks[]` entry is bound to exactly one credential. There is no automatic credential fallback. To probe the same subnet with multiple credentials, list it twice with different `credential` values; each device that responds to either credential will appear as a target (with its responding credential exposed via `.Credential.*`).
38 + - **Outbound interface**: probes use the host's default routing. There is no per-pipeline bind-address or VRF option — on multi-homed hosts, configure the OS routing table so the Netdata host reaches each target subnet via the correct interface.
39 + - The discoverer reads only the standard `system` MIB. Vendor-specific identification (`.SysInfo.Vendor`, `.Category`, `.Model`) is derived from `sysObjectID` and an enterprise-numbers table; values may be empty or `Unknown` for devices that are not in that table.
40 + - **SNMPv3 engine ID**: gosnmp negotiates the engine ID at the start of each probe (one extra round-trip per probe — usually irrelevant unless `parallel_scans_per_network` is high and the device is rate-limited). Engine IDs are not cached across probes, so devices that rotate engine IDs (rare; some HA pairs do this on failover) are handled transparently.
41 + - **Credential storage**: community strings and SNMPv3 passphrases are stored in plaintext both in `/etc/netdata/go.d/sd/snmp.conf` (file-based pipelines) and in the agent's dynamic-configuration store under `/var/lib/netdata/dyncfg/` (UI-managed pipelines). To avoid plaintext credentials on disk in either path, reference them via `${env:VAR}` or `${file:/path}` (see [Secrets Management](/src/collectors/SECRETS.md)).
42 +setup:
43 + prerequisites:
44 + list:
45 + - title: 'Plan your IP ranges and credentials'
46 + description: |
47 + Decide which subnets to scan and which SNMP credentials apply to each. SNMPv1 and SNMPv2c need a community string. SNMPv3 needs a USM username, security level, and (depending on the level) authentication and privacy passphrases.
48 + - title: 'Allow UDP/161 reachability'
49 + description: |
50 + The Netdata Agent host must be able to reach UDP port 161 on every scanned IP. SNMP devices typically restrict which clients can query them — make sure the Netdata host is allowed by any device-side ACLs.
51 + configuration:
52 + file:
53 + name: 'go.d/sd/snmp.conf'
54 + options:
55 + description: |
56 + The configuration file has two top-level blocks: `discoverer:` (the options below) and `services:` (rules that turn discovered devices into `snmp` collector jobs — see [Service Rules](#service-rules)).
57 +
58 + After editing the file, restart the Netdata Agent to load the updated discovery pipeline.
59 + folding:
60 + title: 'Discoverer options'
61 + enabled: false
62 + list:
63 + - name: 'rescan_interval'
64 + description: 'How often to rescan configured networks for devices.'
65 + default_value: '30m'
66 + required: false
67 + detailed_description: |
68 + Set to `0` to perform a single discovery scan when the agent starts and never rescan. Negative values also disable rescanning.
69 + - name: 'timeout'
70 + description: 'Maximum time to wait for an SNMP device response.'
71 + default_value: '1s'
72 + required: false
73 + - name: 'device_cache_ttl'
74 + description: 'How long to trust cached discovery results before re-probing a device.'
75 + default_value: '12h'
76 + required: false
77 + detailed_description: |
78 + Set to `0` to never expire cached results — once a device is discovered it is never re-probed (until the agent restarts and the cache is invalidated by configuration changes).
79 + - name: 'parallel_scans_per_network'
80 + description: 'How many IPs to probe concurrently within each subnet.'
81 + default_value: 32
82 + required: false
83 + - name: 'credentials'
84 + description: 'List of SNMP credentials referenced by entries in `networks`. At least one credential is required.'
85 + default_value: ''
86 + required: true
87 + detailed_description: |
88 + Each credential has a `name` (used by `networks[].credential`) and a `version`.
89 +
90 + **Accepted `version` values:** `1`, `2`, `2c`, `3`. (`2` is an alias for `2c`.)
91 +
92 + For SNMPv1 and SNMPv2c, set `community`.
93 +
94 + For SNMPv3, set:
95 +
96 + - `username` — USM user name.
97 + - `security_level` — one of `noAuthNoPriv`, `authNoPriv`, `authPriv`.
98 + - `auth_protocol` — one of `md5`, `sha` (HMAC-SHA-1, RFC 3414), `sha224`, `sha256`, `sha384`, `sha512` (HMAC-SHA-2, RFC 7860). Required for `authNoPriv` and `authPriv`.
99 + - `auth_password` — authentication passphrase. Required when `auth_protocol` is set.
100 + - `priv_protocol` — one of `des`, `aes` (AES-128), `aes192`, `aes256`, `aes192c`, `aes256c`. The `c` variants are the Cisco/Reeder draft; check your device's `show snmp user` output to pick the matching one. Required for `authPriv`.
101 + - `priv_password` — privacy passphrase. Required when `priv_protocol` is set.
102 + - `context_name` — only set this if your devices use a non-default SNMPv3 context.
103 +
104 + **Naming note:** the YAML keys are `auth_password` and `priv_password`. The same fields are exposed inside service rule templates as `.Credential.AuthPassphrase` and `.Credential.PrivacyPassphrase` (the Go struct names). Both refer to the same value.
105 +
106 + **Avoid plaintext on disk:** any of these fields can be sourced from environment variables or files using `${env:VAR_NAME}` or `${file:/absolute/path}` — see [Secrets Management](/src/collectors/SECRETS.md).
107 + - name: 'networks'
108 + description: 'List of subnets to scan, each tagged with the credential name to use. At least one network is required.'
109 + default_value: ''
110 + required: true
111 + detailed_description: |
112 + Each entry needs `subnet` (an IP range) and `credential` (the name of an entry from `credentials`).
113 +
114 + **Supported subnet formats** (IPv4 and IPv6):
115 +
116 + - CIDR — `192.168.1.0/24`, `2001:db8::/120`
117 + - Range — `10.0.0.1-10.0.0.50`, `2001:db8::-2001:db8::ff`
118 + - Subnet mask — `192.168.1.0/255.255.255.0`
119 + - Single IP — `192.168.1.10`, `2001:db8::1`
120 +
121 + Maximum **512 IPs** per subnet entry. Split larger blocks across multiple entries.
122 +
123 + For CIDR notation, network and broadcast addresses are excluded (except `/31`, `/32`, `/127`, `/128`).
124 + examples:
125 + folding:
126 + title: 'Configuration examples'
127 + enabled: true
128 + list:
129 + - name: 'Single subnet, SNMPv2c'
130 + description: 'Scan a single /24 with the default `public` community.'
131 + config: |
132 + disabled: no
133 + discoverer:
134 + snmp:
135 + credentials:
136 + - name: public-v2c
137 + version: 2c
138 + community: public
139 + networks:
140 + - subnet: 192.168.1.0/24
141 + credential: public-v2c
142 + services:
143 + - id: snmp
144 + match: '{{ true }}'
145 + - name: 'Multiple subnets, mixed SNMPv2c and SNMPv3'
146 + description: 'Mix SNMPv2c on one subnet with SNMPv3 (authPriv) on another. Credentials are referenced from environment variables to keep them out of plaintext on disk.'
147 + config: |
148 + disabled: no
149 + discoverer:
150 + snmp:
151 + rescan_interval: 1h
152 + credentials:
153 + - name: public-v2c
154 + version: 2c
155 + community: ${env:SNMP_V2C_COMMUNITY}
156 + - name: secure-v3
157 + version: 3
158 + security_level: authPriv
159 + username: netdata-monitor
160 + auth_protocol: sha256
161 + auth_password: ${env:SNMP_V3_AUTH}
162 + priv_protocol: aes256
163 + priv_password: ${env:SNMP_V3_PRIV}
164 + networks:
165 + - subnet: 192.168.10.0/24
166 + credential: public-v2c
167 + - subnet: 10.20.30.0/24
168 + credential: secure-v3
169 + services:
170 + - id: snmp
171 + match: '{{ true }}'
172 + - name: 'IPv6 subnet'
173 + description: 'Scan a small IPv6 range with SNMPv2c.'
174 + config: |
175 + disabled: no
176 + discoverer:
177 + snmp:
178 + credentials:
179 + - name: public-v2c
180 + version: 2c
181 + community: public
182 + networks:
183 + - subnet: 2001:db8:0:1::/120
184 + credential: public-v2c
185 + services:
186 + - id: snmp
187 + match: '{{ true }}'
188 +services:
189 + description: |
190 + A `services:` rule turns each discovered SNMP device into one or more `snmp` collector jobs. Each rule has an `id`, a Go-template `match` expression that decides whether the rule applies to the device, and an optional `config_template` that renders the collector job YAML when the rule matches.
191 +
192 + The default rule shipped with Netdata (`{{ true }}`) creates one job per discovered device and handles both SNMPv2 and SNMPv3 — most users never change it. Customise rules when you want vendor-specific configs (Cisco vs. Juniper, printers vs. routers), per-VLAN overrides, or multiple jobs per device.
193 +
194 + The shared rule model — function reference (sprig + Netdata helpers `match`, `glob`, `promPort`, `toYaml`), `config_template` rendering rules, and the strict-missing-key error semantics — lives on the [Service Discovery](/src/collectors/SERVICE-DISCOVERY.md) hub page. The notes below are SNMP-specific.
195 + evaluation:
196 + description: |
197 + Quick reference — see [Rule evaluation semantics](/src/collectors/SERVICE-DISCOVERY.md#rule-evaluation-semantics) on the hub page for the full model (sequence-output multi-job rendering, module inference from `id`, `missingkey=error`, ordering recommendations).
198 + list:
199 + - name: 'Skip rule (no config_template)'
200 + description: 'A matching rule with no `config_template` drops the device immediately — no job, no further rule evaluation. Use it to exclude devices the catch-all would otherwise pick up. Place **before** any template rule.'
201 + - name: 'Template rule (with config_template)'
202 + description: 'A matching rule with a `config_template` produces one or more jobs and rule evaluation **continues**. A single device can therefore produce jobs from several matching rules.'
203 + - name: 'For SNMP specifically'
204 + description: 'Set `id: snmp` so the rendered job inherits the `snmp` module name automatically, or include `module: snmp` explicitly inside the `config_template` (required when `id` is anything else, e.g. `cisco`).'
205 + template_variables:
206 + description: |
207 + Available inside both `match` expressions and `config_template` bodies. All variables are strings; empty values render as the empty string.
208 + list:
209 + - name: '.IPAddress'
210 + type: 'string'
211 + description: 'IP address of the discovered device. Always set.'
212 + - name: '.SysInfo.Descr'
213 + type: 'string'
214 + description: 'Value of `sysDescr.0` (vendor-supplied free-form description). May be empty.'
215 + - name: '.SysInfo.Contact'
216 + type: 'string'
217 + description: 'Value of `sysContact.0`. May be empty.'
218 + - name: '.SysInfo.Name'
219 + type: 'string'
220 + description: 'Value of `sysName.0` (typically the device hostname or FQDN). Defaults to the literal string `unknown` when the device does not return one.'
221 + - name: '.SysInfo.Location'
222 + type: 'string'
223 + description: 'Value of `sysLocation.0`. May be empty.'
224 + - name: '.SysInfo.Organization'
225 + type: 'string'
226 + description: 'Vendor or organization parsed from `sysObjectID` against the embedded enterprise-numbers table. Defaults to `Unknown` when the OID is not in the table.'
227 + - name: '.SysInfo.Vendor'
228 + type: 'string'
229 + description: 'Vendor name inferred from `sysObjectID` and `sysDescr` via the bundled overrides. Empty when no override matches.'
230 + - name: '.SysInfo.Category'
231 + type: 'string'
232 + description: 'Device category (e.g. `router`, `switch`, `printer`). Sourced from the bundled SNMP overrides; empty when no override matches the device. The set of category values is determined by the overrides, not a closed enum.'
233 + - name: '.SysInfo.Model'
234 + type: 'string'
235 + description: 'Device model inferred from `sysObjectID` and `sysDescr` via the bundled overrides. Empty when no override matches.'
236 + - name: '.Credential.Name'
237 + type: 'string'
238 + description: 'Name of the credential entry that successfully probed the device.'
239 + - name: '.Credential.Version'
240 + type: 'string'
241 + description: 'Configured version string: `1`, `2`, `2c`, or `3`. Use `eq .Credential.Version "1" "2" "2c"` to branch v1/v2c vs v3.'
242 + - name: '.Credential.Community'
243 + type: 'string'
244 + description: 'Community string (SNMPv1/v2c). Empty for SNMPv3.'
245 + - name: '.Credential.UserName'
246 + type: 'string'
247 + description: 'SNMPv3 USM user name. Empty for v1/v2c.'
248 + - name: '.Credential.SecurityLevel'
249 + type: 'string'
250 + description: 'SNMPv3 security level (`noAuthNoPriv`, `authNoPriv`, `authPriv`).'
251 + - name: '.Credential.AuthProtocol'
252 + type: 'string'
253 + description: 'SNMPv3 auth protocol (`md5`, `sha`, `sha224`–`sha512`).'
254 + - name: '.Credential.AuthPassphrase'
255 + type: 'string'
256 + description: 'SNMPv3 authentication passphrase. **YAML key for the same value: `auth_password`** (see [credentials option](#option-credentials)).'
257 + - name: '.Credential.PrivacyProtocol'
258 + type: 'string'
259 + description: 'SNMPv3 privacy protocol (`des`, `aes`, `aes192`, `aes256`, `aes192c`, `aes256c`).'
260 + - name: '.Credential.PrivacyPassphrase'
261 + type: 'string'
262 + description: 'SNMPv3 privacy passphrase. **YAML key for the same value: `priv_password`** (see [credentials option](#option-credentials)).'
263 + examples:
264 + description: 'Each example shows one or more entries from the `services:` array. Order matters — see [How rules are evaluated](#how-rules-are-evaluated).'
265 + list:
266 + - name: 'Default catch-all rule'
267 + description: |
268 + Generate one `snmp` collector job per discovered device. This is the rule produced by the stock conf and is sufficient for most deployments — it handles both SNMPv2 and SNMPv3 by branching on `.Credential.Version`. The `id: snmp` makes the module name infer to `snmp` automatically.
269 + config: |
270 + - id: snmp
271 + match: '{{ true }}'
272 + config_template: |
273 + {{- if .SysInfo.Name }}
274 + name: {{ .SysInfo.Name }}-ip-{{ .IPAddress }}
275 + {{- else }}
276 + name: ip-{{ .IPAddress }}
277 + {{- end }}
278 + hostname: {{ .IPAddress }}
279 + options:
280 + version: {{ .Credential.Version }}
281 + {{- if eq .Credential.Version "1" "2" "2c" }}
282 + community: {{ .Credential.Community }}
283 + {{- else }}
284 + user:
285 + name: {{ .Credential.UserName }}
286 + level: {{ .Credential.SecurityLevel }}
287 + auth_proto: {{ .Credential.AuthProtocol }}
288 + auth_key: {{ .Credential.AuthPassphrase }}
289 + priv_proto: {{ .Credential.PrivacyProtocol }}
290 + priv_key: {{ .Credential.PrivacyPassphrase }}
291 + {{- end }}
292 + - name: 'Skip rule for management VIPs'
293 + description: |
294 + Drop devices whose `sysName` starts with `vip-` so they are not monitored. A skip rule is a rule with **no** `config_template`. Place it before the catch-all so its match wins first. (The second rule below is the [Default catch-all rule](#default-catch-all-rule) — paste its full body in place of the placeholder comment.)
295 + config: |
296 + - id: skip-vips
297 + match: '{{ glob .SysInfo.Name "vip-*" }}'
298 + - id: snmp
299 + match: '{{ true }}'
300 + config_template: |
301 + # paste the body from the "Default catch-all rule" example here
302 + - name: 'Vendor-specific override (Cisco) with catch-all suppressed'
303 + description: |
304 + Apply a Cisco-specific config to devices whose vendor matches `Cisco*`, then prevent the catch-all from producing a duplicate job for the same devices. This is the recommended three-rule pattern for any vendor-specific override:
305 +
306 + 1. **Specific template rule** (`cisco`) — renders the Cisco-tuned job. `id: cisco` does **not** map to a real collector module, so `module: snmp` is set explicitly in the template.
307 + 2. **Skip rule** (`skip-cisco-from-catchall`) — drops Cisco devices from the remaining pipeline so step 3 does not also fire for them.
308 + 3. **Catch-all template rule** (`snmp`) — handles every non-Cisco device.
309 + config: |
310 + - id: cisco
311 + match: '{{ glob .SysInfo.Vendor "Cisco*" }}'
312 + config_template: |
313 + module: snmp
314 + name: cisco-{{ .SysInfo.Name }}-{{ .IPAddress }}
315 + hostname: {{ .IPAddress }}
316 + options:
317 + version: {{ .Credential.Version }}
318 + {{- if eq .Credential.Version "1" "2" "2c" }}
319 + community: {{ .Credential.Community }}
320 + {{- else }}
321 + user:
322 + name: {{ .Credential.UserName }}
323 + level: {{ .Credential.SecurityLevel }}
324 + auth_proto: {{ .Credential.AuthProtocol }}
325 + auth_key: {{ .Credential.AuthPassphrase }}
326 + priv_proto: {{ .Credential.PrivacyProtocol }}
327 + priv_key: {{ .Credential.PrivacyPassphrase }}
328 + {{- end }}
329 +
330 + - id: skip-cisco-from-catchall
331 + match: '{{ glob .SysInfo.Vendor "Cisco*" }}'
332 +
333 + - id: snmp
334 + match: '{{ true }}'
335 + config_template: |
336 + # ... (same as the catch-all in the Default catch-all rule example above)
337 + - name: 'Category-based rule (HP printers)'
338 + description: |
339 + Match by `.SysInfo.Category` to apply a printer-specific config. Category values are populated from the bundled SNMP overrides — `printer` is one of the standard categories produced by the override file shipped with Netdata. Pair with a follow-up skip rule the same way as the Cisco example if you want to suppress the catch-all for printers.
340 + config: |
341 + - id: hp-printer
342 + match: '{{ and (eq .SysInfo.Category "printer") (glob .SysInfo.Vendor "HP*" "Hewlett*") }}'
343 + config_template: |
344 + module: snmp
345 + name: printer-{{ .SysInfo.Name }}-{{ .IPAddress }}
346 + hostname: {{ .IPAddress }}
347 + update_every: 30
348 + options:
349 + version: {{ .Credential.Version }}
350 + community: {{ .Credential.Community }}
351 +verify:
352 + description: 'After enabling the discoverer, confirm it is finding devices and producing jobs.'
353 + checks:
354 + list:
355 + - name: 'Confirm devices are being probed'
356 + description: |
357 + Watch the agent log for SNMP discoverer messages. On a successful probe you should see lines like:
358 +
359 + ```text
360 + discoverer=snmp ... device '192.168.1.10': successfully discovered (sysName: 'sw01.example.com', network: '192.168.1.0/24')
361 + ```
362 +
363 + With systemd:
364 +
365 + ```bash
366 + journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep "discoverer=snmp"
367 + ```
368 +
369 + Without systemd:
370 +
371 + ```bash
372 + grep "discoverer=snmp" /var/log/netdata/collector.log
373 + ```
374 + - name: 'Confirm jobs are being created'
375 + description: |
376 + Discovered devices should produce `snmp` collector jobs. In the Netdata UI go to `Collectors -> go.d -> snmp` — each discovered device appears as a job named according to your `config_template` (the default catch-all renders `<sysName>-ip-<address>`).
377 + - name: 'Confirm metrics are being collected'
378 + description: |
379 + Once a job exists, the `snmp` collector takes over and starts collecting metrics. Charts for each device appear under the SNMP integration on the dashboard. If a job is created but metrics never appear, the problem is in the `snmp` collector configuration (rendered by your `config_template`), not in the discoverer.
380 +troubleshooting:
381 + problems:
382 + list:
383 + - name: 'No devices are discovered'
384 + description: |
385 + Check the agent log for `discoverer=snmp` messages. Common causes:
386 +
387 + - The configured subnets do not match where your devices live. Verify with `ping` / `arp` from the Netdata host.
388 + - UDP port 161 is blocked between the Netdata host and the devices. Test with `nc -zu <ip> 161` or `snmpwalk -v2c -c <community> <ip> sysDescr.0`.
389 + - The credentials do not match what the devices accept. SNMPv3 mismatches commonly produce `authentication failure` or `decryption error` log lines.
390 + - A configured subnet exceeds the 512-IP cap and the discoverer rejected it at startup. Look for `subnet '...' exceeds maximum size of /23` in the log.
391 + - name: 'Wrong devices are matched by a rule'
392 + description: |
393 + Rule order matters — see [How rules are evaluated](#how-rules-are-evaluated). Place vendor-specific or device-specific rules **before** the catch-all. If you need to suppress the catch-all for a subset of devices, follow the specific rule with a skip rule (no `config_template`) keyed on the same condition.
394 + - name: 'Generated collector jobs fail to start'
395 + description: |
396 + The discoverer creates jobs but does not run them — the `snmp` collector does. Check the `snmp` collector log and the rendered job YAML in the agent's debug output. Common causes:
397 +
398 + - The rendered `config_template` produces invalid YAML for some discovered field values (for example, unescaped colons in `sysName`). YAML-quote dynamic values when in doubt.
399 + - Module name mismatch — the rule `id` (or explicit `module:` field) does not match `snmp`.
400 + - SNMPv3 credentials succeeded for `system` MIB during discovery but the collector cannot read other OIDs (different VACM view); confirm with `snmpwalk` against the device.