| 1 | """ |
| 2 | Generate src/collectors/SERVICE-DISCOVERY.md from integrations/integrations.js. |
| 3 | |
| 4 | Mirrors gen_doc_secrets_page.py: |
| 5 | - reads discovered service-discovery integrations from integrations.js; |
| 6 | - renders the umbrella Service Discovery page; |
| 7 | - carries the canonical SD model, helper-function reference, and rule |
| 8 | semantics that per-discoverer pages link back to. |
| 9 | """ |
| 10 | |
| 11 | from __future__ import annotations |
| 12 | |
| 13 | import json |
| 14 | import pathlib |
| 15 | import re |
| 16 | from typing import Any, Dict, List |
| 17 | |
| 18 | GITHUB_BLOB_PREFIX = "https://github.com/netdata/netdata/blob/master" |
| 19 | TEMPLATE_PATH = pathlib.Path(__file__).resolve().parent / "templates" |
| 20 | |
| 21 | SD_PAGE = { |
| 22 | "title": "# Service Discovery", |
| 23 | "intro": [ |
| 24 | "Stop hand-writing collector jobs for every host, container, or device. Netdata's Service Discovery (SD) finds monitorable targets in your environment and turns them into collector jobs automatically.", |
| 25 | "Each SD pipeline is a `discoverer:` (where to look) plus a list of `services:` rules (how to turn what was found into collector jobs). The discoverer emits **targets**; the rules render **collector job YAML** from those targets using Go templates.", |
| 26 | ], |
| 27 | "jump_to": [ |
| 28 | {"label": "How it works", "anchor": "how-it-works"}, |
| 29 | {"label": "Configuration file structure", "anchor": "configuration-file-structure"}, |
| 30 | {"label": "Rule evaluation semantics", "anchor": "rule-evaluation-semantics"}, |
| 31 | {"label": "Template helper reference", "anchor": "template-helper-reference"}, |
| 32 | {"label": "config_template rendering", "anchor": "config_template-rendering"}, |
| 33 | {"label": "Supported discoverers", "anchor": "supported-discoverers"}, |
| 34 | {"label": "Mixing discoverers", "anchor": "mixing-discoverers"}, |
| 35 | {"label": "Troubleshooting", "anchor": "troubleshooting"}, |
| 36 | ], |
| 37 | "how_it_works": { |
| 38 | "heading": "## How it works", |
| 39 | "intro": [ |
| 40 | "Each discovery pipeline runs in five stages:", |
| 41 | ], |
| 42 | "stages": [ |
| 43 | { |
| 44 | "name": "Discover", |
| 45 | "description": "The discoverer probes its environment for monitorable things — running containers, listening sockets, kubernetes resources, SNMP devices on a subnet, items returned by an HTTP endpoint. What gets probed and how often is controlled by `discoverer:` options.", |
| 46 | }, |
| 47 | { |
| 48 | "name": "Build a target", |
| 49 | "description": "Each discovered thing becomes a target. The target carries discoverer-specific fields (variables) — for example, net_listeners targets have `.Port` / `.Comm` / `.Cmdline`; docker targets have `.Image` / `.Name` / `.Labels`; snmp targets have `.IPAddress` / `.SysInfo.*` / `.Credential.*`. Per-discoverer pages list the full variable set.", |
| 50 | }, |
| 51 | { |
| 52 | "name": "Match against services rules", |
| 53 | "description": "The rule engine evaluates each `services:` rule against each target, top to bottom. A rule's `match` expression is a Go template that must render to the literal string `\"true\"` for the rule to apply.", |
| 54 | }, |
| 55 | { |
| 56 | "name": "Render the collector job", |
| 57 | "description": "When a rule matches, its `config_template` is executed with the target as context, producing collector job YAML. The rendered YAML can be a single job map or a sequence of jobs.", |
| 58 | }, |
| 59 | { |
| 60 | "name": "Hand off to the collector", |
| 61 | "description": "The agent registers the rendered jobs with the matching collector module. From this point on, the collector runs the job — the discoverer is no longer involved.", |
| 62 | }, |
| 63 | ], |
| 64 | }, |
| 65 | "config_file": { |
| 66 | "heading": "## Configuration file structure", |
| 67 | "intro": [ |
| 68 | "Each discoverer has its own file under `/etc/netdata/go.d/sd/`. The filename determines the discoverer kind (`net_listeners.conf`, `docker.conf`, `http.conf`, `snmp.conf`, `k8s.conf`).", |
| 69 | "Every SD file has the same shape:", |
| 70 | ], |
| 71 | "skeleton": """```yaml |
| 72 | disabled: no # set to yes to disable this pipeline |
| 73 | |
| 74 | discoverer: |
| 75 | <kind>: # discoverer kind (must match filename) |
| 76 | # discoverer-specific options — see the per-discoverer page |
| 77 | |
| 78 | services: |
| 79 | - id: <rule-id> |
| 80 | match: <go-template> # must render to the literal string "true" |
| 81 | config_template: | # optional — omit to make this a "skip rule" |
| 82 | # collector job YAML, rendered with target as context |
| 83 | ```""", |
| 84 | "notes": [ |
| 85 | "`disabled: yes` keeps the file on disk but turns the pipeline off.", |
| 86 | "Editing a stock file requires restarting the agent. UI-managed pipelines apply live.", |
| 87 | "Where each discoverer's stock conf ships (with the Netdata package, with the Helm chart, or not at all) is documented on its per-discoverer page.", |
| 88 | ], |
| 89 | }, |
| 90 | "rule_eval": { |
| 91 | "heading": "## Rule evaluation semantics", |
| 92 | "intro": [ |
| 93 | "For each discovered target, the engine walks the `services:` array top-to-bottom. The two rule shapes behave differently:", |
| 94 | ], |
| 95 | "shapes": [ |
| 96 | { |
| 97 | "name": "Skip rule (no `config_template`)", |
| 98 | "description": "When a skip rule matches, the target is dropped immediately — no jobs are produced and **no further rules run for that target**. Use skip rules to exclude targets the catch-all would otherwise pick up. Place them **before** any template rule.", |
| 99 | }, |
| 100 | { |
| 101 | "name": "Template rule (with `config_template`)", |
| 102 | "description": "When a template rule matches, its `config_template` is rendered into one or more collector jobs and rule evaluation **continues** with the next rule. A single target can therefore produce jobs from multiple matching template rules.", |
| 103 | }, |
| 104 | ], |
| 105 | "ordering": [ |
| 106 | "Recommended ordering for a multi-rule pipeline:", |
| 107 | "", |
| 108 | "1. Skip rules — drop targets you don't want monitored.", |
| 109 | "2. Specific template rules — vendor-specific, label-specific, port-specific.", |
| 110 | "3. A catch-all template rule (`match: '{{ true }}'`) — the default fallback.", |
| 111 | "", |
| 112 | "If a specific template rule already produced the right job, follow it with a skip rule keyed on the same condition to suppress the catch-all for those targets.", |
| 113 | ], |
| 114 | }, |
| 115 | "helpers": { |
| 116 | "heading": "## Template helper reference", |
| 117 | "intro": [ |
| 118 | "Match expressions and config templates are [Go `text/template`](https://pkg.go.dev/text/template) bodies with three additional helper sets: the standard Go template builtins, the [sprig](https://masterminds.github.io/sprig/) function library, and a small set of Netdata-specific helpers.", |
| 119 | "All templates run with `missingkey=error`. Referencing a field that does not exist on the target type (e.g. typo `.SysInfoo.Name` instead of `.SysInfo.Name`) makes the template execution fail; the agent logs the error and skips that rule for that target.", |
| 120 | ], |
| 121 | "go_builtins": { |
| 122 | "heading": "### Go template builtins", |
| 123 | "intro": "Standard Go `text/template` syntax: pipelines, conditionals, loops, variable assignment, whitespace control.", |
| 124 | "table": [ |
| 125 | {"name": "`if`/`else if`/`else`/`end`", "description": "Conditional rendering."}, |
| 126 | {"name": "`range`/`end`", "description": "Iterate over a slice or map."}, |
| 127 | {"name": "`with`/`end`", "description": "Set the dot to a value if it is non-empty."}, |
| 128 | {"name": "`{{- ... -}}`", "description": "Whitespace-trim left/right around the action."}, |
| 129 | {"name": "`{{ $var := ... }}`", "description": "Variable assignment, scoped to the enclosing block."}, |
| 130 | {"name": "`eq`, `ne`, `lt`, `le`, `gt`, `ge`", "description": "Comparison. `eq A B C ...` is true if `A` equals **any** of the following arguments."}, |
| 131 | {"name": "`and`, `or`, `not`", "description": "Boolean composition (variadic)."}, |
| 132 | {"name": "`index`", "description": "Index a slice or map: `index .Labels \"app\"`."}, |
| 133 | {"name": "`printf`", "description": "Formatted string."}, |
| 134 | ], |
| 135 | }, |
| 136 | "sprig": { |
| 137 | "heading": "### Sprig functions", |
| 138 | "intro": "The full [sprig library](https://masterminds.github.io/sprig/) is included. The functions most commonly used in stock SD configs:", |
| 139 | "table": [ |
| 140 | {"name": "`default DEFAULT VALUE`", "description": "Return `VALUE` if non-empty; otherwise `DEFAULT`."}, |
| 141 | {"name": "`empty VALUE`", "description": "True if the value is the zero value for its type."}, |
| 142 | {"name": "`hasKey MAP KEY`", "description": "True if `MAP` contains `KEY`. Used heavily by the `http` discoverer."}, |
| 143 | {"name": "`kindIs KIND VALUE`", "description": "True if `VALUE`'s reflect kind matches: `string`, `map`, `slice`, `bool`, …"}, |
| 144 | {"name": "`lower S` / `upper S`", "description": "Lowercase / uppercase a string."}, |
| 145 | {"name": "`trim S` / `trimPrefix PREFIX S` / `trimSuffix SUFFIX S`", "description": "Whitespace and prefix/suffix trimming."}, |
| 146 | {"name": "`replace OLD NEW S`", "description": "String replace."}, |
| 147 | {"name": "`regexFind RE S`", "description": "Return the first regex match, or empty."}, |
| 148 | {"name": "`regexMatch RE S`", "description": "True if `S` matches `RE` (substring match unless anchored)."}, |
| 149 | {"name": "`printf FMT V...`", "description": "Same as Go's `fmt.Sprintf`."}, |
| 150 | ], |
| 151 | "outro": "See the [sprig docs](https://masterminds.github.io/sprig/) for the full list (string, math, encoding, list, dict, date helpers).", |
| 152 | }, |
| 153 | "netdata": { |
| 154 | "heading": "### Netdata helpers", |
| 155 | "intro": "Custom helpers added to the SD template engine:", |
| 156 | "table": [ |
| 157 | { |
| 158 | "name": "`match TYPE VALUE PATTERN [PATTERN...]`", |
| 159 | "description": "Returns the string `\"true\"` if `VALUE` matches **any** of the patterns under the named matcher type. `TYPE` is one of:", |
| 160 | "sub": [ |
| 161 | "`\"glob\"` — shell glob (`*`, `?`, `[abc]`).", |
| 162 | "`\"sp\"` — Netdata simple patterns (space-separated, `*` wildcard, `!` for negation).", |
| 163 | "`\"re\"` — RE2 regular expression. Matches if the regex matches anywhere in `VALUE` unless explicitly anchored with `^` / `$`.", |
| 164 | "`\"dstar\"` — [doublestar](https://github.com/bmatcuk/doublestar) glob (supports `**` for path-style matching).", |
| 165 | ], |
| 166 | }, |
| 167 | { |
| 168 | "name": "`glob VALUE PATTERN [PATTERN...]`", |
| 169 | "description": "Shortcut for `match \"glob\" VALUE PATTERN...`.", |
| 170 | }, |
| 171 | { |
| 172 | "name": "`promPort PORT`", |
| 173 | "description": "Returns the well-known Prometheus exporter module name registered for `PORT`, or the empty string. `net_listeners`-specific.", |
| 174 | }, |
| 175 | { |
| 176 | "name": "`toYaml VALUE`", |
| 177 | "description": "Serialize `VALUE` as a YAML string. Used by `http` discoverer rules that pass through items as collector job configs.", |
| 178 | }, |
| 179 | ], |
| 180 | "notes": [ |
| 181 | "`match` and `glob` are case-sensitive.", |
| 182 | "`match \"re\" ...` is unanchored. Use `^...$` to require a full-string match.", |
| 183 | "There is **no** standalone `regexp` or `regex` helper. Use `match \"re\"` (or sprig's `regexFind` / `regexMatch`) for regex matching.", |
| 184 | "`match` and `glob` return the **string** `\"true\"` or `\"false\"`, not a Go bool. The rule engine compares the trimmed template output against the literal `\"true\"`, so a top-level `match: '{{ glob .X \"foo*\" }}'` works directly.", |
| 185 | "**Composing with `if` / `and` / `or` is a footgun**: in Go templates a non-empty string is truthy, so both `\"true\"` *and* `\"false\"` evaluate truthy under `if`. `{{ if glob .X \"foo*\" }}` is therefore **wrong**. Either wrap each result with `eq ... \"true\"` to get a real bool, or use an explicit `if-then-true` block:\n ```\n match: '{{ if and (eq (glob .Vendor \"Cisco*\") \"true\") (eq .Category \"router\") }}true{{ end }}'\n ```", |
| 186 | ], |
| 187 | }, |
| 188 | }, |
| 189 | "config_template": { |
| 190 | "heading": "## config_template rendering", |
| 191 | "intro": [ |
| 192 | "When a template rule matches, its `config_template` is executed with the target as the dot context. The rendered output is parsed as YAML to produce one or more collector jobs.", |
| 193 | ], |
| 194 | "rules": [ |
| 195 | { |
| 196 | "name": "Single map → one job", |
| 197 | "description": "If the rendered YAML is a map, a single collector job is created.", |
| 198 | }, |
| 199 | { |
| 200 | "name": "Sequence of maps → one job per element", |
| 201 | "description": "If the rendered YAML is a sequence (top-level YAML `-` items), one job is created per element. Use this for multi-job rules. Example: a `net_listeners` rule that emits both a TCP and a Unix-socket MySQL job from the same target —\n\n ```yaml\n - id: mysql\n match: '{{ or (eq .Port \"3306\") (eq .Comm \"mysqld\") }}'\n config_template: |\n - name: local\n dsn: netdata@unix(/var/run/mysqld/mysqld.sock)/\n - name: local\n dsn: netdata@tcp({{.Address}})/\n ```", |
| 202 | }, |
| 203 | { |
| 204 | "name": "Module inference from rule `id`", |
| 205 | "description": "If the rendered job map has no `module:` key, the rule's `id` is used as the module name. Set `id: snmp` (or `docker`, `http`, …) to omit `module:` from your template; otherwise include `module:` explicitly.", |
| 206 | }, |
| 207 | { |
| 208 | "name": "`id` uniqueness", |
| 209 | "description": "Rule IDs are not required to be unique across rules — multiple rules can share an `id`. The `id` is used for module inference and shows up in agent logs to help you identify which rule produced a job. Pick descriptive IDs (`cisco`, `hp-printers`, `skip-vips`).", |
| 210 | }, |
| 211 | { |
| 212 | "name": "Failure handling", |
| 213 | "description": "Render errors, YAML parse errors, and `missingkey=error` failures are logged at warn level. The agent skips the rule for that target and continues evaluation.", |
| 214 | }, |
| 215 | ], |
| 216 | }, |
| 217 | "supported": { |
| 218 | "heading": "## Supported discoverers", |
| 219 | "intro": "Each discoverer has its own page covering its options, target variables, evaluation specifics, and worked examples.", |
| 220 | }, |
| 221 | "mixing": { |
| 222 | "heading": "## Mixing discoverers", |
| 223 | "body": [ |
| 224 | "All discoverers can run simultaneously. Each `/etc/netdata/go.d/sd/<kind>.conf` is independent. The same target can theoretically be discovered by more than one discoverer (for example, a containerised application appears in both `docker` and `net_listeners`); each discoverer's pipeline is independent and may produce its own job.", |
| 225 | "Use `disabled: yes` at the top of a stock file to keep it on disk but turn the pipeline off.", |
| 226 | "UI-managed pipelines and file-based pipelines coexist. UI-managed pipelines apply live; file-based pipelines require an agent restart to reload.", |
| 227 | ], |
| 228 | }, |
| 229 | "troubleshooting": { |
| 230 | "heading": "## Troubleshooting", |
| 231 | "intro": [ |
| 232 | "Common cross-discoverer problems. For discoverer-specific issues, see the per-discoverer page.", |
| 233 | ], |
| 234 | "problems": [ |
| 235 | { |
| 236 | "name": "No targets discovered", |
| 237 | "description": "Check the agent log for `discoverer=<kind>` lines. Confirm `disabled: no` and that the discoverer's prerequisites are met (network reachability, credentials, API access).", |
| 238 | }, |
| 239 | { |
| 240 | "name": "Targets discovered but no jobs created", |
| 241 | "description": "Check that at least one `services:` rule has both a matching `match` expression and a `config_template`. A rule with no `config_template` is a skip rule — it drops the target instead of producing a job.", |
| 242 | }, |
| 243 | { |
| 244 | "name": "`match` always evaluates false", |
| 245 | "description": "Match expressions must render to the literal string `\"true\"`. A bare `{{ if ... }}` block that does not output anything renders to the empty string, which is treated as false. Use `{{ true }}` for catch-all, `{{ glob .X \"...\" }}` for pattern matches, or `{{ if ... }}true{{ end }}` for ad-hoc conditions.", |
| 246 | }, |
| 247 | { |
| 248 | "name": "Template render error", |
| 249 | "description": "Look for `failed to execute services[N]->config_template on target` in the log. The most common cause is a typo in a variable reference (e.g. `.SysInfoo.Name`) — `missingkey=error` rejects unknown fields. Use the per-discoverer page's variable table to verify spelling.", |
| 250 | }, |
| 251 | { |
| 252 | "name": "YAML parse error after rendering", |
| 253 | "description": "`failed to parse services[N] template data` means the rendered output is not valid YAML. Common cause: a discovered string field contains a colon, hash, or other YAML special character. YAML-quote dynamic values (`name: \"{{ .X }}\"`) when they may be irregular.", |
| 254 | }, |
| 255 | ], |
| 256 | }, |
| 257 | } |
| 258 | |
| 259 | |
| 260 | def _extract_integrations_json(js_text: str) -> str: |
| 261 | after_categories = js_text.split("export const categories = ", 1)[1] |
| 262 | _, after_integrations = after_categories.split("export const integrations = ", 1) |
| 263 | return re.split(r"\n\s*export const|\Z", after_integrations, maxsplit=1)[0].strip().rstrip(';').strip() |
| 264 | |
| 265 | |
| 266 | def load_integrations(js_path: str = "integrations/integrations.js") -> Any: |
| 267 | with open(js_path, "r", encoding="utf-8") as f: |
| 268 | js_data = f.read() |
| 269 | return json.loads(_extract_integrations_json(js_data)) |
| 270 | |
| 271 | |
| 272 | def iterate_integrations(integrations: Any): |
| 273 | if isinstance(integrations, dict): |
| 274 | for integ in integrations.values(): |
| 275 | if isinstance(integ, dict): |
| 276 | yield integ |
| 277 | elif isinstance(integrations, list): |
| 278 | for integ in integrations: |
| 279 | if isinstance(integ, dict): |
| 280 | yield integ |
| 281 | |
| 282 | |
| 283 | def collect_sd_integrations(integrations: Any) -> List[Dict[str, Any]]: |
| 284 | items = [] |
| 285 | for integ in iterate_integrations(integrations): |
| 286 | if integ.get("integration_type") != "service_discovery": |
| 287 | continue |
| 288 | meta = integ.get("meta", {}) |
| 289 | if not isinstance(meta, dict): |
| 290 | continue |
| 291 | if not isinstance(meta.get("name"), str) or not isinstance(meta.get("kind"), str): |
| 292 | continue |
| 293 | items.append(integ) |
| 294 | items.sort(key=lambda item: item["meta"]["name"].lower()) |
| 295 | return items |
| 296 | |
| 297 | |
| 298 | def get_repo_path_from_blob_url(url: str) -> str: |
| 299 | if url.startswith(GITHUB_BLOB_PREFIX): |
| 300 | return url[len(GITHUB_BLOB_PREFIX):] |
| 301 | return url |
| 302 | |
| 303 | |
| 304 | def get_sd_readme_link(integ: Dict[str, Any]) -> str: |
| 305 | edit_link = integ.get("edit_link", "") if isinstance(integ, dict) else "" |
| 306 | repo_path = get_repo_path_from_blob_url(edit_link) |
| 307 | if repo_path.endswith("/metadata.yaml"): |
| 308 | return repo_path[: -len("metadata.yaml")] + "README.md" |
| 309 | return "" |
| 310 | |
| 311 | |
| 312 | _jinja_env = None |
| 313 | |
| 314 | |
| 315 | def get_jinja_env(): |
| 316 | global _jinja_env |
| 317 | if _jinja_env is None: |
| 318 | from jinja2 import Environment, FileSystemLoader, select_autoescape |
| 319 | _jinja_env = Environment( |
| 320 | loader=FileSystemLoader(TEMPLATE_PATH), |
| 321 | autoescape=select_autoescape(), |
| 322 | block_start_string='[%', |
| 323 | block_end_string='%]', |
| 324 | variable_start_string='[[', |
| 325 | variable_end_string=']]', |
| 326 | comment_start_string='[#', |
| 327 | comment_end_string='#]', |
| 328 | trim_blocks=True, |
| 329 | lstrip_blocks=True, |
| 330 | ) |
| 331 | return _jinja_env |
| 332 | |
| 333 | |
| 334 | def build_discoverers_context(integrations: Any) -> List[Dict[str, str]]: |
| 335 | items = [] |
| 336 | for integ in collect_sd_integrations(integrations): |
| 337 | meta = integ.get("meta", {}) |
| 338 | kind = meta.get("kind", "") |
| 339 | name = meta.get("name", "") |
| 340 | tagline = meta.get("tagline", "") |
| 341 | readme_link = get_sd_readme_link(integ) |
| 342 | items.append({ |
| 343 | "name": name, |
| 344 | "kind": kind, |
| 345 | "config_file": f"/etc/netdata/go.d/sd/{kind}.conf", |
| 346 | "name_link": f'[{name}]({readme_link})' if readme_link else name, |
| 347 | "tagline": tagline, |
| 348 | }) |
| 349 | return items |
| 350 | |
| 351 | |
| 352 | def build_page_context() -> Dict[str, Any]: |
| 353 | return { |
| 354 | "title": SD_PAGE["title"], |
| 355 | "intro": SD_PAGE["intro"], |
| 356 | "jump_to_line": " • ".join( |
| 357 | f'[{jump["label"]}](#{jump["anchor"]})' for jump in SD_PAGE["jump_to"] |
| 358 | ), |
| 359 | "how_it_works": SD_PAGE["how_it_works"], |
| 360 | "config_file": SD_PAGE["config_file"], |
| 361 | "rule_eval": SD_PAGE["rule_eval"], |
| 362 | "helpers": SD_PAGE["helpers"], |
| 363 | "config_template": SD_PAGE["config_template"], |
| 364 | "supported": SD_PAGE["supported"], |
| 365 | "mixing": SD_PAGE["mixing"], |
| 366 | "troubleshooting": SD_PAGE["troubleshooting"], |
| 367 | } |
| 368 | |
| 369 | |
| 370 | def render_sd_md(integrations: Any) -> str: |
| 371 | template = get_jinja_env().get_template("service_discovery.md") |
| 372 | return template.render( |
| 373 | page=build_page_context(), |
| 374 | discoverers=build_discoverers_context(integrations), |
| 375 | ) |
| 376 | |
| 377 | |
| 378 | def generate_sd_md() -> None: |
| 379 | integrations = load_integrations() |
| 380 | content = render_sd_md(integrations) |
| 381 | |
| 382 | outfile = pathlib.Path("./src/collectors/SERVICE-DISCOVERY.md") |
| 383 | outfile.parent.mkdir(parents=True, exist_ok=True) |
| 384 | |
| 385 | tmp = outfile.with_suffix(outfile.suffix + ".tmp") |
| 386 | tmp.write_text(content.rstrip("\n") + "\n", encoding="utf-8") |
| 387 | tmp.replace(outfile) |
| 388 | |
| 389 | |
| 390 | if __name__ == "__main__": |
| 391 | generate_sd_md() |