| 1 | """ |
| 2 | Generate src/collectors/SECRETS.md from integrations/integrations.js. |
| 3 | |
| 4 | This script: |
| 5 | - reads discovered secretstore integrations from integrations.js; |
| 6 | - renders a shared Secrets Management entry page; |
| 7 | - keeps shared resolver documentation in one dedicated configuration block. |
| 8 | """ |
| 9 | |
| 10 | from __future__ import annotations |
| 11 | |
| 12 | import json |
| 13 | import pathlib |
| 14 | import re |
| 15 | from typing import Any, Dict, List |
| 16 | |
| 17 | GITHUB_BLOB_PREFIX = "https://github.com/netdata/netdata/blob/master" |
| 18 | TEMPLATE_PATH = pathlib.Path(__file__).resolve().parent / "templates" |
| 19 | |
| 20 | SECRETS_PAGE = { |
| 21 | "title": "# Secrets Management", |
| 22 | "intro": [ |
| 23 | "Keep collector credentials out of plain-text configuration files.", |
| 24 | "Netdata lets you reference secret values in collector configs instead of storing them directly in YAML. " |
| 25 | "Depending on where the secret lives, you can resolve it from environment variables, local files, local " |
| 26 | "commands, or remote secretstore backends.", |
| 27 | ], |
| 28 | "jump_to": [ |
| 29 | {"label": "Resolver Quick Reference", "anchor": "resolver-quick-reference"}, |
| 30 | {"label": "Choosing a Resolver", "anchor": "choosing-a-resolver"}, |
| 31 | {"label": "Environment Variables", "anchor": "environment-variables"}, |
| 32 | {"label": "Files", "anchor": "files"}, |
| 33 | {"label": "Commands", "anchor": "commands"}, |
| 34 | {"label": "Secretstores", "anchor": "secretstores"}, |
| 35 | {"label": "Supported Secretstore Backends", "anchor": "supported-secretstore-backends"}, |
| 36 | {"label": "How It Works", "anchor": "how-it-works"}, |
| 37 | {"label": "Troubleshooting", "anchor": "troubleshooting"}, |
| 38 | ], |
| 39 | "quick_reference": [ |
| 40 | { |
| 41 | "resolver": "Environment variable", |
| 42 | "syntax": "`${env:VAR_NAME}`", |
| 43 | "best_for": "Secrets already injected into the Netdata service environment", |
| 44 | "notes": "Value is trimmed. The variable must exist.", |
| 45 | }, |
| 46 | { |
| 47 | "resolver": "File", |
| 48 | "syntax": "`${file:/absolute/path}`", |
| 49 | "best_for": "Secrets stored in local files on disk", |
| 50 | "notes": "The path must be absolute. File contents are trimmed.", |
| 51 | }, |
| 52 | { |
| 53 | "resolver": "Command", |
| 54 | "syntax": "`${cmd:/absolute/path/to/command args}`", |
| 55 | "best_for": "Secrets returned by a trusted local command", |
| 56 | "notes": "The command path must be absolute. Netdata uses a 10-second timeout.", |
| 57 | }, |
| 58 | { |
| 59 | "resolver": "Secretstore", |
| 60 | "syntax": "`${store:<kind>:<name>:<operand>}`", |
| 61 | "best_for": "Secrets stored in remote backends such as Vault, AWS, Azure, or GCP", |
| 62 | "notes": "Configure the secretstore first, then reference it from collector configs.", |
| 63 | }, |
| 64 | ], |
| 65 | "choosing_a_resolver": [ |
| 66 | "Use `${env:...}` or `${file:...}` for simple setups where secrets are already available locally on the Netdata host.", |
| 67 | "Use `${cmd:...}` when you need dynamic secret retrieval via a trusted local command, such as 1Password CLI or a custom script.", |
| 68 | "Use `${store:...}` when your organization manages secrets centrally in a cloud provider or Vault and you want Netdata to pull from that source directly.", |
| 69 | "You can use different resolver types across different collectors, different jobs within the same collector, or even within the same configuration value. See [Mixing resolver types](#mixing-resolver-types).", |
| 70 | ], |
| 71 | "sections": { |
| 72 | "env": { |
| 73 | "heading": "## Environment Variables", |
| 74 | "body": "Use `${env:VARIABLE_NAME}` to read a secret from the Netdata process environment.", |
| 75 | "example": """```yaml |
| 76 | jobs: |
| 77 | - name: mysql_prod |
| 78 | password: "${env:MYSQL_PASSWORD}" |
| 79 | ```""", |
| 80 | "notes": [ |
| 81 | "Netdata trims leading and trailing whitespace from the environment variable value.", |
| 82 | "The variable must be set in the environment of the Netdata service or process that runs the collector.", |
| 83 | ], |
| 84 | }, |
| 85 | "file": { |
| 86 | "heading": "## Files", |
| 87 | "body": "Use `${file:/absolute/path}` to read a secret from a local file on disk.", |
| 88 | "example": """```yaml |
| 89 | jobs: |
| 90 | - name: mysql_prod |
| 91 | password: "${file:/run/secrets/mysql_password}" |
| 92 | ```""", |
| 93 | "notes": [ |
| 94 | "The file path must be absolute.", |
| 95 | "Netdata trims leading and trailing whitespace from the file contents.", |
| 96 | "The file must exist on the Netdata host and be readable by the `netdata` user.", |
| 97 | "**Docker Secrets**: Docker mounts secrets as files under `/run/secrets/` inside the container. Use `${file:/run/secrets/<secret-name>}` to read them.", |
| 98 | "**Kubernetes Secrets**: If you mount Kubernetes Secrets as volume files in the Netdata pod, reference them with `${file:/path/to/mounted/secret}`.", |
| 99 | ], |
| 100 | }, |
| 101 | "cmd": { |
| 102 | "heading": "## Commands", |
| 103 | "body": "Use `${cmd:/absolute/path/to/command args}` to execute a trusted local command and use its stdout as the secret value.", |
| 104 | "example": """```yaml |
| 105 | jobs: |
| 106 | - name: mysql_prod |
| 107 | password: "${cmd:/usr/bin/op read op://vault/netdata/mysql/password}" |
| 108 | ```""", |
| 109 | "notes": [ |
| 110 | "The command path must be absolute.", |
| 111 | "Arguments are split on whitespace. Netdata does not interpret shell quoting, pipes, redirects, or variable expansion unless you explicitly run a shell such as `/bin/sh -c`.", |
| 112 | "Netdata uses a 10-second timeout for command resolvers.", |
| 113 | "Netdata trims leading and trailing whitespace from stdout and ignores stderr.", |
| 114 | ], |
| 115 | }, |
| 116 | }, |
| 117 | "store": { |
| 118 | "heading": "## Secretstores", |
| 119 | "body": "Use secretstores when you want Netdata collectors to fetch secrets from remote backends at runtime instead of storing them locally in collector configs.", |
| 120 | "reference_intro": "Configure a secretstore first, then reference it from collector configs with:", |
| 121 | "reference_syntax": "${store:<kind>:<name>:<operand>}", |
| 122 | "reference_parts": [ |
| 123 | {"name": "`kind`", "description": "Secretstore backend kind, such as `vault` or `aws-sm`."}, |
| 124 | {"name": "`name`", "description": "The store name you configured in Netdata, such as `vault_prod`."}, |
| 125 | {"name": "`operand`", "description": "Backend-specific identifier for the secret you want to read."}, |
| 126 | ], |
| 127 | "example": """```yaml |
| 128 | jobs: |
| 129 | - name: mysql_prod |
| 130 | password: "${store:vault:vault_prod:secret/data/netdata/mysql#password}" |
| 131 | ```""", |
| 132 | "ui_steps": [ |
| 133 | "Open the Netdata Dynamic Configuration UI.", |
| 134 | "Go to `Collectors -> go.d -> SecretStores`.", |
| 135 | "Choose the backend kind you want to configure.", |
| 136 | "Give the secretstore a name.", |
| 137 | "Fill in the backend-specific settings.", |
| 138 | "Save the secretstore and use its `${store:<kind>:<name>:<operand>}` reference in collector configs.", |
| 139 | ], |
| 140 | "file_intro": "Each secretstore backend has its own file under `/etc/netdata/go.d/ss/`:", |
| 141 | "file_note": "File-based secretstores are loaded at agent startup. If you edit these files, restart the Netdata Agent to apply the changes.", |
| 142 | "file_directory": ( |
| 143 | "If the `/etc/netdata/go.d/ss/` directory does not exist, create it:\n\n" |
| 144 | "```bash\n" |
| 145 | "sudo mkdir -p /etc/netdata/go.d/ss\n" |
| 146 | "sudo chown netdata:netdata /etc/netdata/go.d/ss\n" |
| 147 | "sudo chmod 0750 /etc/netdata/go.d/ss\n" |
| 148 | "```\n\n" |
| 149 | "Secretstore configuration files may contain sensitive values such as tokens or client secrets. " |
| 150 | "Restrict directory and file permissions to the `netdata` user." |
| 151 | ), |
| 152 | "mixing": ( |
| 153 | "You can mix different resolver types in the same configuration value or the same config file. " |
| 154 | "For example, you might read the username from an environment variable and the password from a secretstore:\n\n" |
| 155 | "```yaml\n" |
| 156 | "jobs:\n" |
| 157 | " - name: mysql_prod\n" |
| 158 | ' dsn: "${env:MYSQL_USER}:${store:vault:vault_prod:secret/data/netdata/mysql#password}@tcp(127.0.0.1:3306)/"\n' |
| 159 | "```\n\n" |
| 160 | "Different jobs within the same collector config file can also use different resolver types." |
| 161 | ), |
| 162 | "multiple_stores": ( |
| 163 | "Each secretstore config file can contain multiple `jobs` entries, each with a unique store name. " |
| 164 | "You can use different secretstore backends simultaneously. " |
| 165 | "For example, you might configure a Vault store for database credentials and an AWS Secrets Manager store for API keys, " |
| 166 | "then reference each one using its `${store:<kind>:<name>:<operand>}` syntax in the relevant collector configs." |
| 167 | ), |
| 168 | }, |
| 169 | "secretstores": { |
| 170 | "heading": "## Supported Secretstore Backends", |
| 171 | "intro": "Use the backend README for provider-specific authentication, operand rules, configuration examples, and troubleshooting.", |
| 172 | }, |
| 173 | "how_it_works": [ |
| 174 | "Secrets are resolved each time a collector job starts or restarts.", |
| 175 | "If a secret cannot be resolved, the collector job will fail to start and log an error.", |
| 176 | "Updating a secretstore automatically restarts running and failed collector jobs that use it so they pick up the new credentials.", |
| 177 | "Accepted or disabled jobs keep their state and use the updated secretstore the next time they start.", |
| 178 | "If a secretstore change applies successfully but some dependent collector restarts fail, Netdata reports those restart failures.", |
| 179 | ], |
| 180 | "security_notes": [ |
| 181 | "Prefer secret references over plain-text credentials in collector configs.", |
| 182 | "Prefer platform-native identity modes for production when a backend supports them, such as instance roles, managed identities, or metadata-based credentials.", |
| 183 | "Secretstore configuration values (such as tokens and client secrets) also support `${env:...}`, `${file:...}`, and `${cmd:...}` resolvers. Use them to avoid storing backend credentials in plain text. Note that `${store:...}` references are not supported inside secretstore configurations.", |
| 184 | "Keep local secret material readable only by the `netdata` user, including token files, service account files, and any files used with `${file:...}`.", |
| 185 | "Use `${cmd:...}` only with trusted local commands and absolute paths.", |
| 186 | ], |
| 187 | "troubleshooting": { |
| 188 | "intro": [ |
| 189 | "Secret resolution failures appear in agent logs and usually surface as collector jobs failing to start.", |
| 190 | "Start by checking the resolver syntax you used in the collector config.", |
| 191 | "For `${env:...}`, make sure the variable exists in the Netdata process environment.", |
| 192 | "For `${file:...}`, make sure the path is absolute and the file is readable by `netdata`.", |
| 193 | "For `${cmd:...}`, make sure the command path is absolute and the command completes within 10 seconds.", |
| 194 | "For `${store:...}`, check the backend README for provider-specific operand rules, authentication requirements, and troubleshooting.", |
| 195 | ], |
| 196 | "errors": [ |
| 197 | {"syntax": "`${env:VAR_NAME}`", "message": "environment variable is not set"}, |
| 198 | {"syntax": "`${file:relative/path}`", "message": "file path must be absolute"}, |
| 199 | {"syntax": "`${cmd:echo hello}`", "message": "command path must be absolute"}, |
| 200 | {"syntax": "`${cmd:/path/to/slow-command}`", "message": "command timed out after 10s"}, |
| 201 | ], |
| 202 | }, |
| 203 | } |
| 204 | |
| 205 | |
| 206 | def _extract_integrations_json(js_text: str) -> str: |
| 207 | """Extract integrations JSON from integrations.js.""" |
| 208 | after_categories = js_text.split("export const categories = ", 1)[1] |
| 209 | _, after_integrations = after_categories.split("export const integrations = ", 1) |
| 210 | return re.split(r"\n\s*export const|\Z", after_integrations, maxsplit=1)[0].strip().rstrip(';').strip() |
| 211 | |
| 212 | |
| 213 | def load_integrations(js_path: str = "integrations/integrations.js") -> Any: |
| 214 | """Load integrations catalog from JavaScript.""" |
| 215 | with open(js_path, "r", encoding="utf-8") as f: |
| 216 | js_data = f.read() |
| 217 | return json.loads(_extract_integrations_json(js_data)) |
| 218 | |
| 219 | |
| 220 | def iterate_integrations(integrations: Any): |
| 221 | """Yield integration objects from dict or list.""" |
| 222 | if isinstance(integrations, dict): |
| 223 | for integ in integrations.values(): |
| 224 | if isinstance(integ, dict): |
| 225 | yield integ |
| 226 | elif isinstance(integrations, list): |
| 227 | for integ in integrations: |
| 228 | if isinstance(integ, dict): |
| 229 | yield integ |
| 230 | |
| 231 | |
| 232 | def collect_secretstore_integrations(integrations: Any) -> List[Dict[str, Any]]: |
| 233 | """Collect secretstore integration entries.""" |
| 234 | items = [] |
| 235 | |
| 236 | for integ in iterate_integrations(integrations): |
| 237 | if integ.get("integration_type") != "secretstore": |
| 238 | continue |
| 239 | meta = integ.get("meta", {}) |
| 240 | if not isinstance(meta, dict): |
| 241 | continue |
| 242 | if not isinstance(meta.get("name"), str) or not isinstance(meta.get("kind"), str): |
| 243 | continue |
| 244 | items.append(integ) |
| 245 | |
| 246 | items.sort(key=lambda item: item["meta"]["name"].lower()) |
| 247 | return items |
| 248 | |
| 249 | |
| 250 | def get_repo_path_from_blob_url(url: str) -> str: |
| 251 | """Convert a GitHub blob URL to a repo-root markdown path when possible.""" |
| 252 | if url.startswith(GITHUB_BLOB_PREFIX): |
| 253 | return url[len(GITHUB_BLOB_PREFIX):] |
| 254 | return url |
| 255 | |
| 256 | |
| 257 | def get_secretstore_readme_link(integ: Dict[str, Any]) -> str: |
| 258 | """Generate the repo-local README link for a secretstore backend.""" |
| 259 | edit_link = integ.get("edit_link", "") if isinstance(integ, dict) else "" |
| 260 | repo_path = get_repo_path_from_blob_url(edit_link) |
| 261 | if repo_path.endswith("/metadata.yaml"): |
| 262 | return repo_path[: -len("metadata.yaml")] + "README.md" |
| 263 | return "" |
| 264 | |
| 265 | |
| 266 | _jinja_env = None |
| 267 | |
| 268 | |
| 269 | def get_jinja_env(): |
| 270 | """Return the shared Jinja environment used by docs templates.""" |
| 271 | global _jinja_env |
| 272 | |
| 273 | if _jinja_env is None: |
| 274 | from jinja2 import Environment, FileSystemLoader, select_autoescape |
| 275 | |
| 276 | _jinja_env = Environment( |
| 277 | loader=FileSystemLoader(TEMPLATE_PATH), |
| 278 | autoescape=select_autoescape(), |
| 279 | block_start_string='[%', |
| 280 | block_end_string='%]', |
| 281 | variable_start_string='[[', |
| 282 | variable_end_string=']]', |
| 283 | comment_start_string='[#', |
| 284 | comment_end_string='#]', |
| 285 | trim_blocks=True, |
| 286 | lstrip_blocks=True, |
| 287 | ) |
| 288 | |
| 289 | return _jinja_env |
| 290 | |
| 291 | |
| 292 | def build_secretstores_context(integrations: Any) -> List[Dict[str, str]]: |
| 293 | """Build template context for discovered secretstore backends.""" |
| 294 | items = [] |
| 295 | |
| 296 | for integ in collect_secretstore_integrations(integrations): |
| 297 | meta = integ.get("meta", {}) |
| 298 | summary = integ.get("collector_configs_summary", {}) |
| 299 | if not isinstance(summary, dict): |
| 300 | summary = {} |
| 301 | kind = meta.get("kind", "") |
| 302 | readme_link = get_secretstore_readme_link(integ) |
| 303 | name = meta.get("name", "") |
| 304 | items.append( |
| 305 | { |
| 306 | "name": name, |
| 307 | "kind": kind, |
| 308 | "config_file": f"/etc/netdata/go.d/ss/{kind}.conf", |
| 309 | "name_link": f'[{name}]({readme_link})', |
| 310 | "operand_format": summary.get("operand_format", "See backend README"), |
| 311 | "example_operand": summary.get("example_operand", "See backend README"), |
| 312 | } |
| 313 | ) |
| 314 | |
| 315 | return items |
| 316 | |
| 317 | |
| 318 | def build_page_context() -> Dict[str, Any]: |
| 319 | """Build template-friendly page context.""" |
| 320 | return { |
| 321 | "title": SECRETS_PAGE["title"], |
| 322 | "intro": SECRETS_PAGE["intro"], |
| 323 | "jump_to_line": " • ".join( |
| 324 | f'[{jump["label"]}](#{jump["anchor"]})' for jump in SECRETS_PAGE["jump_to"] |
| 325 | ), |
| 326 | "quick_reference": SECRETS_PAGE["quick_reference"], |
| 327 | "choosing_a_resolver": SECRETS_PAGE["choosing_a_resolver"], |
| 328 | "sections": [ |
| 329 | SECRETS_PAGE["sections"]["env"], |
| 330 | SECRETS_PAGE["sections"]["file"], |
| 331 | SECRETS_PAGE["sections"]["cmd"], |
| 332 | ], |
| 333 | "store": SECRETS_PAGE["store"], |
| 334 | "secretstores": { |
| 335 | "heading": SECRETS_PAGE["secretstores"]["heading"], |
| 336 | "intro": SECRETS_PAGE["secretstores"]["intro"], |
| 337 | }, |
| 338 | "how_it_works": SECRETS_PAGE["how_it_works"], |
| 339 | "security_notes": SECRETS_PAGE["security_notes"], |
| 340 | "troubleshooting": SECRETS_PAGE["troubleshooting"], |
| 341 | } |
| 342 | |
| 343 | |
| 344 | def render_secrets_md(integrations: Any) -> str: |
| 345 | """Render the shared Secrets Management entry page.""" |
| 346 | template = get_jinja_env().get_template("secrets.md") |
| 347 | return template.render( |
| 348 | page=build_page_context(), |
| 349 | secretstores=build_secretstores_context(integrations), |
| 350 | ) |
| 351 | |
| 352 | |
| 353 | def generate_secrets_md() -> None: |
| 354 | """Generate SECRETS.md from integrations.js and shared resolver content.""" |
| 355 | integrations = load_integrations() |
| 356 | content = render_secrets_md(integrations) |
| 357 | |
| 358 | outfile = pathlib.Path("./src/collectors/SECRETS.md") |
| 359 | outfile.parent.mkdir(parents=True, exist_ok=True) |
| 360 | |
| 361 | tmp = outfile.with_suffix(outfile.suffix + ".tmp") |
| 362 | tmp.write_text(content.rstrip("\n") + "\n", encoding="utf-8") |
| 363 | tmp.replace(outfile) |
| 364 | |
| 365 | |
| 366 | if __name__ == "__main__": |
| 367 | generate_secrets_md() |