doc: improve COLLECTORS.md generation (#21225)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Ilya Mashchenko committed
Oct 28, 2025 at 14:36 UTC
9ac40379125f4d68fe90c700752914cfca82486f
2 files changed
+387
-48
docs/.map/map.csv
+1
-1
@@ -3,6 +3,7 @@ https://github.com/netdata/netdata/edit/master/docs/welcome-to-netdata.md,Welcom
3
https://github.com/netdata/netdata/edit/master/docs/netdata-enterprise-evaluation-corrected.md,Enterprise Evaluation Guide,Published,Welcome to Netdata,
4
https://github.com/netdata/netdata/edit/master/docs/realtime-monitoring.md,Real-time Monitoring,Published,Welcome to Netdata,
5
https://github.com/netdata/netdata/edit/master/docs/scalability.md,Scalability,Published,Welcome to Netdata,
6
+https://github.com/netdata/netdata/edit/master/src/collectors/COLLECTORS.md,Monitor Anything,Published,Welcome to Netdata,Netdata gathers real-time metrics from hundreds of data sources using collectors. Most require zero configuration and are pre-configured out of the box.
7
https://github.com/netdata/netdata/edit/master/docs/fleet-configuration-management.md,Fleet Deployment and Configuration Management,Published,Welcome to Netdata,
8
https://github.com/netdata/netdata/edit/master/docs/getting-started-netdata/guide.md,Getting Started,Published,root,
9
https://github.com/netdata/netdata/edit/master/docs/Demo-Sites.md,Live Demo,Published,root,
@@ -122,7 +123,6 @@ https://github.com/netdata/netdata/edit/master/docs/observability-centralization
123
https://github.com/netdata/netdata/edit/master/docs/collecting-metrics/system-metrics.md,System metrics,Unpublished,Collecting Metrics,"Netdata collects thousands of metrics from physical and virtual systems, IoT/edge devices, and containers with zero configuration."
124
https://github.com/netdata/netdata/edit/master/docs/collecting-metrics/application-metrics.md,Application metrics,Unpublished,Collecting Metrics,"Monitor and troubleshoot every application on your infrastructure with per-second metrics, zero configuration, and meaningful charts."
125
https://github.com/netdata/netdata/edit/master/docs/collecting-metrics/container-metrics.md,Container metrics,Unpublished,Collecting Metrics,Use Netdata to collect per-second utilization and application-level metrics from Linux/Docker containers and Kubernetes clusters.
125
-https://github.com/netdata/netdata/edit/master/src/collectors/COLLECTORS.md,Monitor Anything,Published,Collecting Metrics,Netdata gathers real-time metrics from hundreds of data sources using collectors. Most require zero configuration and are pre-configured out of the box.
126
https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/collector/snmp/profile-format.md,SNMP Profile Format,Published,Collecting Metrics,Learn how Netdata’s SNMP collector uses profiles.
127
collectors_integrations,,,,
128
,,,,
integrations/gen_doc_collector_page.py
+386
-47
@@ -1,69 +1,408 @@
1
"""
2
-This script reads the integrations/integrations.js file and generates the list of data collection integrations inside collectors/COLLECTORS.md, with proper links that Learn can replace into Learn links.
2
+Generate the integrations section in COLLECTORS.md from integrations/integrations.js
3
+Key behavior:
4
+- Use **children of 'data-collection' (second-level categories) as section headings**. Any child category IDs on integrations
5
+ are rolled up to their section-level parent.
6
+- Read categories from meta.monitored_instance.categories (array of strings).
7
+- If an integration has no categories, assign categories where any ancestor has collector_default=true,
8
+ rolled up to its section-level parent(s).
9
+- Render Markdown tables: | Integration | Description |.
10
"""
11
12
+from __future__ import annotations
13
+
14
import json
15
import pathlib
16
+import re
17
+from typing import Any, Dict, List, Tuple, Iterable, Optional
18
+
19
+
20
+def _extract_json_blobs(js_text: str) -> Tuple[str, str]:
21
+ after_categories = js_text.split("export const categories = ", 1)[1]
22
+ categories_str, after_integrations = after_categories.split("export const integrations = ", 1)
23
+ integrations_str = after_integrations
24
+
25
+ def _cleanup(s: str) -> str:
26
+ s = re.split(r"\n\s*export const|\Z", s, maxsplit=1)[0].strip()
27
+ if s.endswith(';'):
28
+ s = s[:-1]
29
+ return s.strip()
30
+
31
+ return _cleanup(categories_str), _cleanup(integrations_str)
32
+
33
+
34
+def _load_catalog(js_path: str = 'integrations/integrations.js'):
35
+ with open(js_path, 'r', encoding='utf-8') as f:
36
+ js_data = f.read()
37
+ categories_str, integrations_str = _extract_json_blobs(js_data)
38
+ categories = json.loads(categories_str) # expected array tree
39
+ integrations = json.loads(integrations_str) # expected dict map
40
+ return categories, integrations
41
+
42
+
43
+def _build_category_maps(categories: List[Dict[str, Any]]):
44
+ """Build maps: id->title, id->parent_id, section_level list (ordered), and defaults rolled to section-level.
45
+
46
+ We use the children of 'data-collection' as section headings (second-level categories).
47
+
48
+ """
49
+ id_to_parent: Dict[str, Optional[str]] = {}
50
+ id_to_title: Dict[str, str] = {}
51
+ section_level_ids: List[str] = [] # Second-level categories under data-collection
52
+ default_ids: List[str] = [] # category ids where collector_default==true
53
+
54
+ def walk(nodes: Iterable[Dict[str, Any]], parent: Optional[str], depth: int = 0):
55
+ for node in nodes or []:
56
+ if not isinstance(node, dict):
57
+ continue
58
+ cid = node.get('id')
59
+ title = node.get('name') or node.get('title') or (cid or '')
60
+ if not cid:
61
+ continue
62
+ id_to_parent[cid] = parent
63
+ id_to_title[cid] = title
64
+
65
+ # If this is a child of 'data-collection', it's a section heading
66
+ if parent == 'data-collection':
67
+ section_level_ids.append(cid)
68
+
69
+ if node.get('collector_default') is True:
70
+ default_ids.append(cid)
71
+
72
+ children = node.get('children') or []
73
+ if isinstance(children, list) and children:
74
+ walk(children, cid, depth + 1)
75
+
76
+ walk(categories if isinstance(categories, list) else [], parent=None, depth=0)
77
+
78
+ # Find section-level ancestor (child of data-collection)
79
+ def section_ancestor(cid: str) -> Optional[str]:
80
+ # Walk up until we find a category whose parent is 'data-collection'
81
+ cur = cid
82
+ seen = set()
83
+ while cur is not None and cur in id_to_parent and cur not in seen:
84
+ seen.add(cur)
85
+ parent = id_to_parent.get(cur)
86
+ if parent == 'data-collection':
87
+ return cur
88
+ if parent is None:
89
+ # If we reach the top without finding data-collection, this might be a top-level category
90
+ return None
91
+ cur = parent
92
+ return None
93
+
94
+ # Roll default ids to their section-level parents (unique)
95
+ default_section_level = []
96
+ for d in default_ids:
97
+ section = section_ancestor(d)
98
+ if section and section not in default_section_level:
99
+ default_section_level.append(section)
100
+
101
+ return id_to_title, id_to_parent, section_level_ids, default_section_level, section_ancestor
102
+
103
+
104
+def _desc_for_integration(integ: Dict[str, Any]) -> str:
105
+ """Generate user-friendly description for integration."""
106
+ overview = integ.get('overview')
107
+ if isinstance(overview, dict):
108
+ dc = overview.get('data_collection')
109
+ if isinstance(dc, dict):
110
+ md = dc.get('metrics_description')
111
+ if isinstance(md, str) and md.strip():
112
+ return re.sub(r"\s+", " ", md.strip())
113
+ mi = integ.get('meta', {}).get('monitored_instance', {})
114
+ if isinstance(mi, dict):
115
+ mi_desc = mi.get('description')
116
+ if isinstance(mi_desc, str) and mi_desc.strip():
117
+ return re.sub(r"\s+", " ", mi_desc.strip())
118
+ for key in ('short_description', 'description', 'summary'):
119
+ v = integ.get(key)
120
+ if isinstance(v, str) and v.strip():
121
+ return re.sub(r"\s+", " ", v.strip())
122
+ name = (mi.get('name') if isinstance(mi, dict) else None) or integ.get('name') or 'Integration'
123
+ return f"Metrics for {name}"
124
+
125
+
126
+def _to_slug(text: str) -> str:
127
+ """Convert a string to a slug suitable for URLs and anchors."""
128
+ return text.lower().replace(' ', '_').replace('/', '-').replace('(', '').replace(')', '')
129
+
130
+
131
+def _doc_link(integ: Dict[str, Any], display_name: str) -> str:
132
+ base = (integ.get('edit_link') or '') if isinstance(integ, dict) else ''
133
+ base = base.replace('metadata.yaml', '')
134
+ slug = _to_slug(display_name)
135
+ return f"{base}integrations/{slug}.md"
136
+
137
+
138
+def _collect_sections(categories: List[Dict[str, Any]], integrations: Dict[str, Any]):
139
+ id_to_title, id_to_parent, section_level_ids, default_section_level, section_ancestor = _build_category_maps(
140
+ categories)
141
+
142
+ # Custom section order: Linux Systems first, then others, but exclude "Other" category
143
+ ordered_sections = []
144
+ linux_id = None
145
+ other_id = None
146
+
147
+ for cid in section_level_ids:
148
+ if 'linux-systems' in cid.lower():
149
+ linux_id = cid
150
+ elif id_to_title.get(cid, '').lower() == 'other':
151
+ other_id = cid
152
+
153
+ if linux_id:
154
+ ordered_sections.append(linux_id)
155
+
156
+ for cid in section_level_ids:
157
+ if cid != linux_id and cid != other_id:
158
+ ordered_sections.append(cid)
159
+
160
+ # Prepare buckets for section-level categories
161
+ per_section: Dict[str, List[Tuple[str, str, str]]] = {cid: [] for cid in ordered_sections}
162
+ if other_id:
163
+ per_section[other_id] = []
164
+ other_bucket: List[Tuple[str, str, str]] = []
165
8
-# Open integrations/integrations.js and extract the dictionaries
9
-with open('integrations/integrations.js') as dataFile:
10
- data = dataFile.read()
166
+ items = integrations.items() if isinstance(integrations, dict) else enumerate(
167
+ integrations if isinstance(integrations, list) else [])
168
+ for _key, integ in items:
169
+ if not isinstance(integ, dict):
170
+ continue
171
+ mi = integ.get('meta', {}).get('monitored_instance', {})
172
+ name = (mi.get('name') if isinstance(mi, dict) else None) or integ.get('name')
173
+ if not isinstance(name, str) or not name.strip():
174
+ continue
175
+ link = _doc_link(integ, name)
176
+ desc = _desc_for_integration(integ)
177
12
- categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0]
13
- integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1]
178
+ cats = []
179
+ if isinstance(mi, dict):
180
+ cats = mi.get('categories') or []
181
+ if isinstance(cats, str):
182
+ cats = [cats]
183
+ if not isinstance(cats, list):
184
+ cats = []
185
15
- categories = json.loads(categories_str)
16
- integrations = json.loads(integrations_str)
186
+ if not cats:
187
+ # use default section-level categories
188
+ target_sections = list(default_section_level) if default_section_level else []
189
+ else:
190
+ # roll each category id to its section-level ancestor
191
+ target_sections = []
192
+ for cid in cats:
193
+ section = section_ancestor(cid) if isinstance(cid, str) else None
194
+ if section and (section in per_section or section == other_id) and section not in target_sections:
195
+ target_sections.append(section)
196
18
-cat_dict = {}
19
-data_col_cat = {}
197
+ if not target_sections:
198
+ other_bucket.append((name, link, desc))
199
+ else:
200
+ for sec in target_sections:
201
+ if sec in per_section or sec == other_id:
202
+ per_section[sec].append((name, link, desc))
203
204
+ # Build ordered sections with non-empty tables
205
+ sections: List[Tuple[str, List[Tuple[str, str, str]]]] = []
206
+ for cid in ordered_sections:
207
+ items = per_section.get(cid, [])
208
+ if items:
209
+ items.sort(key=lambda t: t[0].lower())
210
+ sections.append((id_to_title.get(cid, cid), items))
211
+
212
+ # Add "Other" section last - either from other_bucket or from the Other category
213
+ if other_id and per_section.get(other_id):
214
+ items = per_section[other_id]
215
+ items.sort(key=lambda t: t[0].lower())
216
+ sections.append((id_to_title.get(other_id, "Other"), items))
217
+ elif other_bucket:
218
+ other_bucket.sort(key=lambda t: t[0].lower())
219
+ sections.append(("Other", other_bucket))
220
+
221
+ return sections
222
+
223
+
224
+def _render_tech_navigation() -> str:
225
+ """Render the 'Find Your Technology' navigation section."""
226
+ # Configuration for "Find Your Technology" section
227
+ tech_categories = [
228
+ {
229
+ "title": "Cloud & Infrastructure:",
230
+ "items": [
231
+ ("AWS", "#cloud-provider-managed"),
232
+ ("Azure", "#cloud-provider-managed"),
233
+ ("GCP", "#cloud-provider-managed"),
234
+ ("Kubernetes", "#kubernetes"),
235
+ ("Docker", "#containers-and-vms"),
236
+ ("VMware", "#containers-and-vms"),
237
+ ]
238
+ },
239
+ {
240
+ "title": "Databases & Caching:",
241
+ "items": [
242
+ ("MySQL", "#databases"),
243
+ ("PostgreSQL", "#databases"),
244
+ ("MongoDB", "#databases"),
245
+ ("Redis", "#databases"),
246
+ ("Elasticsearch", "#search-engines"),
247
+ ("Oracle", "#databases"),
248
+ ]
249
+ },
250
+ {
251
+ "title": "Web & Application:",
252
+ "items": [
253
+ ("NGINX", "#web-servers-and-web-proxies"),
254
+ ("Apache", "#web-servers-and-web-proxies"),
255
+ ("HAProxy", "#web-servers-and-web-proxies"),
256
+ ("Tomcat", "#web-servers-and-web-proxies"),
257
+ ("PHP-FPM", "#web-servers-and-web-proxies"),
258
+ ]
259
+ },
260
+ {
261
+ "title": "Message Queues:",
262
+ "items": [
263
+ ("Kafka", "#message-brokers"),
264
+ ("RabbitMQ", "#message-brokers"),
265
+ ("ActiveMQ", "#message-brokers"),
266
+ ("NATS", "#message-brokers"),
267
+ ("Pulsar", "#message-brokers"),
268
+ ]
269
+ },
270
+ {
271
+ "title": "Operating Systems:",
272
+ "items": [
273
+ ("Linux", "#linux-systems"),
274
+ ("Windows", "#windows-systems"),
275
+ ("macOS", "#macos-systems"),
276
+ ("FreeBSD", "#freebsd"),
277
+ ]
278
+ },
279
+ ]
280
+
281
+ # Build "Find Your Technology" section
282
+ tech_lines = []
283
+ for category in tech_categories:
284
+ links = " • ".join([f"[{name}]({anchor})" for name, anchor in category["items"]])
285
+ tech_lines.append(f"**{category['title']}**\n{links}\n")
286
+
287
+ tech_section = "\n".join(tech_lines)
288
+
289
+ return f"""### Find Your Technology
290
+
291
+**Select your primary infrastructure to jump directly to relevant integrations:**
292
+
293
+{tech_section}
294
+**Don't see what you need?** We support [Prometheus endpoints](#generic-data-collection), [SNMP devices](#generic-data-collection), [StatsD](#beyond-the-850-integrations), and [custom data sources](#generic-data-collection).
295
+"""
296
+
297
+
298
+def _render_generic_collectors() -> str:
299
+ """Render the 'Beyond the 850+ integrations' section with generic collectors."""
300
+ # Configuration for generic collectors
301
+ generic_collectors = [
302
+ {
303
+ "name": "Prometheus collector",
304
+ "link": "/src/go/plugin/go.d/collector/prometheus/README.md",
305
+ "description": "Any application exposing Prometheus metrics"
306
+ },
307
+ {
308
+ "name": "StatsD collector",
309
+ "link": "/src/collectors/statsd.plugin/README.md",
310
+ "description": "Applications instrumented with [StatsD](https://blog.netdata.cloud/introduction-to-statsd/)"
311
+ },
312
+ {
313
+ "name": "Pandas collector",
314
+ "link": "/src/collectors/python.d.plugin/pandas/README.md",
315
+ "description": "Structured data from CSV, JSON, XML, and more"
316
+ },
317
+ ]
318
+
319
+ # Build generic collectors section
320
+ collector_lines = []
321
+ for collector in generic_collectors:
322
+ collector_lines.append(f"- **[{collector['name']}]({collector['link']})** - {collector['description']}")
323
+
324
+ collectors_section = "\n".join(collector_lines)
325
+
326
+ return f"""## Beyond the 850+ integrations
327
+
328
+Netdata can monitor virtually any application through generic collectors:
329
+
330
+{collectors_section}
331
+
332
+Need a dedicated integration? [Submit a feature request](https://github.com/netdata/netdata/issues/new/choose) on GitHub.
333
+"""
334
+
335
+
336
+def _render_header() -> str:
337
+ """Render the marketing header and navigation sections."""
338
+ tech_nav = _render_tech_navigation()
339
+ generic_section = _render_generic_collectors()
340
+
341
+ return f"""# Monitor anything with Netdata
342
+
343
+**850+ integrations. Zero configuration. Deploy anywhere.**
344
+
345
+Netdata uses collectors to help you gather metrics from your favorite applications and services and view them in real-time, interactive charts. The following list includes all the integrations where Netdata can gather metrics from.
346
+
347
+Learn more about [how collectors work](/src/collectors/README.md), and then learn how to [enable or configure](/src/collectors/REFERENCE.md#enable-or-disable-collectors-and-plugins) a specific collector.
348
+
349
+### Why Teams Choose Us
350
+
351
+- ✅ **850+ integrations** automatically discovered and configured
352
+- ✅ **Zero configuration** required - monitors start collecting data immediately
353
+- ✅ **No vendor lock-in** - Deploy anywhere, own your data
354
+- ✅ **1-second resolution** - Real-time visibility, not delayed averages
355
+- ✅ **Flexible deployment** - On-premise, cloud, or hybrid
356
+
357
+{tech_nav}
358
+
359
+{generic_section}
360
+
361
+"""
362
22
-def recursive(categories):
23
- for category in categories:
24
- data_col_cat[category['id']] = category['name']
25
- if category['children']:
26
- recursive(category['children'])
363
364
+def _render_tables(sections: List[Tuple[str, List[Tuple[str, str, str]]]]) -> str:
365
+ lines: List[str] = []
366
+ for title, items in sections:
367
+ if not items:
368
+ continue
369
29
-recursive(categories[1]['children'])
370
+ # Convert title to anchor-compatible format
371
+ anchor = title.lower().replace(' ', '-').replace('/', '-').replace('(', '').replace(')', '')
372
+ lines.append(f"### {title}\n\n")
373
+ lines.append("| Integration | Description |\n|-------------|-------------|\n")
374
375
+ for name, link, desc in items:
376
+ desc = desc.replace('|', '\\|')
377
+ lines.append(f"| [{name}]({link}) | {desc} |\n")
378
+ lines.append("\n")
379
+ return ''.join(lines)
380
32
-def construct_dict(array, integration):
33
- for element in array:
34
- if element not in cat_dict:
35
- cat_dict[element] = list()
36
- cat_dict[element].append(integration)
381
382
+def main() -> None:
383
+ categories, integrations = _load_catalog()
384
+ sections = _collect_sections(categories, integrations)
385
39
-md = ""
386
+ header = _render_header()
387
+ tables = _render_tables(sections)
388
+ md = header + "## Available Data Collection Integrations\n\n" + tables
389
41
-for integration in integrations:
42
- if integration['integration_type'] == "collector":
43
- construct_dict(integration['meta']['monitored_instance']['categories'], integration)
390
+ outfile = pathlib.Path("./src/collectors/COLLECTORS.md")
391
+ txt = outfile.read_text(encoding='utf-8')
392
45
-for category_id, integrations in sorted(cat_dict.items()):
46
- heading = '#' * len(category_id.split('.'))
393
+ # Find the start of the content to replace
394
+ if "## Available Data Collection Integrations" in txt:
395
+ pre = txt.split("## Available Data Collection Integrations")[0]
396
+ elif "# Monitor anything with Netdata" in txt:
397
+ # If the header exists, keep only what's before it
398
+ pre = txt.split("# Monitor anything with Netdata")[0]
399
+ else:
400
+ # Otherwise keep everything before the marker
401
+ pre = txt.split("## Add your application to Netdata")[0] if "## Add your application to Netdata" in txt else ""
402
48
- for cat in data_col_cat:
49
- if cat == category_id:
50
- name = data_col_cat[cat]
403
+ new_txt = pre.rstrip() + "\n\n" + md
404
+ outfile.write_text(new_txt.rstrip('\n') + "\n", encoding='utf-8')
405
52
- md += f'#{heading} {name}\n\n'
53
- names = []
54
- for integration in integrations:
55
- name = integration['meta']['monitored_instance']['name']
56
- link = (integration['edit_link'].replace("metadata.yaml", "") +
57
- "integrations/" + name.lower().
58
- replace(" ", "_").
59
- replace("/", "-").
60
- replace("(", "").
61
- replace(")", "") + ".md")
62
- names.append(f"[{name}]({link})")
63
- for integration_name in sorted(names):
64
- md += "- " + integration_name + "\n\n"
406
66
-outfile = pathlib.Path("./src/collectors/COLLECTORS.md")
67
-output = outfile.read_text().split("## Available Data Collection Integrations")[0]
68
-output += "## Available Data Collection Integrations\n<!-- AUTOGENERATED PART BY integrations/gen_doc_collector_page.py SCRIPT, DO NOT EDIT MANUALLY -->\n" + md
69
-outfile.write_text(output.rstrip('\n') + "\n")
407
+if __name__ == '__main__':
408
+ main()