docs: improve COLLECTORS.md generation (#21233)
Ilya Mashchenko committed
Oct 28, 2025 at 20:55 UTC
8f654c87eaf944c705ec631d6be30c343d0dabac
1 file changed
+365
-187
integrations/gen_doc_collector_page.py
+365
-187
@@ -1,12 +1,11 @@
1
"""
2
Generate the integrations section in COLLECTORS.md from integrations/integrations.js
3
-Key behavior:
4
-- Use **children of 'data-collection' (second-level categories) as section headings**. Any child category IDs on integrations
5
- are rolled up to their section-level parent.
6
-- Read categories from meta.monitored_instance.categories (array of strings).
7
-- If an integration has no categories, assign categories where any ancestor has collector_default=true,
8
- rolled up to its section-level parent(s).
9
-- Render Markdown tables: | Integration | Description |.
3
+
4
+This script:
5
+- Reads category tree and integrations from integrations.js
6
+- Uses second-level categories (children of 'data-collection') as section headings
7
+- Groups integrations by their section-level category
8
+- Generates markdown tables with integration name, link, and description
9
"""
10
11
from __future__ import annotations
@@ -14,185 +13,257 @@ from __future__ import annotations
13
import json
14
import pathlib
15
import re
17
-from typing import Any, Dict, List, Tuple, Iterable, Optional
16
+from typing import Any, Dict, List, Optional, Tuple
17
+
18
19
+# =============================================================================
20
+# Data Loading
21
+# =============================================================================
22
20
-def _extract_json_blobs(js_text: str) -> Tuple[str, str]:
23
+def _extract_json_from_js(js_text: str) -> Tuple[str, str]:
24
+ """Extract categories and integrations JSON from JavaScript file."""
25
after_categories = js_text.split("export const categories = ", 1)[1]
26
categories_str, after_integrations = after_categories.split("export const integrations = ", 1)
27
integrations_str = after_integrations
28
25
- def _cleanup(s: str) -> str:
29
+ def cleanup(s: str) -> str:
30
+ """Remove export statements and trailing semicolons."""
31
s = re.split(r"\n\s*export const|\Z", s, maxsplit=1)[0].strip()
27
- if s.endswith(';'):
28
- s = s[:-1]
29
- return s.strip()
32
+ return s.rstrip(';').strip()
33
31
- return _cleanup(categories_str), _cleanup(integrations_str)
34
+ return cleanup(categories_str), cleanup(integrations_str)
35
36
34
-def _load_catalog(js_path: str = 'integrations/integrations.js'):
37
+def load_catalog(js_path: str = 'integrations/integrations.js') -> Tuple[List[Dict], Dict]:
38
+ """Load and parse categories and integrations from JavaScript file."""
39
with open(js_path, 'r', encoding='utf-8') as f:
40
js_data = f.read()
37
- categories_str, integrations_str = _extract_json_blobs(js_data)
38
- categories = json.loads(categories_str) # expected array tree
39
- integrations = json.loads(integrations_str) # expected dict map
41
+
42
+ categories_str, integrations_str = _extract_json_from_js(js_data)
43
+ categories = json.loads(categories_str)
44
+ integrations = json.loads(integrations_str)
45
+
46
return categories, integrations
47
48
43
-def _build_category_maps(categories: List[Dict[str, Any]]):
44
- """Build maps: id->title, id->parent_id, section_level list (ordered), and defaults rolled to section-level.
49
+# =============================================================================
50
+# Category Processing
51
+# =============================================================================
52
46
- We use the children of 'data-collection' as section headings (second-level categories).
53
+class CategoryMapper:
54
+ """Maps category IDs to titles, parents, and section-level ancestors."""
55
48
- """
49
- id_to_parent: Dict[str, Optional[str]] = {}
50
- id_to_title: Dict[str, str] = {}
51
- section_level_ids: List[str] = [] # Second-level categories under data-collection
52
- default_ids: List[str] = [] # category ids where collector_default==true
53
-
54
- def walk(nodes: Iterable[Dict[str, Any]], parent: Optional[str], depth: int = 0):
55
- for node in nodes or []:
56
- if not isinstance(node, dict):
57
- continue
58
- cid = node.get('id')
59
- title = node.get('name') or node.get('title') or (cid or '')
60
- if not cid:
61
- continue
62
- id_to_parent[cid] = parent
63
- id_to_title[cid] = title
64
-
65
- # If this is a child of 'data-collection', it's a section heading
66
- if parent == 'data-collection':
67
- section_level_ids.append(cid)
56
+ def __init__(self, categories: List[Dict[str, Any]]):
57
+ self.id_to_parent: Dict[str, Optional[str]] = {}
58
+ self.id_to_title: Dict[str, str] = {}
59
+ self.section_level_ids: List[str] = [] # Children of 'data-collection'
60
+ self.default_section_ids: List[str] = [] # Section IDs with collector_default=true
61
+
62
+ self._build_maps(categories)
63
+
64
+ def _build_maps(self, categories: List[Dict[str, Any]]) -> None:
65
+ """Build internal mappings by walking the category tree."""
66
+ default_ids = []
67
+
68
+ def walk(nodes: List[Dict[str, Any]], parent: Optional[str]) -> None:
69
+ for node in nodes or []:
70
+ if not isinstance(node, dict):
71
+ continue
72
69
- if node.get('collector_default') is True:
70
- default_ids.append(cid)
73
+ cid = node.get('id')
74
+ if not cid:
75
+ continue
76
72
- children = node.get('children') or []
73
- if isinstance(children, list) and children:
74
- walk(children, cid, depth + 1)
77
+ title = node.get('name') or node.get('title') or cid
78
+ self.id_to_parent[cid] = parent
79
+ self.id_to_title[cid] = title
80
76
- walk(categories if isinstance(categories, list) else [], parent=None, depth=0)
81
+ # Track section-level categories (children of 'data-collection')
82
+ if parent == 'data-collection':
83
+ self.section_level_ids.append(cid)
84
78
- # Find section-level ancestor (child of data-collection)
79
- def section_ancestor(cid: str) -> Optional[str]:
80
- # Walk up until we find a category whose parent is 'data-collection'
85
+ # Track categories with collector_default=true
86
+ if node.get('collector_default') is True:
87
+ default_ids.append(cid)
88
+
89
+ # Recurse into children
90
+ children = node.get('children', [])
91
+ if isinstance(children, list):
92
+ walk(children, cid)
93
+
94
+ walk(categories, parent=None)
95
+
96
+ # Roll up default IDs to their section-level ancestors
97
+ for cid in default_ids:
98
+ section = self.get_section_ancestor(cid)
99
+ if section and section not in self.default_section_ids:
100
+ self.default_section_ids.append(section)
101
+
102
+ def get_section_ancestor(self, cid: str) -> Optional[str]:
103
+ """Find the section-level ancestor (child of 'data-collection') for a category."""
104
cur = cid
105
seen = set()
83
- while cur is not None and cur in id_to_parent and cur not in seen:
106
+
107
+ while cur and cur in self.id_to_parent and cur not in seen:
108
seen.add(cur)
85
- parent = id_to_parent.get(cur)
109
+ parent = self.id_to_parent.get(cur)
110
+
111
if parent == 'data-collection':
112
return cur
113
if parent is None:
89
- # If we reach the top without finding data-collection, this might be a top-level category
114
return None
115
+
116
cur = parent
117
+
118
+ return None
119
+
120
+
121
+# =============================================================================
122
+# Text Processing
123
+# =============================================================================
124
+
125
+def extract_first_sentence(text: str) -> str:
126
+ """Extract the first sentence from text (up to the first period)."""
127
+ if not text:
128
+ return text
129
+
130
+ # Match first sentence ending with period followed by space/newline
131
+ match = re.match(r'^(.*?\.)\s', text)
132
+ if match:
133
+ return match.group(1).strip()
134
+
135
+ # If text ends with period, use all of it
136
+ if text.endswith('.'):
137
+ return text.strip()
138
+
139
+ # No period found - use all text
140
+ return text.strip()
141
+
142
+
143
+def extract_description_from_overview(overview: str) -> Optional[str]:
144
+ """Extract first substantial paragraph from markdown overview section."""
145
+ # Split by ## Overview heading
146
+ parts = overview.split('## Overview', 1)
147
+ if len(parts) <= 1:
148
+ return None
149
+
150
+ # Get text after ## Overview
151
+ text_after = parts[1].strip()
152
+
153
+ # Split into lines and find first substantial paragraph
154
+ lines = text_after.split('\n')
155
+ paragraph = []
156
+
157
+ for line in lines:
158
+ line = line.strip()
159
+
160
+ # Skip empty lines at start
161
+ if not line and not paragraph:
162
+ continue
163
+
164
+ # Skip metadata lines (Plugin:, Module:, headings)
165
+ if line.startswith(('#', 'Plugin:', 'Module:')):
166
+ if paragraph: # Stop if we already have content
167
+ break
168
+ continue
169
+
170
+ # Collect paragraph lines
171
+ if line:
172
+ paragraph.append(line)
173
+ elif paragraph: # Empty line after content = end of paragraph
174
+ break
175
+
176
+ if not paragraph:
177
return None
178
94
- # Roll default ids to their section-level parents (unique)
95
- default_section_level = []
96
- for d in default_ids:
97
- section = section_ancestor(d)
98
- if section and section not in default_section_level:
99
- default_section_level.append(section)
179
+ text = ' '.join(paragraph)
180
+ first_sentence = extract_first_sentence(text)
181
101
- return id_to_title, id_to_parent, section_level_ids, default_section_level, section_ancestor
182
+ # Normalize whitespace
183
+ return re.sub(r'\s+', ' ', first_sentence) if first_sentence else None
184
185
104
-def _desc_for_integration(integ: Dict[str, Any]) -> str:
105
- """Generate user-friendly description for integration."""
186
+def get_integration_description(integ: Dict[str, Any]) -> str:
187
+ """Get user-friendly description for an integration."""
188
+ # Try overview markdown first
189
overview = integ.get('overview')
107
- if isinstance(overview, dict):
108
- dc = overview.get('data_collection')
109
- if isinstance(dc, dict):
110
- md = dc.get('metrics_description')
111
- if isinstance(md, str) and md.strip():
112
- return re.sub(r"\s+", " ", md.strip())
190
+ if isinstance(overview, str) and overview.strip():
191
+ desc = extract_description_from_overview(overview)
192
+ if desc:
193
+ return desc
194
+
195
+ # Fallback to monitored_instance.description
196
mi = integ.get('meta', {}).get('monitored_instance', {})
197
if isinstance(mi, dict):
115
- mi_desc = mi.get('description')
116
- if isinstance(mi_desc, str) and mi_desc.strip():
117
- return re.sub(r"\s+", " ", mi_desc.strip())
118
- for key in ('short_description', 'description', 'summary'):
119
- v = integ.get(key)
120
- if isinstance(v, str) and v.strip():
121
- return re.sub(r"\s+", " ", v.strip())
122
- name = (mi.get('name') if isinstance(mi, dict) else None) or integ.get('name') or 'Integration'
123
- return f"Metrics for {name}"
124
-
125
-
126
-def _to_slug(text: str) -> str:
127
- """Convert a string to a slug suitable for URLs and anchors."""
198
+ desc = mi.get('description')
199
+ if isinstance(desc, str) and desc.strip():
200
+ first_sentence = extract_first_sentence(desc.strip())
201
+ if first_sentence:
202
+ return re.sub(r'\s+', ' ', first_sentence)
203
+
204
+ # Generic fallback
205
+ name = (mi.get('name') if isinstance(mi, dict) else None) or integ.get('name') or 'this integration'
206
+ return f"Monitor {name}"
207
+
208
+
209
+# =============================================================================
210
+# Link Generation
211
+# =============================================================================
212
+
213
+def to_slug(text: str) -> str:
214
+ """Convert text to URL-friendly slug."""
215
return text.lower().replace(' ', '_').replace('/', '-').replace('(', '').replace(')', '')
216
217
131
-def _doc_link(integ: Dict[str, Any], display_name: str) -> str:
132
- base = (integ.get('edit_link') or '') if isinstance(integ, dict) else ''
133
- base = base.replace('metadata.yaml', '')
134
- slug = _to_slug(display_name)
135
- return f"{base}integrations/{slug}.md"
218
+def get_integration_doc_link(integ: Dict[str, Any], display_name: str) -> str:
219
+ """Generate documentation link for an integration.
220
221
+ Example:
222
+ edit_link: .../collector/rspamd/metadata.yaml
223
+ output: .../collector/rspamd/integrations/rspamd.md
224
+ """
225
+ edit_link = integ.get('edit_link', '') if isinstance(integ, dict) else ''
226
+ if not edit_link:
227
+ return ''
228
138
-def _collect_sections(categories: List[Dict[str, Any]], integrations: Dict[str, Any]):
139
- id_to_title, id_to_parent, section_level_ids, default_section_level, section_ancestor = _build_category_maps(
140
- categories)
229
+ base = edit_link.replace('metadata.yaml', '')
230
+ slug = to_slug(display_name)
231
142
- # Custom section order: Linux Systems first, then others, but exclude "Other" category
143
- ordered_sections = []
144
- linux_id = None
145
- other_id = None
232
+ return f"{base}integrations/{slug}.md"
233
147
- for cid in section_level_ids:
148
- if 'linux-systems' in cid.lower():
149
- linux_id = cid
150
- elif id_to_title.get(cid, '').lower() == 'other':
151
- other_id = cid
234
153
- if linux_id:
154
- ordered_sections.append(linux_id)
235
+# =============================================================================
236
+# Section Collection
237
+# =============================================================================
238
156
- for cid in section_level_ids:
157
- if cid != linux_id and cid != other_id:
158
- ordered_sections.append(cid)
239
+def collect_integrations_by_section(
240
+ categories: List[Dict[str, Any]],
241
+ integrations: Dict[str, Any]
242
+) -> List[Tuple[str, List[Tuple[str, str, str]]]]:
243
+ """Group integrations by their section-level category.
244
+
245
+ Returns:
246
+ List of (section_title, [(name, link, description), ...])
247
+ """
248
+ mapper = CategoryMapper(categories)
249
160
- # Prepare buckets for section-level categories
161
- per_section: Dict[str, List[Tuple[str, str, str]]] = {cid: [] for cid in ordered_sections}
250
+ # Determine section order (Linux first, Other last)
251
+ ordered_sections = _get_ordered_sections(mapper)
252
+
253
+ # Initialize buckets for each section
254
+ per_section = {cid: [] for cid in ordered_sections}
255
+ other_id = _find_other_category_id(mapper)
256
if other_id:
257
per_section[other_id] = []
164
- other_bucket: List[Tuple[str, str, str]] = []
258
+ other_bucket = []
259
166
- items = integrations.items() if isinstance(integrations, dict) else enumerate(
167
- integrations if isinstance(integrations, list) else [])
168
- for _key, integ in items:
169
- if not isinstance(integ, dict):
170
- continue
171
- mi = integ.get('meta', {}).get('monitored_instance', {})
172
- name = (mi.get('name') if isinstance(mi, dict) else None) or integ.get('name')
173
- if not isinstance(name, str) or not name.strip():
260
+ # Process each integration
261
+ for integ in _iterate_integrations(integrations):
262
+ entry = _process_integration(integ, mapper, ordered_sections, other_id)
263
+ if not entry:
264
continue
175
- link = _doc_link(integ, name)
176
- desc = _desc_for_integration(integ)
265
178
- cats = []
179
- if isinstance(mi, dict):
180
- cats = mi.get('categories') or []
181
- if isinstance(cats, str):
182
- cats = [cats]
183
- if not isinstance(cats, list):
184
- cats = []
185
-
186
- if not cats:
187
- # use default section-level categories
188
- target_sections = list(default_section_level) if default_section_level else []
189
- else:
190
- # roll each category id to its section-level ancestor
191
- target_sections = []
192
- for cid in cats:
193
- section = section_ancestor(cid) if isinstance(cid, str) else None
194
- if section and (section in per_section or section == other_id) and section not in target_sections:
195
- target_sections.append(section)
266
+ name, link, desc, target_sections = entry
267
268
if not target_sections:
269
other_bucket.append((name, link, desc))
@@ -201,19 +272,116 @@ def _collect_sections(categories: List[Dict[str, Any]], integrations: Dict[str,
272
if sec in per_section or sec == other_id:
273
per_section[sec].append((name, link, desc))
274
204
- # Build ordered sections with non-empty tables
205
- sections: List[Tuple[str, List[Tuple[str, str, str]]]] = []
275
+ # Build final section list
276
+ return _build_section_list(mapper, ordered_sections, per_section, other_id, other_bucket)
277
+
278
+
279
+def _get_ordered_sections(mapper: CategoryMapper) -> List[str]:
280
+ """Get section IDs in order: Linux first, others alphabetically, Other excluded."""
281
+ linux_id = None
282
+ other_sections = []
283
+
284
+ for cid in mapper.section_level_ids:
285
+ if 'linux-systems' in cid.lower():
286
+ linux_id = cid
287
+ elif mapper.id_to_title.get(cid, '').lower() != 'other':
288
+ other_sections.append(cid)
289
+
290
+ ordered = []
291
+ if linux_id:
292
+ ordered.append(linux_id)
293
+ ordered.extend(other_sections)
294
+
295
+ return ordered
296
+
297
+
298
+def _find_other_category_id(mapper: CategoryMapper) -> Optional[str]:
299
+ """Find the 'Other' category ID if it exists."""
300
+ for cid in mapper.section_level_ids:
301
+ if mapper.id_to_title.get(cid, '').lower() == 'other':
302
+ return cid
303
+ return None
304
+
305
+
306
+def _iterate_integrations(integrations: Any):
307
+ """Yield integration objects from dict or list."""
308
+ if isinstance(integrations, dict):
309
+ for integ in integrations.values():
310
+ if isinstance(integ, dict):
311
+ yield integ
312
+ elif isinstance(integrations, list):
313
+ for integ in integrations:
314
+ if isinstance(integ, dict):
315
+ yield integ
316
+
317
+
318
+def _process_integration(
319
+ integ: Dict[str, Any],
320
+ mapper: CategoryMapper,
321
+ ordered_sections: List[str],
322
+ other_id: Optional[str]
323
+) -> Optional[Tuple[str, str, str, List[str]]]:
324
+ """Process a single integration and determine its target sections.
325
+
326
+ Returns:
327
+ (name, link, description, target_section_ids) or None if invalid
328
+ """
329
+ # Get integration name
330
+ mi = integ.get('meta', {}).get('monitored_instance', {})
331
+ name = (mi.get('name') if isinstance(mi, dict) else None) or integ.get('name')
332
+ if not isinstance(name, str) or not name.strip():
333
+ return None
334
+
335
+ # Generate link and description
336
+ link = get_integration_doc_link(integ, name)
337
+ desc = get_integration_description(integ)
338
+
339
+ # Get categories
340
+ cats = mi.get('categories') if isinstance(mi, dict) else None
341
+ if isinstance(cats, str):
342
+ cats = [cats]
343
+ if not isinstance(cats, list):
344
+ cats = []
345
+
346
+ # Determine target sections
347
+ if not cats:
348
+ # Use default sections
349
+ target_sections = list(mapper.default_section_ids)
350
+ else:
351
+ # Roll up each category to its section-level ancestor
352
+ target_sections = []
353
+ for cid in cats:
354
+ section = mapper.get_section_ancestor(cid) if isinstance(cid, str) else None
355
+ if section and section not in target_sections:
356
+ # Only include if it's in our ordered sections or is the other_id
357
+ if section in ordered_sections or section == other_id:
358
+ target_sections.append(section)
359
+
360
+ return (name, link, desc, target_sections)
361
+
362
+
363
+def _build_section_list(
364
+ mapper: CategoryMapper,
365
+ ordered_sections: List[str],
366
+ per_section: Dict[str, List[Tuple[str, str, str]]],
367
+ other_id: Optional[str],
368
+ other_bucket: List[Tuple[str, str, str]]
369
+) -> List[Tuple[str, List[Tuple[str, str, str]]]]:
370
+ """Build final list of sections with their sorted integrations."""
371
+ sections = []
372
+
373
+ # Add ordered sections
374
for cid in ordered_sections:
375
items = per_section.get(cid, [])
376
if items:
377
items.sort(key=lambda t: t[0].lower())
210
- sections.append((id_to_title.get(cid, cid), items))
378
+ sections.append((mapper.id_to_title.get(cid, cid), items))
379
212
- # Add "Other" section last - either from other_bucket or from the Other category
380
+ # Add "Other" section last
381
if other_id and per_section.get(other_id):
382
items = per_section[other_id]
383
items.sort(key=lambda t: t[0].lower())
216
- sections.append((id_to_title.get(other_id, "Other"), items))
384
+ sections.append((mapper.id_to_title.get(other_id, "Other"), items))
385
elif other_bucket:
386
other_bucket.sort(key=lambda t: t[0].lower())
387
sections.append(("Other", other_bucket))
@@ -221,9 +389,40 @@ def _collect_sections(categories: List[Dict[str, Any]], integrations: Dict[str,
389
return sections
390
391
392
+# =============================================================================
393
+# Markdown Rendering
394
+# =============================================================================
395
+
396
+def render_header() -> str:
397
+ """Render the header section with marketing content and navigation."""
398
+ tech_nav = _render_tech_navigation()
399
+ generic_section = _render_generic_collectors()
400
+
401
+ return f"""# Monitor anything with Netdata
402
+
403
+**850+ integrations. Zero configuration. Deploy anywhere.**
404
+
405
+Netdata uses collectors to help you gather metrics from your favorite applications and services and view them in real-time, interactive charts. The following list includes all the integrations where Netdata can gather metrics from.
406
+
407
+Learn more about [how collectors work](/src/collectors/README.md), and then learn how to [enable or configure](/src/collectors/REFERENCE.md#enable-or-disable-collectors-and-plugins) a specific collector.
408
+
409
+### Why Teams Choose Us
410
+
411
+- ✅ **850+ integrations** automatically discovered and configured
412
+- ✅ **Zero configuration** required - monitors start collecting data immediately
413
+- ✅ **No vendor lock-in** - Deploy anywhere, own your data
414
+- ✅ **1-second resolution** - Real-time visibility, not delayed averages
415
+- ✅ **Flexible deployment** - On-premise, cloud, or hybrid
416
+
417
+{tech_nav}
418
+
419
+{generic_section}
420
+
421
+"""
422
+
423
+
424
def _render_tech_navigation() -> str:
225
- """Render the 'Find Your Technology' navigation section."""
226
- # Configuration for "Find Your Technology" section
425
+ """Render the 'Find Your Technology' quick navigation section."""
426
tech_categories = [
427
{
428
"title": "Cloud & Infrastructure:",
@@ -278,7 +477,6 @@ def _render_tech_navigation() -> str:
477
},
478
]
479
281
- # Build "Find Your Technology" section
480
tech_lines = []
481
for category in tech_categories:
482
links = " • ".join([f"[{name}]({anchor})" for name, anchor in category["items"]])
@@ -296,8 +494,7 @@ def _render_tech_navigation() -> str:
494
495
496
def _render_generic_collectors() -> str:
299
- """Render the 'Beyond the 850+ integrations' section with generic collectors."""
300
- # Configuration for generic collectors
497
+ """Render the 'Beyond the 850+ integrations' section."""
498
generic_collectors = [
499
{
500
"name": "Prometheus collector",
@@ -316,7 +513,6 @@ def _render_generic_collectors() -> str:
513
},
514
]
515
319
- # Build generic collectors section
516
collector_lines = []
517
for collector in generic_collectors:
518
collector_lines.append(f"- **[{collector['name']}]({collector['link']})** - {collector['description']}")
@@ -333,70 +529,52 @@ Need a dedicated integration? [Submit a feature request](https://github.com/netd
529
"""
530
531
336
-def _render_header() -> str:
337
- """Render the marketing header and navigation sections."""
338
- tech_nav = _render_tech_navigation()
339
- generic_section = _render_generic_collectors()
340
-
341
- return f"""# Monitor anything with Netdata
342
-
343
-**850+ integrations. Zero configuration. Deploy anywhere.**
532
+def render_tables(sections: List[Tuple[str, List[Tuple[str, str, str]]]]) -> str:
533
+ """Render markdown tables for all sections."""
534
+ lines = []
535
345
-Netdata uses collectors to help you gather metrics from your favorite applications and services and view them in real-time, interactive charts. The following list includes all the integrations where Netdata can gather metrics from.
346
-
347
-Learn more about [how collectors work](/src/collectors/README.md), and then learn how to [enable or configure](/src/collectors/REFERENCE.md#enable-or-disable-collectors-and-plugins) a specific collector.
348
-
349
-### Why Teams Choose Us
350
-
351
-- ✅ **850+ integrations** automatically discovered and configured
352
-- ✅ **Zero configuration** required - monitors start collecting data immediately
353
-- ✅ **No vendor lock-in** - Deploy anywhere, own your data
354
-- ✅ **1-second resolution** - Real-time visibility, not delayed averages
355
-- ✅ **Flexible deployment** - On-premise, cloud, or hybrid
356
-
357
-{tech_nav}
358
-
359
-{generic_section}
360
-
361
-"""
362
-
363
-
364
-def _render_tables(sections: List[Tuple[str, List[Tuple[str, str, str]]]]) -> str:
365
- lines: List[str] = []
536
for title, items in sections:
537
if not items:
538
continue
539
370
- # Convert title to anchor-compatible format
371
- anchor = title.lower().replace(' ', '-').replace('/', '-').replace('(', '').replace(')', '')
540
lines.append(f"### {title}\n\n")
541
lines.append("| Integration | Description |\n|-------------|-------------|\n")
542
543
for name, link, desc in items:
544
+ # Escape pipe characters in description
545
desc = desc.replace('|', '\\|')
546
lines.append(f"| [{name}]({link}) | {desc} |\n")
547
+
548
lines.append("\n")
549
+
550
return ''.join(lines)
551
552
382
-def main() -> None:
383
- categories, integrations = _load_catalog()
384
- sections = _collect_sections(categories, integrations)
553
+# =============================================================================
554
+# Main Execution
555
+# =============================================================================
556
386
- header = _render_header()
387
- tables = _render_tables(sections)
388
- md = header + "## Available Data Collection Integrations\n\n" + tables
557
+def generate_collectors_md() -> None:
558
+ """Generate COLLECTORS.md from integrations.js."""
559
+ # Load data
560
+ categories, integrations = load_catalog()
561
390
- outfile = pathlib.Path("./src/collectors/COLLECTORS.md")
562
+ # Process integrations
563
+ sections = collect_integrations_by_section(categories, integrations)
564
565
+ # Render markdown
566
+ header = render_header()
567
+ tables = render_tables(sections)
568
+ content = header + "## Available Data Collection Integrations\n\n" + tables
569
+
570
+ # Write to file atomically
571
+ outfile = pathlib.Path("./src/collectors/COLLECTORS.md")
572
outfile.parent.mkdir(parents=True, exist_ok=True)
573
394
- # Always overwrite the file with freshly generated content (no partial preserves)
395
- # Write atomically to avoid partial writes
574
tmp = outfile.with_suffix(outfile.suffix + ".tmp")
397
- tmp.write_text(md.rstrip('\n') + "\n", encoding='utf-8')
575
+ tmp.write_text(content.rstrip('\n') + "\n", encoding='utf-8')
576
tmp.replace(outfile)
577
578
579
if __name__ == '__main__':
402
- main()
580
+ generate_collectors_md()