master
py 588 lines 20 KB
Raw
1 """
2 Generate the integrations section in COLLECTORS.md from integrations/integrations.js
3
4 This script:
5 - Reads category tree and integrations from integrations.js
6 - Uses data-collection section categories, plus the top-level flows category,
7 as section headings
8 - Groups integrations by their section-level category
9 - Generates markdown tables with integration name, link, and description
10 """
11
12 from __future__ import annotations
13
14 import json
15 import pathlib
16 import re
17 from typing import Any, Dict, List, Optional, Tuple
18
19
20 # =============================================================================
21 # Data Loading
22 # =============================================================================
23
24 def _extract_json_from_js(js_text: str) -> Tuple[str, str]:
25 """Extract categories and integrations JSON from JavaScript file."""
26 after_categories = js_text.split("export const categories = ", 1)[1]
27 categories_str, after_integrations = after_categories.split("export const integrations = ", 1)
28 integrations_str = after_integrations
29
30 def cleanup(s: str) -> str:
31 """Remove export statements and trailing semicolons."""
32 s = re.split(r"\n\s*export const|\Z", s, maxsplit=1)[0].strip()
33 return s.rstrip(';').strip()
34
35 return cleanup(categories_str), cleanup(integrations_str)
36
37
38 def load_catalog(js_path: str = 'integrations/integrations.js') -> Tuple[List[Dict], Dict]:
39 """Load and parse categories and integrations from JavaScript file."""
40 with open(js_path, 'r', encoding='utf-8') as f:
41 js_data = f.read()
42
43 categories_str, integrations_str = _extract_json_from_js(js_data)
44 categories = json.loads(categories_str)
45 integrations = json.loads(integrations_str)
46
47 return categories, integrations
48
49
50 # =============================================================================
51 # Category Processing
52 # =============================================================================
53
54 class CategoryMapper:
55 """Maps category IDs to titles, parents, and section-level ancestors."""
56
57 def __init__(self, categories: List[Dict[str, Any]]):
58 self.id_to_parent: Dict[str, Optional[str]] = {}
59 self.id_to_title: Dict[str, str] = {}
60 self.section_level_ids: List[str] = [] # Monitor Anything section IDs
61 self.default_section_ids: List[str] = [] # Section IDs with collector_default=true
62
63 self._build_maps(categories)
64
65 def _build_maps(self, categories: List[Dict[str, Any]]) -> None:
66 """Build internal mappings by walking the category tree."""
67 default_ids = []
68
69 def walk(nodes: List[Dict[str, Any]], parent: Optional[str]) -> None:
70 for node in nodes or []:
71 if not isinstance(node, dict):
72 continue
73
74 cid = node.get('id')
75 if not cid:
76 continue
77
78 title = node.get('name') or node.get('title') or cid
79 self.id_to_parent[cid] = parent
80 self.id_to_title[cid] = title
81
82 # Track Monitor Anything sections. Most are children of
83 # data-collection; Network Flows is a top-level integrations
84 # category because it includes protocols and enrichment inputs.
85 if parent == 'data-collection' or (parent is None and cid == 'flows'):
86 self.section_level_ids.append(cid)
87
88 # Track categories with collector_default=true
89 if node.get('collector_default') is True:
90 default_ids.append(cid)
91
92 # Recurse into children
93 children = node.get('children', [])
94 if isinstance(children, list):
95 walk(children, cid)
96
97 walk(categories, parent=None)
98
99 # Roll up default IDs to their section-level ancestors
100 for cid in default_ids:
101 section = self.get_section_ancestor(cid)
102 if section and section not in self.default_section_ids:
103 self.default_section_ids.append(section)
104
105 def get_section_ancestor(self, cid: str) -> Optional[str]:
106 """Find the Monitor Anything section ancestor for a category."""
107 cur = cid
108 seen = set()
109
110 while cur and cur in self.id_to_parent and cur not in seen:
111 seen.add(cur)
112 if cur in self.section_level_ids:
113 return cur
114
115 parent = self.id_to_parent.get(cur)
116
117 if parent == 'data-collection':
118 return cur
119 if parent is None:
120 return None
121
122 cur = parent
123
124 return None
125
126
127 # =============================================================================
128 # Text Processing
129 # =============================================================================
130
131 def extract_first_sentence(text: str) -> str:
132 """Extract the first sentence from text (up to the first period)."""
133 if not text:
134 return text
135
136 # Match first sentence ending with period followed by space/newline
137 match = re.match(r'^(.*?\.)\s', text)
138 if match:
139 return match.group(1).strip()
140
141 # If text ends with period, use all of it
142 if text.endswith('.'):
143 return text.strip()
144
145 # No period found - use all text
146 return text.strip()
147
148
149 def extract_description_from_overview(overview: str) -> Optional[str]:
150 """Extract first substantial paragraph from markdown overview section."""
151 # Split by ## Overview heading
152 parts = overview.split('## Overview', 1)
153 if len(parts) <= 1:
154 return None
155
156 # Get text after ## Overview
157 text_after = parts[1].strip()
158
159 # Split into lines and find first substantial paragraph
160 lines = text_after.split('\n')
161 paragraph = []
162
163 for line in lines:
164 line = line.strip()
165
166 # Skip empty lines at start
167 if not line and not paragraph:
168 continue
169
170 # Skip metadata lines (Plugin:, Module:, headings)
171 if line.startswith(('#', 'Plugin:', 'Module:')):
172 if paragraph: # Stop if we already have content
173 break
174 continue
175
176 # Collect paragraph lines
177 if line:
178 paragraph.append(line)
179 elif paragraph: # Empty line after content = end of paragraph
180 break
181
182 if not paragraph:
183 return None
184
185 text = ' '.join(paragraph)
186 first_sentence = extract_first_sentence(text)
187
188 # Normalize whitespace
189 return re.sub(r'\s+', ' ', first_sentence) if first_sentence else None
190
191
192 def get_integration_description(integ: Dict[str, Any]) -> str:
193 """Get user-friendly description for an integration."""
194 # Try overview markdown first
195 overview = integ.get('overview')
196 if isinstance(overview, str) and overview.strip():
197 desc = extract_description_from_overview(overview)
198 if desc:
199 return desc
200
201 # Fallback to monitored_instance.description
202 mi = integ.get('meta', {}).get('monitored_instance', {})
203 if isinstance(mi, dict):
204 desc = mi.get('description')
205 if isinstance(desc, str) and desc.strip():
206 first_sentence = extract_first_sentence(desc.strip())
207 if first_sentence:
208 return re.sub(r'\s+', ' ', first_sentence)
209
210 # Generic fallback
211 name = (mi.get('name') if isinstance(mi, dict) else None) or integ.get('name') or 'this integration'
212 return f"Monitor {name}"
213
214
215 # =============================================================================
216 # Link Generation
217 # =============================================================================
218
219 def to_slug(text: str) -> str:
220 """Convert text to URL-friendly slug."""
221 return text.lower().replace(' ', '_').replace('/', '-').replace('(', '').replace(')', '')
222
223
224 def get_integration_doc_link(integ: Dict[str, Any], display_name: str) -> str:
225 """Generate documentation link for an integration.
226
227 Example:
228 edit_link: .../collector/rspamd/metadata.yaml
229 output: .../collector/rspamd/integrations/rspamd.md
230 """
231 edit_link = integ.get('edit_link', '') if isinstance(integ, dict) else ''
232 if not edit_link:
233 return ''
234
235 base = edit_link.replace('metadata.yaml', '')
236 slug = to_slug(display_name)
237
238 return f"{base}integrations/{slug}.md"
239
240
241 # =============================================================================
242 # Section Collection
243 # =============================================================================
244
245 def collect_integrations_by_section(
246 categories: List[Dict[str, Any]],
247 integrations: Dict[str, Any]
248 ) -> List[Tuple[str, List[Tuple[str, str, str]]]]:
249 """Group integrations by their section-level category.
250
251 Returns:
252 List of (section_title, [(name, link, description), ...])
253 """
254 mapper = CategoryMapper(categories)
255
256 # Determine section order (Linux first, Other last)
257 ordered_sections = _get_ordered_sections(mapper)
258
259 # Initialize buckets for each section
260 per_section = {cid: [] for cid in ordered_sections}
261 other_id = _find_other_category_id(mapper)
262 if other_id:
263 per_section[other_id] = []
264 other_bucket = []
265
266 # Process each integration
267 for integ in _iterate_integrations(integrations):
268 entry = _process_integration(integ, mapper, ordered_sections, other_id)
269 if not entry:
270 continue
271
272 name, link, desc, target_sections = entry
273
274 if not target_sections:
275 other_bucket.append((name, link, desc))
276 else:
277 for sec in target_sections:
278 if sec in per_section or sec == other_id:
279 per_section[sec].append((name, link, desc))
280
281 # Build final section list
282 return _build_section_list(mapper, ordered_sections, per_section, other_id, other_bucket)
283
284
285 def _get_ordered_sections(mapper: CategoryMapper) -> List[str]:
286 """Get section IDs in order: Linux first, others alphabetically, Other excluded."""
287 linux_id = None
288 other_sections = []
289
290 for cid in mapper.section_level_ids:
291 if 'linux-systems' in cid.lower():
292 linux_id = cid
293 elif mapper.id_to_title.get(cid, '').lower() != 'other':
294 other_sections.append(cid)
295
296 ordered = []
297 if linux_id:
298 ordered.append(linux_id)
299 ordered.extend(other_sections)
300
301 return ordered
302
303
304 def _find_other_category_id(mapper: CategoryMapper) -> Optional[str]:
305 """Find the 'Other' category ID if it exists."""
306 for cid in mapper.section_level_ids:
307 if mapper.id_to_title.get(cid, '').lower() == 'other':
308 return cid
309 return None
310
311
312 def _iterate_integrations(integrations: Any):
313 """Yield integration objects from dict or list."""
314 if isinstance(integrations, dict):
315 for integ in integrations.values():
316 if isinstance(integ, dict):
317 yield integ
318 elif isinstance(integrations, list):
319 for integ in integrations:
320 if isinstance(integ, dict):
321 yield integ
322
323
324 def _process_integration(
325 integ: Dict[str, Any],
326 mapper: CategoryMapper,
327 ordered_sections: List[str],
328 other_id: Optional[str]
329 ) -> Optional[Tuple[str, str, str, List[str]]]:
330 """Process a single integration and determine its target sections.
331
332 Returns:
333 (name, link, description, target_section_ids) or None if invalid
334 """
335 # Get integration name
336 mi = integ.get('meta', {}).get('monitored_instance', {})
337 name = (mi.get('name') if isinstance(mi, dict) else None) or integ.get('name')
338 if not isinstance(name, str) or not name.strip():
339 return None
340
341 # Generate link and description
342 link = get_integration_doc_link(integ, name)
343 desc = get_integration_description(integ)
344
345 # Get categories
346 cats = mi.get('categories') if isinstance(mi, dict) else None
347 if isinstance(cats, str):
348 cats = [cats]
349 if not isinstance(cats, list):
350 cats = []
351
352 # Determine target sections
353 if not cats:
354 # Use default sections
355 target_sections = list(mapper.default_section_ids)
356 else:
357 # Roll up each category to its section-level ancestor
358 target_sections = []
359 for cid in cats:
360 section = mapper.get_section_ancestor(cid) if isinstance(cid, str) else None
361 if section and section not in target_sections:
362 # Only include if it's in our ordered sections or is the other_id
363 if section in ordered_sections or section == other_id:
364 target_sections.append(section)
365
366 return (name, link, desc, target_sections)
367
368
369 def _build_section_list(
370 mapper: CategoryMapper,
371 ordered_sections: List[str],
372 per_section: Dict[str, List[Tuple[str, str, str]]],
373 other_id: Optional[str],
374 other_bucket: List[Tuple[str, str, str]]
375 ) -> List[Tuple[str, List[Tuple[str, str, str]]]]:
376 """Build final list of sections with their sorted integrations."""
377 sections = []
378
379 # Add ordered sections
380 for cid in ordered_sections:
381 items = per_section.get(cid, [])
382 if items:
383 items.sort(key=lambda t: t[0].lower())
384 sections.append((mapper.id_to_title.get(cid, cid), items))
385
386 # Add "Other" section last
387 if other_id and per_section.get(other_id):
388 items = per_section[other_id]
389 items.sort(key=lambda t: t[0].lower())
390 sections.append((mapper.id_to_title.get(other_id, "Other"), items))
391 elif other_bucket:
392 other_bucket.sort(key=lambda t: t[0].lower())
393 sections.append(("Other", other_bucket))
394
395 return sections
396
397
398 # =============================================================================
399 # Markdown Rendering
400 # =============================================================================
401
402 def render_header() -> str:
403 """Render the header section with marketing content and navigation."""
404 tech_nav = _render_tech_navigation()
405 generic_section = _render_generic_collectors()
406
407 return f"""<!-- markdownlint-disable-file -->
408
409 # Monitor anything with Netdata
410
411 **850+ integrations. Zero configuration. Deploy anywhere.**
412
413 Netdata uses collectors to help you gather metrics from your favorite applications and services and view them in real-time, interactive charts. The following list includes all the integrations where Netdata can gather metrics from.
414
415 Learn more about [how collectors work](/src/collectors/README.md), and then learn how to [enable or configure](/src/collectors/REFERENCE.md#enable-or-disable-collectors-and-plugins) a specific collector.
416
417 ### Why Teams Choose Us
418
419 - ✅ **850+ integrations** automatically discovered and configured
420 - ✅ **Zero configuration** required - monitors start collecting data immediately
421 - ✅ **No vendor lock-in** - Deploy anywhere, own your data
422 - ✅ **1-second resolution** - Real-time visibility, not delayed averages
423 - ✅ **Flexible deployment** - On-premise, cloud, or hybrid
424
425 {tech_nav}
426
427 {generic_section}
428
429 """
430
431
432 def _render_tech_navigation() -> str:
433 """Render the 'Find Your Technology' quick navigation section."""
434 tech_categories = [
435 {
436 "title": "Cloud & Infrastructure:",
437 "items": [
438 ("AWS", "#cloud-provider-managed"),
439 ("Azure", "#cloud-provider-managed"),
440 ("GCP", "#cloud-provider-managed"),
441 ("Kubernetes", "#kubernetes"),
442 ("Docker", "#containers-and-vms"),
443 ("VMware", "#containers-and-vms"),
444 ]
445 },
446 {
447 "title": "Databases & Caching:",
448 "items": [
449 ("MySQL", "#databases"),
450 ("PostgreSQL", "#databases"),
451 ("MongoDB", "#databases"),
452 ("Redis", "#databases"),
453 ("Elasticsearch", "#search-engines"),
454 ("Oracle", "#databases"),
455 ]
456 },
457 {
458 "title": "Web & Application:",
459 "items": [
460 ("NGINX", "#web-servers-and-web-proxies"),
461 ("Apache", "#web-servers-and-web-proxies"),
462 ("HAProxy", "#web-servers-and-web-proxies"),
463 ("Tomcat", "#web-servers-and-web-proxies"),
464 ("PHP-FPM", "#web-servers-and-web-proxies"),
465 ]
466 },
467 {
468 "title": "Message Queues:",
469 "items": [
470 ("Kafka", "#message-brokers"),
471 ("RabbitMQ", "#message-brokers"),
472 ("ActiveMQ", "#message-brokers"),
473 ("NATS", "#message-brokers"),
474 ("Pulsar", "#message-brokers"),
475 ]
476 },
477 {
478 "title": "Operating Systems:",
479 "items": [
480 ("Linux", "#linux-systems"),
481 ("Windows", "#windows-systems"),
482 ("macOS", "#macos-systems"),
483 ("FreeBSD", "#freebsd"),
484 ]
485 },
486 ]
487
488 tech_lines = []
489 for category in tech_categories:
490 links = "".join([f"[{name}]({anchor})" for name, anchor in category["items"]])
491 tech_lines.append(f"**{category['title']}**\n{links}\n")
492
493 tech_section = "\n".join(tech_lines)
494
495 return f"""### Find Your Technology
496
497 **Select your primary infrastructure to jump directly to relevant integrations:**
498
499 {tech_section}
500 **Don't see what you need?** We support [Prometheus endpoints](#generic-data-collection), [SNMP devices](#generic-data-collection), [StatsD](#beyond-the-850-integrations), and [custom data sources](#generic-data-collection).
501 """
502
503
504 def _render_generic_collectors() -> str:
505 """Render the 'Beyond the 850+ integrations' section."""
506 generic_collectors = [
507 {
508 "name": "Prometheus collector",
509 "link": "/src/go/plugin/go.d/collector/prometheus/README.md",
510 "description": "Any application exposing Prometheus metrics"
511 },
512 {
513 "name": "StatsD collector",
514 "link": "/src/collectors/statsd.plugin/README.md",
515 "description": "Applications instrumented with [StatsD](https://blog.netdata.cloud/introduction-to-statsd/)"
516 },
517 {
518 "name": "Pandas collector",
519 "link": "/src/collectors/python.d.plugin/pandas/README.md",
520 "description": "Structured data from CSV, JSON, XML, and more"
521 },
522 ]
523
524 collector_lines = []
525 for collector in generic_collectors:
526 collector_lines.append(f"- **[{collector['name']}]({collector['link']})** - {collector['description']}")
527
528 collectors_section = "\n".join(collector_lines)
529
530 return f"""## Beyond the 850+ integrations
531
532 Netdata can monitor virtually any application through generic collectors:
533
534 {collectors_section}
535
536 Need a dedicated integration? [Submit a feature request](https://github.com/netdata/netdata/issues/new/choose) on GitHub.
537 """
538
539
540 def render_tables(sections: List[Tuple[str, List[Tuple[str, str, str]]]]) -> str:
541 """Render markdown tables for all sections."""
542 lines = []
543
544 for title, items in sections:
545 if not items:
546 continue
547
548 lines.append(f"### {title}\n\n")
549 lines.append("| Integration | Description |\n|-------------|-------------|\n")
550
551 for name, link, desc in items:
552 # Escape pipe characters in description
553 desc = desc.replace('|', '\\|')
554 lines.append(f"| [{name}]({link}) | {desc} |\n")
555
556 lines.append("\n")
557
558 return ''.join(lines)
559
560
561 # =============================================================================
562 # Main Execution
563 # =============================================================================
564
565 def generate_collectors_md() -> None:
566 """Generate COLLECTORS.md from integrations.js."""
567 # Load data
568 categories, integrations = load_catalog()
569
570 # Process integrations
571 sections = collect_integrations_by_section(categories, integrations)
572
573 # Render markdown
574 header = render_header()
575 tables = render_tables(sections)
576 content = header + "## Available Data Collection Integrations\n\n" + tables
577
578 # Write to file atomically
579 outfile = pathlib.Path("./src/collectors/COLLECTORS.md")
580 outfile.parent.mkdir(parents=True, exist_ok=True)
581
582 tmp = outfile.with_suffix(outfile.suffix + ".tmp")
583 tmp.write_text(content.rstrip('\n') + "\n", encoding='utf-8')
584 tmp.replace(outfile)
585
586
587 if __name__ == '__main__':
588 generate_collectors_md()