chore: refactor get_doc_integrations.py to use main() and improve structure (#20983)
Ilya Mashchenko committed
Sep 19, 2025 at 12:39 UTC
97d61e8cae8cee9c030197effda4e0fef1c8a5a5
1 file changed
+324
-426
integrations/gen_docs_integrations.py
+324
-426
@@ -1,3 +1,4 @@
1
+#!/usr/bin/env python3
2
import argparse
3
import json
4
import re
@@ -5,182 +6,165 @@ import shutil
6
import sys
7
from pathlib import Path
8
8
-# Dictionary responsible for making the symbolic links at the end of the script's run.
9
+# Registry used to decide which README.md should symlink to which generated file
10
symlink_dict = {}
11
12
13
+# -----------------------------
14
+# FS utilities
15
+# -----------------------------
16
def cleanup(only_base_paths=None):
17
"""
14
- clean directories that are either data collection or exporting integrations
15
- If only_base_paths is provided (list of base dirs), clean ONLY those.
18
+ Clean generated /integrations folders.
19
+ - If only_base_paths is provided (list of base dirs), clean ONLY those.
20
+ - Otherwise, do a full cleanup (legacy behavior).
21
"""
17
- if only_base_paths:
18
- for base in only_base_paths:
19
- p = Path(base) / "integrations"
20
- if p.exists():
21
- shutil.rmtree(p)
22
- return
23
-
24
- for element in Path("src/go/plugin/go.d/collector").glob('**/*/'):
25
- if "integrations" in str(element):
26
- shutil.rmtree(element)
27
- for element in Path("src/collectors").glob('**/*/'):
28
- # print(element)
29
- if "integrations" in str(element):
30
- shutil.rmtree(element)
31
-
32
- for element in Path("src/exporting").glob('**/*/'):
33
- if "integrations" in str(element):
34
- shutil.rmtree(element)
35
- for element in Path("integrations/cloud-notifications").glob('**/*/'):
36
- if "integrations" in str(element) and not "metadata.yaml" in str(element):
37
- shutil.rmtree(element)
38
- for element in Path("integrations/logs").glob('**/*/'):
39
- if "integrations" in str(element) and "metadata.yaml" not in str(element):
40
- shutil.rmtree(element)
41
- for element in Path("integrations/cloud-authentication").glob('**/*/'):
42
- if "integrations" in str(element) and not "metadata.yaml" in str(element):
43
- shutil.rmtree(element)
44
-
45
-
46
-def generate_category_from_name(category_fragment, category_array):
22
+ targets = [
23
+ "src/go/plugin/go.d/collector",
24
+ "src/collectors",
25
+ "src/exporting",
26
+ "integrations/cloud-notifications",
27
+ "integrations/logs",
28
+ "integrations/cloud-authentication",
29
+ ]
30
+ bases = only_base_paths if only_base_paths else targets
31
+ for base in bases:
32
+ for p in Path(base).glob("**/integrations"):
33
+ shutil.rmtree(p, ignore_errors=True)
34
+
35
+
36
+def clean_and_write(md: str, path: Path):
37
"""
48
- Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects.
38
+ Convert custom {% details %} markers to HTML <details> and write file.
39
"""
50
-
51
- category_name = ""
52
- i = 0
53
- dummy_id = category_fragment[0]
54
-
55
- while i < len(category_fragment):
56
- for category in category_array:
57
-
58
- if dummy_id == category['id']:
59
- category_name = category_name + "/" + category["name"]
60
- try:
61
- # print("equals")
62
- # print(fragment, category_fragment[i+1])
63
- dummy_id = dummy_id + "." + category_fragment[i + 1]
64
- # print(dummy_id)
65
- except IndexError:
66
- return category_name.split("/", 1)[1]
67
- category_array = category['children']
68
- break
69
- i += 1
40
+ md = md.replace('{% details open=true summary="', "<details open><summary>")
41
+ md = md.replace('{% details summary="', "<details><summary>")
42
+ md = md.replace('" %}', "</summary>\n")
43
+ md = md.replace("{% /details %}", "</details>\n")
44
+ path.write_text(md, encoding="utf-8")
45
46
72
-def clean_and_write(md, path):
47
+def build_path(meta_yaml_link: str) -> str:
48
"""
74
- This function takes care of the special details element, and converts it to the equivalent that md expects.
75
- Then it writes the buffer on the file provided.
49
+ Convert GitHub edit link to local repo path (without trailing /metadata.yaml).
50
"""
77
- # clean first, replace
78
- md = md.replace("{% details summary=\"", "<details><summary>")
79
- md = md.replace("{% details open=true summary=\"", "<details open><summary>")
80
- md = md.replace("\" %}", "</summary>\n")
81
- md = md.replace("{% /details %}", "</details>\n")
82
-
83
- path.write_text(md)
51
+ return (
52
+ meta_yaml_link.replace("https://github.com/netdata/", "")
53
+ .split("/", 1)[1]
54
+ .replace("edit/master/", "")
55
+ .replace("/metadata.yaml", "")
56
+ )
57
58
86
-def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, mode='default'):
59
+# -----------------------------
60
+# Content builders
61
+# -----------------------------
62
+def add_custom_edit_url(markdown_string: str, meta_yaml_link: str, sidebar_label_string: str,
63
+ mode: str = "default") -> str:
64
"""
88
- Takes a markdown string and adds a "custom_edit_url" metadata to the metadata field
65
+ Inject custom_edit_url into the metadata header.
66
"""
90
-
91
- output = ""
92
- path_to_md_file = ""
93
-
94
- if mode == 'default':
95
- path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}'
96
-
97
- elif mode == 'cloud-notification':
98
- path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
99
-
100
- elif mode == 'agent-notification':
67
+ if mode == "default":
68
+ path_to_md_file = f"{meta_yaml_link.replace('/metadata.yaml', '')}/integrations/{clean_string(sidebar_label_string)}"
69
+ elif mode in ("cloud-notification", "logs", "cloud-authentication"):
70
+ path_to_md_file = meta_yaml_link.replace("metadata.yaml", f"integrations/{clean_string(sidebar_label_string)}")
71
+ elif mode == "agent-notification":
72
path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README")
73
+ else:
74
+ # safe fallback
75
+ path_to_md_file = f"{meta_yaml_link.replace('/metadata.yaml', '')}/integrations/{clean_string(sidebar_label_string)}"
76
103
- elif mode == 'cloud-authentication':
104
- path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
105
-
106
- elif mode == 'logs':
107
- path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
108
-
109
- output = markdown_string.replace(
110
- "<!--startmeta",
111
- f'<!--startmeta\ncustom_edit_url: \"{path_to_md_file}.md\"')
112
-
113
- return output
114
-
115
-
116
-def clean_string(string):
117
- """
118
- simple function to get rid of caps, spaces, slashes and parentheses from a given string
77
+ return markdown_string.replace(
78
+ "<!--startmeta", f"<!--startmeta\ncustom_edit_url: \"{path_to_md_file}.md\""
79
+ )
80
120
- The string represents an integration name, as it would be displayed in the final text
121
- """
81
123
- return string.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "").replace(":", "")
82
+def clean_string(string: str) -> str:
83
+ return (
84
+ string.lower()
85
+ .replace(" ", "_")
86
+ .replace("/", "-")
87
+ .replace("(", "")
88
+ .replace(")", "")
89
+ .replace(":", "")
90
+ )
91
92
126
-def read_integrations_js(path_to_file):
93
+def read_integrations_js(path_to_file: str):
94
"""
128
- Open integrations/integrations.js and extract the dictionaries
95
+ Parse integrations/integrations.js and return (categories, integrations).
96
"""
130
-
97
try:
98
data = Path(path_to_file).read_text()
133
-
99
categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0]
100
integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1]
136
-
101
return json.loads(categories_str), json.loads(integrations_str)
138
-
102
except FileNotFoundError as e:
103
print("Exception", e)
104
+ return [], []
105
106
143
-def create_overview(integration, filename, overview_key_name="overview"):
144
- # empty overview_key_name to have only image on overview
145
- if not overview_key_name:
146
- return f"""# {integration['meta']['name']}
107
+def generate_category_from_name(category_fragment, category_array) -> str:
108
+ """
109
+ Given a split category id (by ".") and categories tree, return Learn path.
110
+ """
111
+ category_name = ""
112
+ i = 0
113
+ dummy_id = category_fragment[0]
114
148
-<img src="https://netdata.cloud/img/{filename}" width="150"/>
149
-"""
115
+ while i < len(category_fragment):
116
+ for category in category_array:
117
+ if dummy_id == category["id"]:
118
+ category_name += f"/{category['name']}"
119
+ try:
120
+ dummy_id = f"{dummy_id}.{category_fragment[i + 1]}"
121
+ except IndexError:
122
+ return category_name.split("/", 1)[1]
123
+ category_array = category["children"]
124
+ break
125
+ i += 1
126
+ return category_name.split("/", 1)[1] if category_name else ""
127
151
- split = re.split(r'(#.*\n)', integration[overview_key_name], 1)
128
129
+def create_overview(integration, filename: str, overview_key_name: str = "overview") -> str:
130
+ # Empty overview_key_name => only image on overview
131
+ if not overview_key_name:
132
+ return f"# {integration['meta']['name']}\n\n<img src=\"https://netdata.cloud/img/{filename}\" width=\"150\"/>\n"
133
+
134
+ split = re.split(r"(#.*\n)", integration[overview_key_name], 1)
135
first_overview_part = split[1]
136
rest_overview_part = split[2]
137
138
if not filename:
157
- return f"""{first_overview_part}{rest_overview_part}
158
-"""
139
+ return f"{first_overview_part}{rest_overview_part}"
140
141
return f"""{first_overview_part}
142
143
<img src="https://netdata.cloud/img/{filename}" width="150"/>
144
164
-{rest_overview_part}
165
-"""
145
+{rest_overview_part}"""
146
147
168
-def build_readme_from_integration(integration, mode=''):
169
- # COLLECTORS
170
- if mode == 'collector':
148
+def build_readme_from_integration(integration, categories, mode: str = ""):
149
+ """
150
+ Build the README markdown string for an integration.
151
+ Returns (meta_yaml, sidebar_label, learn_rel_path, md, community_badge)
152
+ """
153
+ md = ""
154
+ meta_yaml = ""
155
+ sidebar_label = ""
156
+ learn_rel_path = ""
157
172
- try:
173
- # initiate the variables for the collector
174
- meta_yaml = integration['edit_link'].replace("blob", "edit")
175
- sidebar_label = integration['meta']['monitored_instance']['name']
158
+ try:
159
+ if mode == "collector":
160
+ meta_yaml = integration["edit_link"].replace("blob", "edit")
161
+ sidebar_label = integration["meta"]["monitored_instance"]["name"]
162
learn_rel_path = generate_category_from_name(
177
- integration['meta']['monitored_instance']['categories'][0].split("."), categories).replace(
178
- "Data Collection", "Collecting Metrics")
179
- most_popular = integration['meta']['most_popular']
163
+ integration["meta"]["monitored_instance"]["categories"][0].split("."), categories
164
+ ).replace("Data Collection", "Collecting Metrics")
165
+ most_popular = integration["meta"]["most_popular"]
166
181
- # build the markdown string
182
- md = \
183
- f"""<!--startmeta
167
+ md = f"""<!--startmeta
168
meta_yaml: "{meta_yaml}"
169
sidebar_label: "{sidebar_label}"
170
learn_status: "Published"
@@ -191,39 +175,23 @@ endmeta-->
175
176
{create_overview(integration, integration['meta']['monitored_instance']['icon_filename'])}"""
177
194
- if integration['metrics']:
195
- md += f"""
196
-{integration['metrics']}
197
-"""
198
-
199
- if integration['alerts']:
200
- md += f"""
201
-{integration['alerts']}
202
-"""
203
-
204
- if integration['setup']:
205
- md += f"""
206
-{integration['setup']}
207
-"""
208
-
209
- if integration['troubleshooting']:
210
- md += f"""
211
-{integration['troubleshooting']}
212
-"""
213
- except Exception as e:
214
- print("Exception in collector md construction", e, integration['id'])
215
-
216
- # EXPORTERS
217
- elif mode == 'exporter':
218
- try:
219
- # initiate the variables for the exporter
220
- meta_yaml = integration['edit_link'].replace("blob", "edit")
221
- sidebar_label = integration['meta']['name']
222
- learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
223
-
224
- # build the markdown string
225
- md = \
226
- f"""<!--startmeta
178
+ if integration.get("metrics"):
179
+ md += f"\n{integration['metrics']}\n"
180
+ if integration.get("alerts"):
181
+ md += f"\n{integration['alerts']}\n"
182
+ if integration.get("setup"):
183
+ md += f"\n{integration['setup']}\n"
184
+ if integration.get("troubleshooting"):
185
+ md += f"\n{integration['troubleshooting']}\n"
186
+
187
+ elif mode == "exporter":
188
+ meta_yaml = integration["edit_link"].replace("blob", "edit")
189
+ sidebar_label = integration["meta"]["name"]
190
+ learn_rel_path = generate_category_from_name(
191
+ integration["meta"]["categories"][0].split("."), categories
192
+ )
193
+
194
+ md = f"""<!--startmeta
195
meta_yaml: "{meta_yaml}"
196
sidebar_label: "{sidebar_label}"
197
learn_status: "Published"
@@ -233,29 +201,19 @@ endmeta-->
201
202
{create_overview(integration, integration['meta']['icon_filename'])}"""
203
236
- if integration['setup']:
237
- md += f"""
238
-{integration['setup']}
239
-"""
204
+ if integration.get("setup"):
205
+ md += f"\n{integration['setup']}\n"
206
+ if integration.get("troubleshooting"):
207
+ md += f"\n{integration['troubleshooting']}\n"
208
241
- if integration['troubleshooting']:
242
- md += f"""
243
-{integration['troubleshooting']}
244
-"""
245
- except Exception as e:
246
- print("Exception in exporter md construction", e, integration['id'])
209
+ elif mode == "agent-notification":
210
+ meta_yaml = integration["edit_link"].replace("blob", "edit")
211
+ sidebar_label = integration["meta"]["name"]
212
+ learn_rel_path = generate_category_from_name(
213
+ integration["meta"]["categories"][0].split("."), categories
214
+ )
215
248
- # NOTIFICATIONS
249
- elif mode == 'agent-notification':
250
- try:
251
- # initiate the variables for the notification method
252
- meta_yaml = integration['edit_link'].replace("blob", "edit")
253
- sidebar_label = integration['meta']['name']
254
- learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
255
-
256
- # build the markdown string
257
- md = \
258
- f"""<!--startmeta
216
+ md = f"""<!--startmeta
217
meta_yaml: "{meta_yaml}"
218
sidebar_label: "{sidebar_label}"
219
learn_status: "Published"
@@ -265,29 +223,19 @@ endmeta-->
223
224
{create_overview(integration, integration['meta']['icon_filename'], "overview")}"""
225
268
- if integration['setup']:
269
- md += f"""
270
-{integration['setup']}
271
-"""
272
-
273
- if integration['troubleshooting']:
274
- md += f"""
275
-{integration['troubleshooting']}
276
-"""
226
+ if integration.get("setup"):
227
+ md += f"\n{integration['setup']}\n"
228
+ if integration.get("troubleshooting"):
229
+ md += f"\n{integration['troubleshooting']}\n"
230
278
- except Exception as e:
279
- print("Exception in notification md construction", e, integration['id'])
231
+ elif mode == "cloud-notification":
232
+ meta_yaml = integration["edit_link"].replace("blob", "edit")
233
+ sidebar_label = integration["meta"]["name"]
234
+ learn_rel_path = generate_category_from_name(
235
+ integration["meta"]["categories"][0].split("."), categories
236
+ )
237
281
- elif mode == 'cloud-notification':
282
- try:
283
- # initiate the variables for the notification method
284
- meta_yaml = integration['edit_link'].replace("blob", "edit")
285
- sidebar_label = integration['meta']['name']
286
- learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
287
-
288
- # build the markdown string
289
- md = \
290
- f"""<!--startmeta
238
+ md = f"""<!--startmeta
239
meta_yaml: "{meta_yaml}"
240
sidebar_label: "{sidebar_label}"
241
learn_status: "Published"
@@ -297,29 +245,19 @@ endmeta-->
245
246
{create_overview(integration, integration['meta']['icon_filename'], "")}"""
247
300
- if integration['setup']:
301
- md += f"""
302
-{integration['setup']}
303
-"""
304
-
305
- if integration['troubleshooting']:
306
- md += f"""
307
-{integration['troubleshooting']}
308
-"""
248
+ if integration.get("setup"):
249
+ md += f"\n{integration['setup']}\n"
250
+ if integration.get("troubleshooting"):
251
+ md += f"\n{integration['troubleshooting']}\n"
252
310
- except Exception as e:
311
- print("Exception in notification md construction", e, integration['id'])
253
+ elif mode == "logs":
254
+ meta_yaml = integration["edit_link"].replace("blob", "edit")
255
+ sidebar_label = integration["meta"]["name"]
256
+ learn_rel_path = generate_category_from_name(
257
+ integration["meta"]["categories"][0].split("."), categories
258
+ )
259
313
- elif mode == 'logs':
314
- try:
315
- # initiate the variables for the logs integration
316
- meta_yaml = integration['edit_link'].replace("blob", "edit")
317
- sidebar_label = integration['meta']['name']
318
- learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
319
-
320
- # build the markdown string
321
- md = \
322
- f"""<!--startmeta
260
+ md = f"""<!--startmeta
261
meta_yaml: "{meta_yaml}"
262
sidebar_label: "{sidebar_label}"
263
learn_status: "Published"
@@ -329,26 +267,17 @@ endmeta-->
267
268
{create_overview(integration, integration['meta']['icon_filename'])}"""
269
332
- if integration['setup']:
333
- md += f"""
334
-{integration['setup']}
335
-"""
336
-
337
- except Exception as e:
338
- print("Exception in logs md construction", e, integration['id'])
339
-
270
+ if integration.get("setup"):
271
+ md += f"\n{integration['setup']}\n"
272
341
- # AUTHENTICATIONS
342
- elif mode == 'authentication':
343
- if True:
344
- # initiate the variables for the authentication method
345
- meta_yaml = integration['edit_link'].replace("blob", "edit")
346
- sidebar_label = integration['meta']['name']
347
- learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
273
+ elif mode == "authentication":
274
+ meta_yaml = integration["edit_link"].replace("blob", "edit")
275
+ sidebar_label = integration["meta"]["name"]
276
+ learn_rel_path = generate_category_from_name(
277
+ integration["meta"]["categories"][0].split("."), categories
278
+ )
279
349
- # build the markdown string
350
- md = \
351
- f"""<!--startmeta
280
+ md = f"""<!--startmeta
281
meta_yaml: "{meta_yaml}"
282
sidebar_label: "{sidebar_label}"
283
learn_status: "Published"
@@ -358,270 +287,239 @@ endmeta-->
287
288
{create_overview(integration, integration['meta']['icon_filename'])}"""
289
361
- if integration['setup']:
362
- md += f"""
363
-{integration['setup']}
364
-"""
290
+ if integration.get("setup"):
291
+ md += f"\n{integration['setup']}\n"
292
+ if integration.get("troubleshooting"):
293
+ md += f"\n{integration['troubleshooting']}\n"
294
366
- if integration['troubleshooting']:
367
- md += f"""
368
-{integration['troubleshooting']}
369
-"""
295
+ except Exception as e:
296
+ print("Exception building md", e, integration.get("id"))
297
371
- # except Exception as e:
372
- # print("Exception in authentication md construction", e, integration['id'])
373
-
374
- if "community" in integration['meta'].keys():
375
- community = "<img src=\"https://img.shields.io/badge/maintained%20by-Community-blue\" />"
376
- else:
377
- community = "<img src=\"https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44\" />"
298
+ # Community badge
299
+ community = '<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />'
300
+ if "community" in integration["meta"]:
301
+ community = '<img src="https://img.shields.io/badge/maintained%20by-Community-blue" />'
302
303
return meta_yaml, sidebar_label, learn_rel_path, md, community
304
305
382
-def build_path(meta_yaml_link):
306
+def create_overview_banner(md: str, community_badge: str) -> str:
307
"""
384
- function that takes a metadata yaml file link, and makes it into a path that gets used to write to a file.
308
+ Insert the community badge right before the first '##' section.
309
"""
386
- return meta_yaml_link.replace("https://github.com/netdata/", "") \
387
- .split("/", 1)[1] \
388
- .replace("edit/master/", "") \
389
- .replace("/metadata.yaml", "")
310
+ if "##" not in md:
311
+ return f"{md}\n\n{community_badge}\n"
312
+ upper, lower = md.split("##", 1)
313
+ return f"{upper}{community_badge}\n\n##{lower}"
314
315
392
-def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default'):
316
+def write_to_file(path: str, md: str, meta_yaml: str, sidebar_label: str, community: str, integration=None,
317
+ mode: str = "default"):
318
"""
394
- takes the arguments needed to write the integration markdown to the proper file.
319
+ Write the generated markdown into an `integrations/` subdirectory located alongside the `metadata.yaml` file.
320
+ This mirrors the original behavior of placing docs next to their source metadata.
321
"""
322
+ md = create_overview_banner(md, community)
323
397
- upper, lower = md.split("##", 1)
398
-
399
- md = upper + community + f"\n\n##{lower}"
400
-
401
- if mode == 'default':
402
- # Only if the path exists, this caters for running the same script on both the go and netdata repos.
403
- if Path(path).exists():
404
- if not Path(f'{path}/integrations').exists():
405
- Path(f'{path}/integrations').mkdir()
324
+ if mode == "default":
325
+ base = Path(path)
326
+ if base.exists():
327
+ integrations_dir = base / "integrations"
328
+ integrations_dir.mkdir(exist_ok=True)
329
330
try:
408
- md = add_custom_edit_url(md, meta_yaml, sidebar_label)
409
- clean_and_write(
410
- md,
411
- Path(f'{path}/integrations/{clean_string(sidebar_label)}.md')
412
- )
413
-
331
+ md2 = add_custom_edit_url(md, meta_yaml, sidebar_label)
332
+ outfile = integrations_dir / f"{clean_string(sidebar_label)}.md"
333
+ clean_and_write(md2, outfile)
334
except FileNotFoundError as e:
335
print("Exception in writing to file", e)
336
417
- # If we only created one file inside the directory, add the entry to the symlink_dict, so we can make the symbolic link
418
- if len(list(Path(f'{path}/integrations').iterdir())) == 1:
419
- symlink_dict.update(
420
- {path: f'integrations/{clean_string(sidebar_label)}.md'})
337
+ # If there's only one file inside the directory, register it for README symlink
338
+ if len(list(integrations_dir.iterdir())) == 1:
339
+ symlink_dict.update({path: f"integrations/{clean_string(sidebar_label)}.md"})
340
else:
341
try:
342
symlink_dict.pop(path)
343
except KeyError:
425
- # We don't need to print something here.
344
pass
427
- elif mode == 'cloud-notification':
428
-
429
- # for cloud notifications we generate them near their metadata.yaml
430
- name = clean_string(integration['meta']['name'])
431
-
432
- if not Path(f'{path}/integrations').exists():
433
- Path(f'{path}/integrations').mkdir()
434
-
435
- # proper_edit_name = meta_yaml.replace(
436
- # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
437
-
438
- md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notification')
439
-
440
- finalpath = f'{path}/integrations/{name}.md'
345
346
+ elif mode == "cloud-notification":
347
+ name = clean_string(integration["meta"]["name"])
348
+ base = Path(path)
349
+ integrations_dir = base / "integrations"
350
+ integrations_dir.mkdir(exist_ok=True)
351
+ md2 = add_custom_edit_url(md, meta_yaml, sidebar_label, mode="cloud-notification")
352
+ finalpath = integrations_dir / f"{name}.md"
353
try:
443
- clean_and_write(
444
- md,
445
- Path(finalpath)
446
- )
354
+ clean_and_write(md2, finalpath)
355
except FileNotFoundError as e:
356
print("Exception in writing to file", e)
449
- elif mode == 'agent-notification':
450
- # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
451
- # afterwards the ingest will replace this metadata with meta_yaml
452
-
453
- md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notification')
454
-
455
- finalpath = f'{path}/README.md'
357
358
+ elif mode == "agent-notification":
359
+ md2 = add_custom_edit_url(md, meta_yaml, sidebar_label, mode="agent-notification")
360
+ finalpath = Path(path) / "README.md"
361
try:
458
- clean_and_write(
459
- md,
460
- Path(finalpath)
461
- )
462
-
362
+ clean_and_write(md2, finalpath)
363
except FileNotFoundError as e:
364
print("Exception in writing to file", e)
465
- elif mode == 'logs':
466
-
467
- # for logs we generate them near their metadata.yaml
468
-
469
- name = clean_string(integration['meta']['name'])
470
-
471
- if not Path(f'{path}/integrations').exists():
472
- Path(f'{path}/integrations').mkdir()
473
-
474
- # proper_edit_name = meta_yaml.replace(
475
- # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
476
-
477
- md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='logs')
478
-
479
- finalpath = f'{path}/integrations/{name}.md'
365
366
+ elif mode == "logs":
367
+ name = clean_string(integration["meta"]["name"])
368
+ base = Path(path)
369
+ integrations_dir = base / "integrations"
370
+ integrations_dir.mkdir(exist_ok=True)
371
+ md2 = add_custom_edit_url(md, meta_yaml, sidebar_label, mode="logs")
372
+ finalpath = integrations_dir / f"{name}.md"
373
try:
482
- clean_and_write(
483
- md,
484
- Path(finalpath)
485
- )
374
+ clean_and_write(md2, finalpath)
375
except FileNotFoundError as e:
376
print("Exception in writing to file", e)
488
- elif mode == 'authentication':
489
-
490
- name = clean_string(integration['meta']['name'])
491
-
492
- if not Path(f'{path}/integrations').exists():
493
- Path(f'{path}/integrations').mkdir()
494
-
495
- # proper_edit_name = meta_yaml.replace(
496
- # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
497
-
498
- md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-authentication')
499
-
500
- finalpath = f'{path}/integrations/{name}.md'
377
378
+ elif mode == "authentication":
379
+ name = clean_string(integration["meta"]["name"])
380
+ base = Path(path)
381
+ integrations_dir = base / "integrations"
382
+ integrations_dir.mkdir(exist_ok=True)
383
+ md2 = add_custom_edit_url(md, meta_yaml, sidebar_label, mode="cloud-authentication")
384
+ finalpath = integrations_dir / f"{name}.md"
385
try:
503
- clean_and_write(
504
- md,
505
- Path(finalpath)
506
- )
507
-
386
+ clean_and_write(md2, finalpath)
387
except FileNotFoundError as e:
388
print("Exception in writing to file", e)
389
390
512
-def make_symlinks(symlink_dict):
391
+def make_symlinks(symlinks: dict):
392
"""
514
- takes a dictionary with directories that have a 1:1 relationship between their README and the integration (only one) inside the "integrations" folder.
393
+ Create README.md symlinks to the sole file in each /integrations dir.
394
"""
516
- for element in symlink_dict:
517
- if not Path(f'{element}/README.md').exists():
518
- Path(f'{element}/README.md').touch()
395
+ for element in symlinks:
396
+ readme = Path(element) / "README.md"
397
+ if not readme.exists():
398
+ readme.touch()
399
try:
520
- # Remove the README to prevent it being a normal file
521
- Path(f'{element}/README.md').unlink()
400
+ readme.unlink()
401
except FileNotFoundError:
523
- continue
524
- # and then make a symlink to the actual markdown
525
- Path(f'{element}/README.md').symlink_to(symlink_dict[element])
526
-
527
- filepath = Path(f'{element}/{symlink_dict[element]}')
528
- md = filepath.read_text()
529
-
530
- # This preserves the custom_edit_url for most files as it was,
531
- # so the existing links don't break, this is vital for link replacement afterwards
532
- filepath.write_text(md.replace(
533
- f'{element}/{symlink_dict[element]}', f'{element}/README.md'))
534
-
402
+ pass
403
536
-parser = argparse.ArgumentParser(description="Generate integration docs from metadata.yaml files.")
537
-parser.add_argument("-c", "--collector",
538
- help="Only generate docs for this collector (plugin/module), e.g. 'go.d/snmp' or 'apps.plugin/groups'",
539
- default=None)
540
-args = parser.parse_args()
404
+ readme.symlink_to(symlinks[element])
405
542
-categories, integrations = read_integrations_js('integrations/integrations.js')
406
+ filepath = Path(element) / symlinks[element]
407
+ md = filepath.read_text()
408
+ filepath.write_text(md.replace(f"{element}/{symlinks[element]}", f"{element}/README.md"))
409
410
545
-def _base_paths_for_collector(integrations, collector_key):
411
+# -----------------------------
412
+# Filtering helpers
413
+# -----------------------------
414
+def _base_paths_for_collector(integrations, collector_key: str):
415
+ """
416
+ Return local base paths (without /integrations) for a single collector key: 'plugin/module'
417
+ """
418
if not collector_key:
419
return []
420
paths = []
421
for integ in integrations:
550
- if integ.get('integration_type') != 'collector':
422
+ if integ.get("integration_type") != "collector":
423
continue
552
- meta = integ.get('meta', {})
553
- plugin = meta.get('plugin_name')
554
- module = meta.get('module_name')
424
+ meta = integ.get("meta", {})
425
+ plugin = meta.get("plugin_name")
426
+ module = meta.get("module_name")
427
if not plugin or not module:
428
continue
557
- key = plugin + "/" + module
429
+ key = f"{plugin}/{module}"
430
if key == collector_key:
559
- meta_yaml = integ.get('edit_link', '').replace("blob", "edit")
560
- paths.append(build_path(meta_yaml))
431
+ meta_yaml = integ.get("edit_link", "").replace("blob", "edit")
432
+ base = build_path(meta_yaml)
433
+ paths.append(base)
434
return paths
435
436
564
-only_paths = _base_paths_for_collector(integrations, args.collector)
565
-
566
-if args.collector and not only_paths:
567
- print("No matching collector found for:", args.collector)
568
- sys.exit(1)
437
+# -----------------------------
438
+# CLI entry
439
+# -----------------------------
440
+def main():
441
+ parser = argparse.ArgumentParser(description="Generate integration docs from metadata.yaml files.")
442
+ parser.add_argument(
443
+ "-c",
444
+ "--collector",
445
+ help="Generate docs only for this collector (plugin/module), e.g. 'go.d/snmp' or 'apps.plugin/groups'",
446
+ default=None,
447
+ )
448
+ args = parser.parse_args()
449
570
-cleanup(only_paths)
450
+ categories, integrations = read_integrations_js("integrations/integrations.js")
451
572
-# Iterate through every integration
573
-for integration in integrations:
452
if args.collector:
575
- if integration.get('integration_type') != "collector":
576
- continue
577
- meta = integration.get('meta', {})
578
- plugin = meta.get('plugin_name')
579
- module = meta.get('module_name')
580
- if not plugin or not module or (plugin + "/" + module) != args.collector:
581
- continue
582
-
583
- if integration['integration_type'] == "collector":
584
-
585
- meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
586
- integration, mode='collector')
587
- path = build_path(meta_yaml)
588
- write_to_file(path, md, meta_yaml, sidebar_label, community)
589
-
453
+ # compute targets and CLEAN ONLY those
454
+ only_paths = _base_paths_for_collector(integrations, args.collector)
455
+ if not only_paths:
456
+ print(f"No matching collector found for: {args.collector}")
457
+ sys.exit(0)
458
+ cleanup(only_paths)
459
else:
591
- # kind of specific if clause, so we can avoid running excessive code in the go repo
592
- if integration['integration_type'] == "exporter":
593
-
460
+ # full cleanup (legacy behavior)
461
+ cleanup()
462
+
463
+ # Generate
464
+ for integration in integrations:
465
+ itype = integration.get("integration_type")
466
+
467
+ # If -c is used, process ONLY the matching collector; skip everything else
468
+ if args.collector:
469
+ if itype != "collector":
470
+ continue
471
+ meta = integration.get("meta", {})
472
+ plugin = meta.get("plugin_name")
473
+ module = meta.get("module_name")
474
+ if not plugin or not module or f"{plugin}/{module}" != args.collector:
475
+ continue
476
+
477
+ if itype == "collector":
478
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
595
- integration, mode='exporter')
479
+ integration, categories, mode="collector"
480
+ )
481
path = build_path(meta_yaml)
482
write_to_file(path, md, meta_yaml, sidebar_label, community)
483
599
- elif integration['integration_type'] == "agent_notification":
600
-
484
+ elif itype == "exporter" and not args.collector:
485
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
602
- integration, mode='agent-notification')
486
+ integration, categories, mode="exporter"
487
+ )
488
path = build_path(meta_yaml)
604
- write_to_file(path, md, meta_yaml, sidebar_label, community, mode='agent-notification')
605
-
606
- elif integration['integration_type'] == "cloud_notification":
489
+ write_to_file(path, md, meta_yaml, sidebar_label, community)
490
491
+ elif itype == "agent_notification" and not args.collector:
492
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
609
- integration, mode='cloud-notification')
493
+ integration, categories, mode="agent-notification"
494
+ )
495
path = build_path(meta_yaml)
611
- write_to_file(path, md, meta_yaml, sidebar_label, community, mode='cloud-notification')
612
-
613
- elif integration['integration_type'] == "logs":
496
+ write_to_file(path, md, meta_yaml, sidebar_label, community, integration=integration,
497
+ mode="agent-notification")
498
499
+ elif itype == "cloud_notification" and not args.collector:
500
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
616
- integration, mode='logs')
501
+ integration, categories, mode="cloud-notification"
502
+ )
503
path = build_path(meta_yaml)
618
- write_to_file(path, md, meta_yaml, sidebar_label, community, mode='logs')
504
+ write_to_file(path, md, meta_yaml, sidebar_label, community, integration=integration,
505
+ mode="cloud-notification")
506
620
- elif integration['integration_type'] == "authentication":
507
+ elif itype == "logs" and not args.collector:
508
+ meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
509
+ integration, categories, mode="logs"
510
+ )
511
+ path = build_path(meta_yaml)
512
+ write_to_file(path, md, meta_yaml, sidebar_label, community, integration=integration, mode="logs")
513
514
+ elif itype == "authentication" and not args.collector:
515
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
623
- integration, mode='authentication')
516
+ integration, categories, mode="authentication"
517
+ )
518
path = build_path(meta_yaml)
625
- write_to_file(path, md, meta_yaml, sidebar_label, community, mode='authentication')
519
+ write_to_file(path, md, meta_yaml, sidebar_label, community, integration=integration, mode="authentication")
520
+
521
+ make_symlinks(symlink_dict)
522
+
523
627
-make_symlinks(symlink_dict)
524
+if __name__ == "__main__":
525
+ main()