Simplify the script for generating documentation from integrations (#16009)
Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud>
Fotis Voutsas committed
Sep 26, 2023 at 08:55 UTC
ad26c3ddfc534879804ae84a2a076e53cc914fdf
1 file changed
+200
-125
integrations/gen_docs_integrations.py
+200
-125
@@ -1,8 +1,28 @@
1
import json
2
-import os
2
+import shutil
3
+from pathlib import Path
4
5
# Dictionary responsible for making the symbolic links at the end of the script's run.
6
symlink_dict = {}
7
+am_i_inside_go = "go.d.plugin" in str(Path.cwd())
8
+
9
+
10
+def cleanup():
11
+ """
12
+ clean directories that are either data collection or exporting integrations
13
+ """
14
+ if am_i_inside_go:
15
+ for element in Path("modules").glob('**/*/'):
16
+ if "integrations" in str(element):
17
+ shutil.rmtree(element)
18
+ else:
19
+ for element in Path("collectors").glob('**/*/'):
20
+ if "integrations" in str(element):
21
+ shutil.rmtree(element)
22
+
23
+ for element in Path("exporting").glob('**/*/'):
24
+ if "integrations" in str(element):
25
+ shutil.rmtree(element)
26
27
28
def generate_category_from_name(category_fragment, category_array):
@@ -31,7 +51,7 @@ def generate_category_from_name(category_fragment, category_array):
51
i += 1
52
53
34
-def clean_and_write(md, txt):
54
+def clean_and_write(md, path):
55
"""
56
This function takes care of the special details element, and converts it to the equivalent that md expects.
57
Then it writes the buffer on the file provided.
@@ -39,27 +59,63 @@ def clean_and_write(md, txt):
59
# clean first, replace
60
md = md.replace("{% details summary=\"", "<details><summary>").replace(
61
"\" %}", "</summary>\n").replace("{% /details %}", "</details>\n")
42
- # print(md)
43
- # exit()
62
45
- txt.write(md)
63
+ path.write_text(md)
64
65
48
-# Open integrations/integrations.js and extract the dictionaries
49
-with open('integrations/integrations.js') as dataFile:
50
- data = dataFile.read()
66
+def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, mode='default'):
67
+ """
68
+ Takes a markdown string and adds a "custom_edit_url" metadata to the metadata field
69
+ """
70
52
- categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0]
53
- integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1]
71
+ output = ""
72
55
- categories = json.loads(categories_str)
56
- integrations = json.loads(integrations_str)
73
+ if mode == 'default':
74
+ path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}'
75
58
-i = 0
59
-# Iterate through every integration
60
-for integration in integrations:
61
- i += 1
62
- if integration['integration_type'] == "collector":
76
+ elif mode == 'cloud-notifications':
77
+ path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
78
+
79
+ elif mode == 'agent-notifications':
80
+ path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README")
81
+
82
+ output = markdown_string.replace(
83
+ "<!--startmeta",
84
+ f'<!--startmeta\ncustom_edit_url: \"{path_to_md_file}.md\"')
85
+
86
+ return output
87
+
88
+
89
+def clean_string(string):
90
+ """
91
+ simple function to get rid of caps, spaces, slashes and parentheses from a given string
92
+
93
+ The string represents an integration name, as it would be displayed in the final text
94
+ """
95
+
96
+ return string.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")
97
+
98
+
99
+def read_integrations_js(path_to_file):
100
+ """
101
+ Open integrations/integrations.js and extract the dictionaries
102
+ """
103
+
104
+ try:
105
+ data = Path(path_to_file).read_text()
106
+
107
+ categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0]
108
+ integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1]
109
+
110
+ return json.loads(categories_str), json.loads(integrations_str)
111
+
112
+ except FileNotFoundError as e:
113
+ print("Exception", e)
114
+
115
+
116
+def build_readme_from_integration(integration, mode=''):
117
+ # COLLECTORS
118
+ if mode == 'collector':
119
120
try:
121
# initiate the variables for the collector
@@ -99,44 +155,11 @@ endmeta-->
155
md += f"""
156
{integration['troubleshooting']}
157
"""
102
-
103
- path = meta_yaml.replace("https://github.com/netdata/", "") \
104
- .split("/", 1)[1] \
105
- .replace("edit/master/", "") \
106
- .replace("/metadata.yaml", "")
107
-
108
- # Only if the path exists, this caters for running the same script on both the go and netdata repos.
109
- if os.path.exists(path):
110
- try:
111
- if not os.path.exists(f'{path}/integrations'):
112
- os.mkdir(f'{path}/integrations')
113
-
114
- with open(f'{path}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")}.md', 'w+') as txt:
115
- # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
116
- # afterwards the ingest will replace this metadata with meta_yaml
117
- md = md.replace(
118
- "<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("/metadata.yaml", "")}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")}.md\"')
119
-
120
- clean_and_write(md, txt)
121
- except Exception as e:
122
- print("Error in writing to the collector file", e, integration['id'])
123
-
124
- # If we only created one file inside a collector, add the entry to the symlink_dict, so we can make the link
125
- if len(os.listdir(f'{path}/integrations')) == 1:
126
- symlink_dict.update(
127
- {path: f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")}.md'})
128
- else:
129
- try:
130
- symlink_dict.pop(path)
131
- except KeyError:
132
- # We don't need to print something here.
133
- pass
134
-
158
except Exception as e:
159
print("Exception in collector md construction", e, integration['id'])
160
138
- # kind of specific if clause, so we can avoid running excessive code in the go repo
139
- elif integration['integration_type'] == "exporter" and "go.d.plugin" not in os.getcwd():
161
+ # EXPORTERS
162
+ elif mode == 'exporter':
163
try:
164
# initiate the variables for the exporter
165
meta_yaml = integration['edit_link'].replace("blob", "edit")
@@ -164,42 +187,11 @@ endmeta-->
187
md += f"""
188
{integration['troubleshooting']}
189
"""
167
-
168
- path = meta_yaml.replace("https://github.com/netdata/", "") \
169
- .split("/", 1)[1] \
170
- .replace("edit/master/", "") \
171
- .replace("/metadata.yaml", "")
172
-
173
- if os.path.exists(path):
174
- try:
175
- if not os.path.exists(f'{path}/integrations'):
176
- os.mkdir(f'{path}/integrations')
177
-
178
- with open(f'{path}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")}.md', 'w+') as txt:
179
- # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
180
- # afterwards the ingest will replace this metadata with meta_yaml
181
- md = md.replace(
182
- "<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("/metadata.yaml", "")}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")}.md\"')
183
-
184
- clean_and_write(md, txt)
185
- except Exception as e:
186
- print("Error in writing to the file", e, integration['id'])
187
-
188
- # If we only created one file inside a collector, add the entry to the symlink_dict, so we can make the link
189
- if len(os.listdir(f'{path}/integrations')) == 1:
190
- symlink_dict.update(
191
- {path: f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")}.md'})
192
- else:
193
- try:
194
- symlink_dict.pop(path)
195
- except KeyError:
196
- # We don't need to print something here.
197
- pass
190
except Exception as e:
191
print("Exception in exporter md construction", e, integration['id'])
192
201
- # kind of specific if clause, so we can avoid running excessive code in the go repo
202
- elif integration['integration_type'] == "notification" and "go.d.plugin" not in os.getcwd():
193
+ # NOTIFICATIONS
194
+ elif mode == 'notification':
195
try:
196
# initiate the variables for the notification method
197
meta_yaml = integration['edit_link'].replace("blob", "edit")
@@ -228,49 +220,132 @@ endmeta-->
220
{integration['troubleshooting']}
221
"""
222
231
- path = meta_yaml.replace("https://github.com/netdata/", "") \
232
- .split("/", 1)[1] \
233
- .replace("edit/master/", "") \
234
- .replace("/metadata.yaml", "")
223
+ except Exception as e:
224
+ print("Exception in notification md construction", e, integration['id'])
225
+
226
+ return meta_yaml, sidebar_label, learn_rel_path, md
227
236
- if "cloud-notifications" in path:
237
- # for cloud notifications we generate them near their metadata.yaml
238
- name = integration['meta']['name'].lower().replace(" ", "_").replace("(", "").replace(")", "")
239
- if not os.path.exists(f'{path}/integrations'):
240
- os.mkdir(f'{path}/integrations')
228
242
- proper_edit_name = meta_yaml.replace(
243
- "metadata.yaml", f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")}.md\"')
229
+def build_path(meta_yaml_link):
230
+ """
231
+ funtion that takes a metadata yaml file link, and makes it into a path that gets used to write to a file.
232
+ """
233
+ return meta_yaml_link.replace("https://github.com/netdata/", "") \
234
+ .split("/", 1)[1] \
235
+ .replace("edit/master/", "") \
236
+ .replace("/metadata.yaml", "")
237
245
- md = md.replace("<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{proper_edit_name}')
238
247
- finalpath = f'{path}/integrations/{name}.md'
248
- else:
249
- # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
250
- # afterwards the ingest will replace this metadata with meta_yaml
251
- md = md.replace("<!--startmeta",
252
- f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("metadata.yaml", "README.md")}')
253
- finalpath = f'{path}/README.md'
239
+def write_to_file(path, md, meta_yaml, sidebar_label, mode='default'):
240
+ """
241
+ takes the arguments needed to write the integration markdown to the proper file.
242
+ """
243
+ if mode == 'default':
244
+ # Only if the path exists, this caters for running the same script on both the go and netdata repos.
245
+ if Path(path).exists():
246
+ if not Path(f'{path}/integrations').exists():
247
+ Path(f'{path}/integrations').mkdir()
248
+
249
try:
255
- with open(finalpath, 'w') as txt:
256
- clean_and_write(md, txt)
257
- except Exception as e:
258
- print("Exception in notification md construction", e, integration['id'])
250
+ md = add_custom_edit_url(md, meta_yaml, sidebar_label)
251
+ clean_and_write(
252
+ md,
253
+ Path(f'{path}/integrations/{clean_string(sidebar_label)}.md')
254
+ )
255
+
256
+ except FileNotFoundError as e:
257
+ print("Exception in writing to file", e)
258
+
259
+ # If we only created one file inside the directory, add the entry to the symlink_dict, so we can make the symbolic link
260
+ if len(list(Path(f'{path}/integrations').iterdir())) == 1:
261
+ symlink_dict.update(
262
+ {path: f'integrations/{clean_string(sidebar_label)}.md'})
263
+ else:
264
+ try:
265
+ symlink_dict.pop(path)
266
+ except KeyError:
267
+ # We don't need to print something here.
268
+ pass
269
+ elif mode == 'notification':
270
260
- except Exception as e:
261
- print("Exception in for loop", e, "\n", integration)
262
-
263
-for element in symlink_dict:
264
- # Remove the README to prevent it being a normal file
265
- os.remove(f'{element}/README.md')
266
- # and then make a symlink to the actual markdown
267
- os.symlink(symlink_dict[element], f'{element}/README.md')
268
-
269
- with open(f'{element}/{symlink_dict[element]}', 'r') as txt:
270
- md = txt.read()
271
-
272
- # This preserves the custom_edit_url for most files as it was,
273
- # so the existing links don't break, this is vital for link replacement afterwards
274
- with open(f'{element}/{symlink_dict[element]}', 'w+') as txt:
275
- md = md.replace(f'{element}/{symlink_dict[element]}', f'{element}/README.md')
276
- txt.write(md)
271
+ if "cloud-notifications" in path:
272
+ # for cloud notifications we generate them near their metadata.yaml
273
+ name = clean_string(integration['meta']['name'])
274
+
275
+ if not Path(f'{path}/integrations').exists():
276
+ Path(f'{path}/integrations').mkdir()
277
+
278
+ # proper_edit_name = meta_yaml.replace(
279
+ # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
280
+
281
+ md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notifications')
282
+
283
+ finalpath = f'{path}/integrations/{name}.md'
284
+ else:
285
+ # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
286
+ # afterwards the ingest will replace this metadata with meta_yaml
287
+ md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notifications')
288
+
289
+ finalpath = f'{path}/README.md'
290
+
291
+ try:
292
+ clean_and_write(
293
+ md,
294
+ Path(finalpath)
295
+ )
296
+
297
+ except FileNotFoundError as e:
298
+ print("Exception in writing to file", e)
299
+
300
+
301
+def make_symlinks(symlink_dict):
302
+ """
303
+ takes a dictionary with directories that have a 1:1 relationship between their README and the integration (only one) inside the "integrations" folder.
304
+ """
305
+ for element in symlink_dict:
306
+ # Remove the README to prevent it being a normal file
307
+ Path.unlink(f'{element}/README.md')
308
+ # and then make a symlink to the actual markdown
309
+ Path(f'{element}/README.md').symlink_to(symlink_dict[element])
310
+
311
+ filepath = Path(f'{element}/{symlink_dict[element]}')
312
+ md = filepath.read_text()
313
+
314
+ # This preserves the custom_edit_url for most files as it was,
315
+ # so the existing links don't break, this is vital for link replacement afterwards
316
+ filepath.write_text(md.replace(
317
+ f'{element}/{symlink_dict[element]}', f'{element}/README.md'))
318
+
319
+
320
+cleanup()
321
+
322
+categories, integrations = read_integrations_js('integrations/integrations.js')
323
+
324
+
325
+# Iterate through every integration
326
+for integration in integrations:
327
+
328
+ if integration['integration_type'] == "collector":
329
+
330
+ meta_yaml, sidebar_label, learn_rel_path, md = build_readme_from_integration(integration, mode='collector')
331
+ path = build_path(meta_yaml)
332
+ write_to_file(path, md, meta_yaml, sidebar_label)
333
+
334
+ elif not am_i_inside_go:
335
+ # kind of specific if clause, so we can avoid running excessive code in the go repo
336
+ if integration['integration_type'] == "exporter":
337
+
338
+ meta_yaml, sidebar_label, learn_rel_path, md = build_readme_from_integration(integration, mode='exporter')
339
+ path = build_path(meta_yaml)
340
+ write_to_file(path, md, meta_yaml, sidebar_label)
341
+
342
+ # kind of specific if clause, so we can avoid running excessive code in the go repo
343
+ elif integration['integration_type'] == "notification":
344
+
345
+ meta_yaml, sidebar_label, learn_rel_path, md = build_readme_from_integration(
346
+ integration, mode='notification')
347
+ path = build_path(meta_yaml)
348
+ write_to_file(path, md, meta_yaml, sidebar_label, mode='notification')
349
+
350
+
351
+make_symlinks(symlink_dict)