Initial tooling for Integrations Documentation (#15893)
Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud> Co-authored-by: Tasos Katsoulas <12612986+tkatsoulas@users.noreply.github.com>
Fotis Voutsas committed
Sep 18, 2023 at 09:46 UTC
9c6c5d42a9a5efc4e8514cb1a78d219f983f7cd7
2 files changed
+339
.github/workflows/generate-integrations-docs.yml
new
+63
@@ -0,0 +1,63 @@
1
+---
2
+# CI workflow used to generate documentation from integrations/integrations.js.
3
+
4
+name: Generate Integrations Documentation
5
+on:
6
+ push:
7
+ branches:
8
+ - master
9
+ paths:
10
+ - 'integrations/integrations.js'
11
+ workflow_dispatch: null
12
+concurrency: # This keeps multiple instances of the job from running concurrently for the same ref.
13
+ group: generate-integrations-docs-${{ github.ref }}
14
+ cancel-in-progress: true
15
+jobs:
16
+ generate-integrations-documentation:
17
+ name: Generate Integrations Documentation
18
+ runs-on: ubuntu-latest
19
+ if: github.repository == 'netdata/netdata'
20
+ steps:
21
+ - name: Checkout Agent
22
+ id: checkout-agent
23
+ uses: actions/checkout@v4
24
+ with:
25
+ fetch-depth: 1
26
+ submodules: recursive
27
+ - name: Generate Integrations Documentation
28
+ id: generate
29
+ run: |
30
+ python3 integrations/gen_docs_integrations.py
31
+ - name: Create PR
32
+ id: create-pr
33
+ uses: peter-evans/create-pull-request@v5
34
+ with:
35
+ token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
36
+ commit-message: Generate Integrations Documentation
37
+ branch: integrations-docs
38
+ title: Integrations Documentation
39
+ body: |
40
+ Generate Documentation from `integrations/integrations.js` based on the latest code.
41
+
42
+ This PR was auto-generated by
43
+ `.github/workflows/generate-integrations-docs.yml`.
44
+ - name: Failure Notification
45
+ uses: rtCamp/action-slack-notify@v2
46
+ env:
47
+ SLACK_COLOR: 'danger'
48
+ SLACK_FOOTER: ''
49
+ SLACK_ICON_EMOJI: ':github-actions:'
50
+ SLACK_TITLE: 'Integrations Documentation generation failed:'
51
+ SLACK_USERNAME: 'GitHub Actions'
52
+ SLACK_MESSAGE: |-
53
+ ${{ github.repository }}: Failed to create PR generating documentation from integrations.js
54
+ Checkout Agent: ${{ steps.checkout-agent.outcome }}
55
+ Generate Integrations: ${{ steps.generate.outcome }}
56
+ Create PR: ${{ steps.create-pr.outcome }}
57
+ SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
58
+ if: >-
59
+ ${{
60
+ failure()
61
+ && startsWith(github.ref, 'refs/heads/master')
62
+ && github.repository == 'netdata/netdata'
63
+ }}
integrations/gen_docs_integrations.py
new
+276
@@ -0,0 +1,276 @@
1
+import json
2
+import os
3
+
4
+# Dictionary responsible for making the symbolic links at the end of the script's run.
5
+symlink_dict = {}
6
+
7
+
8
+def generate_category_from_name(category_fragment, category_array):
9
+ """
10
+ Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects.
11
+ """
12
+
13
+ category_name = ""
14
+ i = 0
15
+ dummy_id = category_fragment[0]
16
+
17
+ while i < len(category_fragment):
18
+ for category in category_array:
19
+
20
+ if dummy_id == category['id']:
21
+ category_name = category_name + "/" + category["name"]
22
+ try:
23
+ # print("equals")
24
+ # print(fragment, category_fragment[i+1])
25
+ dummy_id = dummy_id + "." + category_fragment[i+1]
26
+ # print(dummy_id)
27
+ except IndexError:
28
+ return category_name.split("/", 1)[1]
29
+ category_array = category['children']
30
+ break
31
+ i += 1
32
+
33
+
34
+def clean_and_write(md, txt):
35
+ """
36
+ This function takes care of the special details element, and converts it to the equivalent that md expects.
37
+ Then it writes the buffer on the file provided.
38
+ """
39
+ # clean first, replace
40
+ md = md.replace("{% details summary=\"", "<details><summary>").replace(
41
+ "\" %}", "</summary>\n").replace("{% /details %}", "</details>\n")
42
+ # print(md)
43
+ # exit()
44
+
45
+ txt.write(md)
46
+
47
+
48
+# Open integrations/integrations.js and extract the dictionaries
49
+with open('integrations/integrations.js') as dataFile:
50
+ data = dataFile.read()
51
+
52
+ categories_str = data.split("export const categories = ")[1].split("export const integrations = ")[0]
53
+ integrations_str = data.split("export const categories = ")[1].split("export const integrations = ")[1]
54
+
55
+ categories = json.loads(categories_str)
56
+ integrations = json.loads(integrations_str)
57
+
58
+i = 0
59
+# Iterate through every integration
60
+for integration in integrations:
61
+ i += 1
62
+ if integration['integration_type'] == "collector":
63
+
64
+ try:
65
+ # initiate the variables for the collector
66
+ meta_yaml = integration['edit_link'].replace("blob", "edit")
67
+ sidebar_label = integration['meta']['monitored_instance']['name']
68
+ learn_rel_path = generate_category_from_name(
69
+ integration['meta']['monitored_instance']['categories'][0].split("."), categories)
70
+ # build the markdown string
71
+ md = \
72
+ f"""<!--startmeta
73
+meta_yaml: "{meta_yaml}"
74
+sidebar_label: "{sidebar_label}"
75
+learn_status: "Published"
76
+learn_rel_path: "{learn_rel_path}"
77
+message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE"
78
+endmeta-->
79
+
80
+{integration['overview']}
81
+"""
82
+
83
+ if integration['metrics']:
84
+ md += f"""
85
+{integration['metrics']}
86
+"""
87
+
88
+ if integration['alerts']:
89
+ md += f"""
90
+{integration['alerts']}
91
+"""
92
+
93
+ if integration['setup']:
94
+ md += f"""
95
+{integration['setup']}
96
+"""
97
+
98
+ if integration['troubleshooting']:
99
+ md += f"""
100
+{integration['troubleshooting']}
101
+"""
102
+
103
+ path = meta_yaml.replace("https://github.com/netdata/", "") \
104
+ .split("/", 1)[1] \
105
+ .replace("edit/master/", "") \
106
+ .replace("/metadata.yaml", "")
107
+
108
+ # Only if the path exists, this caters for running the same script on both the go and netdata repos.
109
+ if os.path.exists(path):
110
+ try:
111
+ if not os.path.exists(f'{path}/integrations'):
112
+ os.mkdir(f'{path}/integrations')
113
+
114
+ with open(f'{path}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md', 'w+') as txt:
115
+ # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
116
+ # afterwards the ingest will replace this metadata with meta_yaml
117
+ md = md.replace(
118
+ "<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("/metadata.yaml", "")}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md\"')
119
+
120
+ clean_and_write(md, txt)
121
+ except Exception as e:
122
+ print("Error in writing to the collector file", e, integration['id'])
123
+
124
+ # If we only created one file inside a collector, add the entry to the symlink_dict, so we can make the link
125
+ if len(os.listdir(f'{path}/integrations')) == 1:
126
+ symlink_dict.update(
127
+ {path: f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md'})
128
+ else:
129
+ try:
130
+ symlink_dict.pop(path)
131
+ except KeyError:
132
+ # We don't need to print something here.
133
+ pass
134
+
135
+ except Exception as e:
136
+ print("Exception in collector md construction", e, integration['id'])
137
+
138
+ # kind of specific if clause, so we can avoid running excessive code in the go repo
139
+ elif integration['integration_type'] == "exporter" and "go.d.plugin" not in os.getcwd():
140
+ try:
141
+ # initiate the variables for the exporter
142
+ meta_yaml = integration['edit_link'].replace("blob", "edit")
143
+ sidebar_label = integration['meta']['name']
144
+ learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
145
+ # build the markdown string
146
+ md = \
147
+ f"""<!--startmeta
148
+meta_yaml: "{meta_yaml}"
149
+sidebar_label: "{sidebar_label}"
150
+learn_status: "Published"
151
+learn_rel_path: "Exporting"
152
+message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE EXPORTER'S metadata.yaml FILE"
153
+endmeta-->
154
+
155
+{integration['overview']}
156
+"""
157
+
158
+ if integration['setup']:
159
+ md += f"""
160
+{integration['setup']}
161
+"""
162
+
163
+ if integration['troubleshooting']:
164
+ md += f"""
165
+{integration['troubleshooting']}
166
+"""
167
+
168
+ path = meta_yaml.replace("https://github.com/netdata/", "") \
169
+ .split("/", 1)[1] \
170
+ .replace("edit/master/", "") \
171
+ .replace("/metadata.yaml", "")
172
+
173
+ if os.path.exists(path):
174
+ try:
175
+ if not os.path.exists(f'{path}/integrations'):
176
+ os.mkdir(f'{path}/integrations')
177
+
178
+ with open(f'{path}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md', 'w+') as txt:
179
+ # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
180
+ # afterwards the ingest will replace this metadata with meta_yaml
181
+ md = md.replace(
182
+ "<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("/metadata.yaml", "")}/integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md\"')
183
+
184
+ clean_and_write(md, txt)
185
+ except Exception as e:
186
+ print("Error in writing to the file", e, integration['id'])
187
+
188
+ # If we only created one file inside a collector, add the entry to the symlink_dict, so we can make the link
189
+ if len(os.listdir(f'{path}/integrations')) == 1:
190
+ symlink_dict.update(
191
+ {path: f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md'})
192
+ else:
193
+ try:
194
+ symlink_dict.pop(path)
195
+ except KeyError:
196
+ # We don't need to print something here.
197
+ pass
198
+ except Exception as e:
199
+ print("Exception in exporter md construction", e, integration['id'])
200
+
201
+ # kind of specific if clause, so we can avoid running excessive code in the go repo
202
+ elif integration['integration_type'] == "notification" and "go.d.plugin" not in os.getcwd():
203
+ try:
204
+ # initiate the variables for the notification method
205
+ meta_yaml = integration['edit_link'].replace("blob", "edit")
206
+ sidebar_label = integration['meta']['name']
207
+ learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
208
+ # build the markdown string
209
+ md = \
210
+ f"""<!--startmeta
211
+meta_yaml: "{meta_yaml}"
212
+sidebar_label: "{sidebar_label}"
213
+learn_status: "Published"
214
+learn_rel_path: "{learn_rel_path.replace("notifications", "Alerting/Notifications")}"
215
+message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE"
216
+endmeta-->
217
+
218
+{integration['overview']}
219
+"""
220
+
221
+ if integration['setup']:
222
+ md += f"""
223
+{integration['setup']}
224
+"""
225
+
226
+ if integration['troubleshooting']:
227
+ md += f"""
228
+{integration['troubleshooting']}
229
+"""
230
+
231
+ path = meta_yaml.replace("https://github.com/netdata/", "") \
232
+ .split("/", 1)[1] \
233
+ .replace("edit/master/", "") \
234
+ .replace("/metadata.yaml", "")
235
+
236
+ if "cloud-notifications" in path:
237
+ # for cloud notifications we generate them near their metadata.yaml
238
+ name = integration['meta']['name'].lower().replace(" ", "_")
239
+ if not os.path.exists(f'{path}/integrations'):
240
+ os.mkdir(f'{path}/integrations')
241
+
242
+ proper_edit_name = meta_yaml.replace(
243
+ "metadata.yaml", f'integrations/{sidebar_label.lower().replace(" ", "_").replace("/", "-")}.md\"')
244
+
245
+ md = md.replace("<!--startmeta", f'<!--startmeta\ncustom_edit_url: \"{proper_edit_name}')
246
+
247
+ finalpath = f'{path}/integrations/{name}.md'
248
+ else:
249
+ # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
250
+ # afterwards the ingest will replace this metadata with meta_yaml
251
+ md = md.replace("<!--startmeta",
252
+ f'<!--startmeta\ncustom_edit_url: \"{meta_yaml.replace("metadata.yaml", "README.md")}')
253
+ finalpath = f'{path}/README.md'
254
+ try:
255
+ with open(finalpath, 'w') as txt:
256
+ clean_and_write(md, txt)
257
+ except Exception as e:
258
+ print("Exception in notification md construction", e, integration['id'])
259
+
260
+ except Exception as e:
261
+ print("Exception in for loop", e, "\n", integration)
262
+
263
+for element in symlink_dict:
264
+ # Remove the README to prevent it being a normal file
265
+ os.remove(f'{element}/README.md')
266
+ # and then make a symlink to the actual markdown
267
+ os.symlink(symlink_dict[element], f'{element}/README.md')
268
+
269
+ with open(f'{element}/{symlink_dict[element]}', 'r') as txt:
270
+ md = txt.read()
271
+
272
+ # This preserves the custom_edit_url for most files as it was,
273
+ # so the existing links don't break, this is vital for link replacement afterwards
274
+ with open(f'{element}/{symlink_dict[element]}', 'w+') as txt:
275
+ md = md.replace(f'{element}/{symlink_dict[element]}', f'{element}/README.md')
276
+ txt.write(md)