Add integrations JSON file for website usage. (#15959)
* Add generation of JSON integrations data. * Clean up generated data for JSON file. * Properly clean up deploy entry info. * Fix argument order for regex substitutions.
Austin S. Hemmelgarn committed
Oct 18, 2023 at 11:30 UTC
8934a18ce48da9307f4e9ee6ccf3e38664189798
5 files changed
+117
-39
integrations/deploy.yaml
+18
-3
@@ -35,6 +35,8 @@
35
--stable-channel{% if $showClaimingOptions %} --claim-token {% claim_token %} --claim-rooms {% $claim_rooms %} --claim-url {% claim_url %}{% /if %}
36
additional_info: &ref_containers >
37
Did you know you can also deploy Netdata on your OS using {% goToCategory navigateToSettings=$navigateToSettings categoryId="deploy.docker-kubernetes" %}Kubernetes{% /goToCategory %} or {% goToCategory categoryId="deploy.docker-kubernetes" %}Docker{% /goToCategory %}?
38
+ clean_additional_info: &ref_clean_containers >
39
+ Did you know you can also deploy Netdata on your OS using Kubernetes or Docker?
40
related_resources: {}
41
platform_info:
42
group: ''
@@ -85,7 +87,7 @@
87
icon_filename: 'rhel.png'
88
most_popular: false
89
platform_info:
88
- group: 'include'
90
+ group: 'no_include'
91
distro: 'rhel'
92
quick_start: -1
93
- <<: *linux
@@ -148,6 +150,18 @@
150
group: 'include'
151
distro: 'centos'
152
quick_start: -1
153
+- <<: *linux
154
+ id: deploy-centos-stream
155
+ meta:
156
+ <<: *linux_meta
157
+ name: CentOS Stream
158
+ link: https://www.centos.org/centos-stream
159
+ icon_filename: 'centos.png'
160
+ most_popular: false
161
+ platform_info:
162
+ group: 'include'
163
+ distro: 'centos-stream'
164
+ quick_start: -1
165
- <<: *linux
166
id: deploy-manjarolinux
167
meta:
@@ -200,9 +214,10 @@
214
methods:
215
- *ks_curl
216
additional_info: *ref_containers
217
+ clean_additional_info: *ref_clean_containers
218
related_resources: {}
219
platform_info:
205
- group: 'include'
220
+ group: 'no_include'
221
distro: 'macos'
222
quick_start: 5
223
- id: deploy-docker
@@ -573,6 +588,6 @@
588
Netdata can also be installed via [FreeBSD ports](https://www.freshports.org/net-mgmt/netdata).
589
related_resources: {}
590
platform_info:
576
- group: 'include'
591
+ group: 'no_include'
592
distro: 'freebsd'
593
quick_start: 6
integrations/gen_integrations.py
+89
-30
@@ -2,8 +2,10 @@
2
3
import json
4
import os
5
+import re
6
import sys
7
8
+from copy import deepcopy
9
from pathlib import Path
10
11
from jsonschema import Draft7Validator, ValidationError
@@ -17,6 +19,7 @@ GO_REPO = 'netdata/go.d.plugin'
19
INTEGRATIONS_PATH = Path(__file__).parent
20
TEMPLATE_PATH = INTEGRATIONS_PATH / 'templates'
21
OUTPUT_PATH = INTEGRATIONS_PATH / 'integrations.js'
22
+JSON_PATH = INTEGRATIONS_PATH / 'integrations.json'
23
CATEGORIES_FILE = INTEGRATIONS_PATH / 'categories.yaml'
24
REPO_PATH = INTEGRATIONS_PATH.parent
25
SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas'
@@ -65,6 +68,9 @@ NOTIFICATION_RENDER_KEYS = [
68
'troubleshooting',
69
]
70
71
+CUSTOM_TAG_PATTERN = re.compile('\\{% if .*?%\\}.*?\\{% /if %\\}|\\{%.*?%\\}', flags=re.DOTALL)
72
+FIXUP_BLANK_PATTERN = re.compile('\\\\\\n *\\n')
73
+
74
GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', False)
75
DEBUG = os.environ.get('DEBUG', False)
76
@@ -438,12 +444,17 @@ def render_collectors(categories, collectors, ids):
444
debug('Removing duplicate collectors.')
445
446
collectors, ids = dedupe_integrations(collectors, ids)
447
+ clean_collectors = []
448
449
idmap = {i['id']: i for i in collectors}
450
451
for item in collectors:
452
debug(f'Processing { item["id"] }.')
453
454
+ item['edit_link'] = make_edit_link(item)
455
+
456
+ clean_item = deepcopy(item)
457
+
458
related = []
459
460
for res in item['meta']['related_resources']['integrations']['list']:
@@ -487,23 +498,27 @@ def render_collectors(categories, collectors, ids):
498
for key in COLLECTOR_RENDER_KEYS:
499
if key in item.keys():
500
template = get_jinja_env().get_template(f'{ key }.md')
490
- data = template.render(entry=item, related=related)
501
+ data = template.render(entry=item, related=related, clean=False)
502
+ clean_data = template.render(entry=item, related=related, clean=True)
503
504
if 'variables' in item['meta']['monitored_instance']:
505
template = get_jinja_env().from_string(data)
506
data = template.render(variables=item['meta']['monitored_instance']['variables'])
507
+ template = get_jinja_env().from_string(clean_data)
508
+ clean_data = template.render(variables=item['meta']['monitored_instance']['variables'])
509
else:
510
data = ''
511
+ clean_data = ''
512
513
item[key] = data
514
+ clean_item[key] = clean_data
515
500
- item['edit_link'] = make_edit_link(item)
516
+ for k in ['_src_path', '_repo', '_index']:
517
+ del item[k], clean_item[k]
518
502
- del item['_src_path']
503
- del item['_repo']
504
- del item['_index']
519
+ clean_collectors.append(clean_item)
520
506
- return collectors, ids
521
+ return collectors, clean_collectors, ids
522
523
524
def render_deploy(distros, categories, deploy, ids):
@@ -514,11 +529,14 @@ def render_deploy(distros, categories, deploy, ids):
529
debug('Checking deployment ids.')
530
531
deploy, ids = dedupe_integrations(deploy, ids)
532
+ clean_deploy = []
533
534
template = get_jinja_env().get_template('platform_info.md')
535
536
for item in deploy:
537
debug(f'Processing { item["id"] }.')
538
+ item['edit_link'] = make_edit_link(item)
539
+ clean_item = deepcopy(item)
540
541
if item['platform_info']['group']:
542
entries = [
@@ -532,16 +550,27 @@ def render_deploy(distros, categories, deploy, ids):
550
else:
551
entries = []
552
535
- data = template.render(entries=entries)
553
+ data = template.render(entries=entries, clean=False)
554
+ clean_data = template.render(entries=entries, clean=True)
555
+
556
+ for method in clean_item['methods']:
557
+ for command in method['commands']:
558
+ command['command'] = CUSTOM_TAG_PATTERN.sub('', command['command'])
559
+ command['command'] = FIXUP_BLANK_PATTERN.sub('', command['command'])
560
561
item['platform_info'] = data
538
- item['edit_link'] = make_edit_link(item)
562
+ clean_item['platform_info'] = clean_data
563
+
564
+ if 'clean_additional_info' in item:
565
+ clean_item['additional_info'] = item['clean_additional_info']
566
+ del item['clean_additional_info'], clean_item['clean_additional_info']
567
540
- del item['_src_path']
541
- del item['_repo']
542
- del item['_index']
568
+ for k in ['_src_path', '_repo', '_index']:
569
+ del item[k], clean_item[k]
570
544
- return deploy, ids
571
+ clean_deploy.append(clean_item)
572
+
573
+ return deploy, clean_deploy, ids
574
575
576
def render_exporters(categories, exporters, ids):
@@ -553,27 +582,37 @@ def render_exporters(categories, exporters, ids):
582
583
exporters, ids = dedupe_integrations(exporters, ids)
584
585
+ clean_exporters = []
586
+
587
for item in exporters:
588
+ item['edit_link'] = make_edit_link(item)
589
+
590
+ clean_item = deepcopy(item)
591
+
592
for key in EXPORTER_RENDER_KEYS:
593
if key in item.keys():
594
template = get_jinja_env().get_template(f'{ key }.md')
560
- data = template.render(entry=item)
595
+ data = template.render(entry=item, clean=False)
596
+ clean_data = template.render(entry=item, clean=True)
597
598
if 'variables' in item['meta']:
599
template = get_jinja_env().from_string(data)
564
- data = template.render(variables=item['meta']['variables'])
600
+ data = template.render(variables=item['meta']['variables'], clean=False)
601
+ template = get_jinja_env().from_string(clean_data)
602
+ clean_data = template.render(variables=item['meta']['variables'], clean=True)
603
else:
604
data = ''
605
+ clean_data = ''
606
607
item[key] = data
608
+ clean_item[key] = clean_data
609
570
- item['edit_link'] = make_edit_link(item)
610
+ for k in ['_src_path', '_repo', '_index']:
611
+ del item[k], clean_item[k]
612
572
- del item['_src_path']
573
- del item['_repo']
574
- del item['_index']
613
+ clean_exporters.append(clean_item)
614
576
- return exporters, ids
615
+ return exporters, clean_exporters, ids
616
617
618
def render_notifications(categories, notifications, ids):
@@ -585,27 +624,37 @@ def render_notifications(categories, notifications, ids):
624
625
notifications, ids = dedupe_integrations(notifications, ids)
626
627
+ clean_notifications = []
628
+
629
for item in notifications:
630
+ item['edit_link'] = make_edit_link(item)
631
+
632
+ clean_item = deepcopy(item)
633
+
634
for key in NOTIFICATION_RENDER_KEYS:
635
if key in item.keys():
636
template = get_jinja_env().get_template(f'{ key }.md')
592
- data = template.render(entry=item)
637
+ data = template.render(entry=item, clean=False)
638
+ clean_data = template.render(entry=item, clean=True)
639
640
if 'variables' in item['meta']:
641
template = get_jinja_env().from_string(data)
596
- data = template.render(variables=item['meta']['variables'])
642
+ data = template.render(variables=item['meta']['variables'], clean=False)
643
+ template = get_jinja_env().from_string(clean_data)
644
+ clean_data = template.render(variables=item['meta']['variables'], clean=True)
645
else:
646
data = ''
647
+ clean_data = ''
648
649
item[key] = data
650
+ clean_item[key] = clean_data
651
602
- item['edit_link'] = make_edit_link(item)
652
+ for k in ['_src_path', '_repo', '_index']:
653
+ del item[k], clean_item[k]
654
604
- del item['_src_path']
605
- del item['_repo']
606
- del item['_index']
655
+ clean_notifications.append(clean_item)
656
608
- return notifications, ids
657
+ return notifications, clean_notifications, ids
658
659
660
def render_integrations(categories, integrations):
@@ -617,6 +666,13 @@ def render_integrations(categories, integrations):
666
OUTPUT_PATH.write_text(data)
667
668
669
+def render_json(categories, integrations):
670
+ JSON_PATH.write_text(json.dumps({
671
+ 'categories': categories,
672
+ 'integrations': integrations,
673
+ }))
674
+
675
+
676
def main():
677
categories = load_categories()
678
distros = load_yaml(DISTROS_FILE)
@@ -625,14 +681,17 @@ def main():
681
exporters = load_exporters()
682
notifications = load_notifications()
683
628
- collectors, ids = render_collectors(categories, collectors, dict())
629
- deploy, ids = render_deploy(distros, categories, deploy, ids)
630
- exporters, ids = render_exporters(categories, exporters, ids)
631
- notifications, ids = render_notifications(categories, notifications, ids)
684
+ collectors, clean_collectors, ids = render_collectors(categories, collectors, dict())
685
+ deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids)
686
+ exporters, clean_exporters, ids = render_exporters(categories, exporters, ids)
687
+ notifications, clean_notifications, ids = render_notifications(categories, notifications, ids)
688
689
integrations = collectors + deploy + exporters + notifications
690
render_integrations(categories, integrations)
691
692
+ clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications
693
+ render_json(categories, clean_integrations)
694
+
695
696
if __name__ == '__main__':
697
sys.exit(main())
integrations/schemas/deploy.json
+4
@@ -68,6 +68,10 @@
68
"type": "string",
69
"description": "Any additional information about this platform."
70
},
71
+ "clean_additional_info": {
72
+ "type": "string",
73
+ "description": "Any additional information about this platform, without any embedded custom tags."
74
+ },
75
"related_resources": {
76
"type": "object",
77
"description": "TBD"
integrations/templates/metrics.md
+2
-2
@@ -1,7 +1,7 @@
1
[% if entry.metrics.scopes %]
2
## Metrics
3
4
-[% if entry.metrics.folding.enabled %]
4
+[% if entry.metrics.folding.enabled and not clean %]
5
{% details summary="[[ entry.metrics.folding.title ]]" %}
6
[% endif %]
7
Metrics grouped by *scope*.
@@ -39,7 +39,7 @@ Metrics:
39
[% endfor %]
40
41
[% endfor %]
42
-[% if entry.metrics.folding.enabled %]
42
+[% if entry.metrics.folding.enabled and not clean %]
43
{% /details %}
44
[% endif %]
45
[% else %]
integrations/templates/setup.md
+4
-4
@@ -59,7 +59,7 @@ There is no configuration file.
59
[[ entry.setup.configuration.options.description ]]
60
61
[% if entry.setup.configuration.options.list %]
62
-[% if entry.setup.configuration.options.folding.enabled %]
62
+[% if entry.setup.configuration.options.folding.enabled and not clean %]
63
{% details summary="[[ entry.setup.configuration.options.folding.title ]]" %}
64
[% endif %]
65
| Name | Description | Default | Required |
@@ -76,7 +76,7 @@ There is no configuration file.
76
77
[% endif %]
78
[% endfor %]
79
-[% if entry.setup.configuration.options.folding.enabled %]
79
+[% if entry.setup.configuration.options.folding.enabled and not clean %]
80
{% /details %}
81
[% endif %]
82
[% elif not entry.setup.configuration.options.description %]
@@ -91,13 +91,13 @@ There are no configuration options.
91
92
[[ example.description ]]
93
94
-[% if example.folding.enabled %]
94
+[% if example.folding.enabled and not clean %]
95
{% details summary="[[ entry.setup.configuration.examples.folding.title ]]" %}
96
[% endif %]
97
```yaml
98
[[ example.config ]]
99
```
100
-[% if example.folding.enabled %]
100
+[% if example.folding.enabled and not clean %]
101
{% /details %}
102
[% endif %]
103
[% endfor %]