update integrations gen script (#19085)
Ilya Mashchenko committed
Nov 25, 2024 at 21:12 UTC
e848a47675f6bf2756c8aad305fb0fac35daa47d
4 files changed
+23
-19
integrations/check_collector_metadata.py
+9
-9
@@ -1,7 +1,6 @@
1
#!/usr/bin/env python3
2
3
import sys
4
-
4
from pathlib import Path
5
6
from jsonschema import ValidationError
@@ -18,17 +17,17 @@ def main():
17
check_path = Path(sys.argv[1])
18
19
if not check_path.is_file():
21
- print(f':error file={ check_path }:{ check_path } does not appear to be a regular file.')
20
+ print(f':error file={check_path}:{check_path} does not appear to be a regular file.')
21
return 1
22
23
if check_path.match(SINGLE_PATTERN):
24
variant = 'single'
26
- print(f':debug:{ check_path } appears to be single-module metadata.')
25
+ print(f':debug:{check_path} appears to be single-module metadata.')
26
elif check_path.match(MULTI_PATTERN):
27
variant = 'multi'
29
- print(f':debug:{ check_path } appears to be multi-module metadata.')
28
+ print(f':debug:{check_path} appears to be multi-module metadata.')
29
else:
31
- print(f':error file={ check_path }:{ check_path } does not match required file name format.')
30
+ print(f':error file={check_path}:{check_path} does not match required file name format.')
31
return 1
32
33
categories = load_yaml(CATEGORIES_FILE)
@@ -42,7 +41,7 @@ def main():
41
data = load_yaml(check_path)
42
43
if not data:
45
- print(f':error file={ check_path }:Failed to load data from { check_path }.')
44
+ print(f':error file={check_path}:Failed to load data from {check_path}.')
45
return 1
46
47
check_modules = []
@@ -51,7 +50,7 @@ def main():
50
try:
51
SINGLE_VALIDATOR.validate(data)
52
except ValidationError as e:
54
- print(f':error file={ check_path }:Failed to validate { check_path } against the schema.')
53
+ print(f':error file={check_path}:Failed to validate {check_path} against the schema.')
54
raise e
55
else:
56
check_modules.append(data)
@@ -59,7 +58,7 @@ def main():
58
try:
59
MULTI_VALIDATOR.validate(data)
60
except ValidationError as e:
62
- print(f':error file={ check_path }:Failed to validate { check_path } against the schema.')
61
+ print(f':error file={check_path}:Failed to validate {check_path} against the schema.')
62
raise e
63
else:
64
for item in data['modules']:
@@ -75,7 +74,8 @@ def main():
74
invalid_cats = set(module['meta']['monitored_instance']['categories']) - valid_categories
75
76
if invalid_cats:
78
- print(f':error file={ check_path }:Invalid categories found in module { idx } in { check_path }: { ", ".joiin(invalid_cats) }.')
77
+ print(
78
+ f':error file={check_path}:Invalid categories found in module {idx} in {check_path}: {", ".join(invalid_cats)}.')
79
failed = True
80
81
if failed:
integrations/gen_doc_collector_page.py
+7
-5
@@ -42,7 +42,6 @@ for integration in integrations:
42
if integration['integration_type'] == "collector":
43
construct_dict(integration['meta']['monitored_instance']['categories'], integration)
44
45
-
45
for category_id, integrations in sorted(cat_dict.items()):
46
heading = '#' * len(category_id.split('.'))
47
@@ -54,14 +53,17 @@ for category_id, integrations in sorted(cat_dict.items()):
53
names = []
54
for integration in integrations:
55
name = integration['meta']['monitored_instance']['name']
57
- link = integration['edit_link'].replace("metadata.yaml", "") + \
58
- "integrations/" + name.lower().replace(" ", "_").replace("/", "-").replace("(", "").replace(")", "")+".md"
56
+ link = (integration['edit_link'].replace("metadata.yaml", "") +
57
+ "integrations/" + name.lower().
58
+ replace(" ", "_").
59
+ replace("/", "-").
60
+ replace("(", "").
61
+ replace(")", "") + ".md")
62
names.append(f"[{name}]({link})")
63
for integration_name in sorted(names):
64
md += "- " + integration_name + "\n\n"
65
63
-
66
outfile = pathlib.Path("./src/collectors/COLLECTORS.md")
67
output = outfile.read_text().split("## Available Data Collection Integrations")[0]
68
output += "## Available Data Collection Integrations\n<!-- AUTOGENERATED PART BY integrations/gen_doc_collector_page.py SCRIPT, DO NOT EDIT MANUALLY -->\n" + md
67
-outfile.write_text(output.rstrip('\n')+"\n")
69
+outfile.write_text(output.rstrip('\n') + "\n")
integrations/gen_docs_integrations.py
+1
-1
@@ -11,7 +11,7 @@ def cleanup():
11
"""
12
clean directories that are either data collection or exporting integrations
13
"""
14
- for element in Path("src/go/plugin/go.d/modules").glob('**/*/'):
14
+ for element in Path("src/go/plugin/go.d/collector").glob('**/*/'):
15
if "integrations" in str(element):
16
shutil.rmtree(element)
17
for element in Path("src/collectors").glob('**/*/'):
integrations/gen_integrations.py
+6
-4
@@ -28,7 +28,7 @@ COLLECTOR_SOURCES = [
28
(AGENT_REPO, REPO_PATH / 'src' / 'collectors', True),
29
(AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'charts.d.plugin', True),
30
(AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'python.d.plugin', True),
31
- (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'go.d' / 'modules', True),
31
+ (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'go.d' / 'collector', True),
32
]
33
34
DEPLOY_SOURCES = [
@@ -998,9 +998,11 @@ def main():
998
collectors, clean_collectors, ids = render_collectors(categories, collectors, dict())
999
deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids)
1000
exporters, clean_exporters, ids = render_exporters(categories, exporters, ids)
1001
- agent_notifications, clean_agent_notifications, ids = render_agent_notifications(categories, agent_notifications,ids)
1002
- cloud_notifications, clean_cloud_notifications, ids = render_cloud_notifications(categories, cloud_notifications,ids)
1003
- logs, clean_logs, ids = render_logs(categories, logs,ids)
1001
+ agent_notifications, clean_agent_notifications, ids = render_agent_notifications(categories, agent_notifications,
1002
+ ids)
1003
+ cloud_notifications, clean_cloud_notifications, ids = render_cloud_notifications(categories, cloud_notifications,
1004
+ ids)
1005
+ logs, clean_logs, ids = render_logs(categories, logs, ids)
1006
authentications, clean_authentications, ids = render_authentications(categories, authentications, ids)
1007
1008
integrations = collectors + deploy + exporters + agent_notifications + cloud_notifications + logs + authentications