@cryptotaxi247 / netdata
Code Commits Issues Pulls Sessions CI/CD
master
master
py 160 lines 4.28 KB
Raw
1 import json
2 import os
3 from pathlib import Path
4
5 from jsonschema import Draft7Validator, ValidationError
6 from referencing import Registry, Resource
7 from referencing.jsonschema import DRAFT7
8 from ruamel.yaml import YAML, YAMLError
9
10
11 AGENT_REPO = 'netdata/netdata'
12
13 INTEGRATIONS_PATH = Path(__file__).parent
14 REPO_PATH = INTEGRATIONS_PATH.parent
15 SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas'
16 METADATA_PATTERN = '*/metadata.yaml'
17
18 COLLECTOR_SOURCES = [
19 (AGENT_REPO, REPO_PATH / 'src' / 'collectors', True),
20 (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'charts.d.plugin', True),
21 (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'python.d.plugin', True),
22 (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'guides', True),
23 (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'go.d' / 'collector', True),
24 (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'scripts.d' / 'collector', True),
25 (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'ibm.d' / 'modules', True),
26 (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'ibm.d' / 'modules' / 'websphere', True),
27 (AGENT_REPO, REPO_PATH / 'src' / 'crates' / 'netdata-otel', True),
28 ]
29
30 GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', False)
31 DEBUG = os.environ.get('DEBUG', False)
32 WARNINGS = []
33
34
35 def debug(msg):
36 if GITHUB_ACTIONS:
37 print(f'::debug::{msg}')
38 elif DEBUG:
39 print(f'>>> {msg}')
40 else:
41 pass
42
43
44 def warn(msg, path):
45 WARNINGS.append((str(path), msg))
46
47 if GITHUB_ACTIONS:
48 print(f'::warning file={path}::{msg}')
49 else:
50 print(f'!!! WARNING:{path}:{msg}')
51
52
53 def fail_on_warnings():
54 if not WARNINGS:
55 return 0
56
57 warned_files = sorted({path for path, _ in WARNINGS})
58 print(f'::error::Integrations generation failed with {len(WARNINGS)} warning(s) across {len(warned_files)} file(s).')
59
60 for path in warned_files:
61 print(f'::error file={path}::Metadata warnings in this file are now fatal for integrations generation.')
62
63 return 1
64
65
66 def retrieve_from_filesystem(uri):
67 path = SCHEMA_PATH / Path(uri)
68 contents = json.loads(path.read_text())
69 return Resource.from_contents(contents, DRAFT7)
70
71
72 registry = Registry(retrieve=retrieve_from_filesystem)
73
74
75 def make_validator(schema_ref):
76 return Draft7Validator(
77 {'$ref': schema_ref},
78 registry=registry,
79 )
80
81
82 COLLECTOR_VALIDATOR = make_validator('./collector.json#')
83
84
85 def get_collector_metadata_entries():
86 ret = []
87
88 for r, d, m in COLLECTOR_SOURCES:
89 if d.exists() and d.is_dir() and m:
90 for item in d.glob(METADATA_PATTERN):
91 ret.append((r, item))
92 elif d.exists() and d.is_file() and not m:
93 if d.match(METADATA_PATTERN):
94 ret.append((r, d))
95
96 return ret
97
98
99 def load_yaml(src):
100 yaml = YAML(typ='safe')
101
102 if not src.is_file():
103 warn(f'{src} is not a file.', src)
104 return False
105
106 try:
107 contents = src.read_text()
108 except (IOError, OSError):
109 warn(f'Failed to read {src}.', src)
110 return False
111
112 try:
113 data = yaml.load(contents)
114 except YAMLError:
115 warn(f'Failed to parse {src} as YAML.', src)
116 return False
117
118 return data
119
120
121 def load_collectors():
122 ret = []
123
124 entries = get_collector_metadata_entries()
125
126 for repo, path in entries:
127 debug(f'Loading {path}.')
128 data = load_yaml(path)
129
130 if not data:
131 continue
132
133 try:
134 COLLECTOR_VALIDATOR.validate(data)
135 except ValidationError as e:
136 warn(
137 f'Failed to validate {path} against the schema: {e.message} (path: {"/".join(str(p) for p in e.absolute_path)})',
138 path)
139 continue
140
141 for idx, item in enumerate(data['modules']):
142 item['meta']['plugin_name'] = data['plugin_name']
143 item['integration_type'] = 'collector'
144 item['_src_path'] = path
145 item['_repo'] = repo
146 item['_index'] = idx
147 ret.append(item)
148
149 return ret
150
151
152 def make_id(meta):
153 if 'monitored_instance' in meta:
154 instance_name = meta['monitored_instance']['name'].replace(' ', '_')
155 elif 'instance_name' in meta:
156 instance_name = meta['instance_name']
157 else:
158 instance_name = '000_unknown'
159
160 return f'{meta["plugin_name"]}-{meta["module_name"]}-{instance_name}'