| 1 | import json |
| 2 | import os |
| 3 | from pathlib import Path |
| 4 | |
| 5 | from jsonschema import Draft7Validator, ValidationError |
| 6 | from referencing import Registry, Resource |
| 7 | from referencing.jsonschema import DRAFT7 |
| 8 | from ruamel.yaml import YAML, YAMLError |
| 9 | |
| 10 | |
| 11 | AGENT_REPO = 'netdata/netdata' |
| 12 | |
| 13 | INTEGRATIONS_PATH = Path(__file__).parent |
| 14 | REPO_PATH = INTEGRATIONS_PATH.parent |
| 15 | SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas' |
| 16 | METADATA_PATTERN = '*/metadata.yaml' |
| 17 | |
| 18 | COLLECTOR_SOURCES = [ |
| 19 | (AGENT_REPO, REPO_PATH / 'src' / 'collectors', True), |
| 20 | (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'charts.d.plugin', True), |
| 21 | (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'python.d.plugin', True), |
| 22 | (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'guides', True), |
| 23 | (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'go.d' / 'collector', True), |
| 24 | (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'scripts.d' / 'collector', True), |
| 25 | (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'ibm.d' / 'modules', True), |
| 26 | (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'plugin' / 'ibm.d' / 'modules' / 'websphere', True), |
| 27 | (AGENT_REPO, REPO_PATH / 'src' / 'crates' / 'netdata-otel', True), |
| 28 | ] |
| 29 | |
| 30 | GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', False) |
| 31 | DEBUG = os.environ.get('DEBUG', False) |
| 32 | WARNINGS = [] |
| 33 | |
| 34 | |
| 35 | def debug(msg): |
| 36 | if GITHUB_ACTIONS: |
| 37 | print(f'::debug::{msg}') |
| 38 | elif DEBUG: |
| 39 | print(f'>>> {msg}') |
| 40 | else: |
| 41 | pass |
| 42 | |
| 43 | |
| 44 | def warn(msg, path): |
| 45 | WARNINGS.append((str(path), msg)) |
| 46 | |
| 47 | if GITHUB_ACTIONS: |
| 48 | print(f'::warning file={path}::{msg}') |
| 49 | else: |
| 50 | print(f'!!! WARNING:{path}:{msg}') |
| 51 | |
| 52 | |
| 53 | def fail_on_warnings(): |
| 54 | if not WARNINGS: |
| 55 | return 0 |
| 56 | |
| 57 | warned_files = sorted({path for path, _ in WARNINGS}) |
| 58 | print(f'::error::Integrations generation failed with {len(WARNINGS)} warning(s) across {len(warned_files)} file(s).') |
| 59 | |
| 60 | for path in warned_files: |
| 61 | print(f'::error file={path}::Metadata warnings in this file are now fatal for integrations generation.') |
| 62 | |
| 63 | return 1 |
| 64 | |
| 65 | |
| 66 | def retrieve_from_filesystem(uri): |
| 67 | path = SCHEMA_PATH / Path(uri) |
| 68 | contents = json.loads(path.read_text()) |
| 69 | return Resource.from_contents(contents, DRAFT7) |
| 70 | |
| 71 | |
| 72 | registry = Registry(retrieve=retrieve_from_filesystem) |
| 73 | |
| 74 | |
| 75 | def make_validator(schema_ref): |
| 76 | return Draft7Validator( |
| 77 | {'$ref': schema_ref}, |
| 78 | registry=registry, |
| 79 | ) |
| 80 | |
| 81 | |
| 82 | COLLECTOR_VALIDATOR = make_validator('./collector.json#') |
| 83 | |
| 84 | |
| 85 | def get_collector_metadata_entries(): |
| 86 | ret = [] |
| 87 | |
| 88 | for r, d, m in COLLECTOR_SOURCES: |
| 89 | if d.exists() and d.is_dir() and m: |
| 90 | for item in d.glob(METADATA_PATTERN): |
| 91 | ret.append((r, item)) |
| 92 | elif d.exists() and d.is_file() and not m: |
| 93 | if d.match(METADATA_PATTERN): |
| 94 | ret.append((r, d)) |
| 95 | |
| 96 | return ret |
| 97 | |
| 98 | |
| 99 | def load_yaml(src): |
| 100 | yaml = YAML(typ='safe') |
| 101 | |
| 102 | if not src.is_file(): |
| 103 | warn(f'{src} is not a file.', src) |
| 104 | return False |
| 105 | |
| 106 | try: |
| 107 | contents = src.read_text() |
| 108 | except (IOError, OSError): |
| 109 | warn(f'Failed to read {src}.', src) |
| 110 | return False |
| 111 | |
| 112 | try: |
| 113 | data = yaml.load(contents) |
| 114 | except YAMLError: |
| 115 | warn(f'Failed to parse {src} as YAML.', src) |
| 116 | return False |
| 117 | |
| 118 | return data |
| 119 | |
| 120 | |
| 121 | def load_collectors(): |
| 122 | ret = [] |
| 123 | |
| 124 | entries = get_collector_metadata_entries() |
| 125 | |
| 126 | for repo, path in entries: |
| 127 | debug(f'Loading {path}.') |
| 128 | data = load_yaml(path) |
| 129 | |
| 130 | if not data: |
| 131 | continue |
| 132 | |
| 133 | try: |
| 134 | COLLECTOR_VALIDATOR.validate(data) |
| 135 | except ValidationError as e: |
| 136 | warn( |
| 137 | f'Failed to validate {path} against the schema: {e.message} (path: {"/".join(str(p) for p in e.absolute_path)})', |
| 138 | path) |
| 139 | continue |
| 140 | |
| 141 | for idx, item in enumerate(data['modules']): |
| 142 | item['meta']['plugin_name'] = data['plugin_name'] |
| 143 | item['integration_type'] = 'collector' |
| 144 | item['_src_path'] = path |
| 145 | item['_repo'] = repo |
| 146 | item['_index'] = idx |
| 147 | ret.append(item) |
| 148 | |
| 149 | return ret |
| 150 | |
| 151 | |
| 152 | def make_id(meta): |
| 153 | if 'monitored_instance' in meta: |
| 154 | instance_name = meta['monitored_instance']['name'].replace(' ', '_') |
| 155 | elif 'instance_name' in meta: |
| 156 | instance_name = meta['instance_name'] |
| 157 | else: |
| 158 | instance_name = '000_unknown' |
| 159 | |
| 160 | return f'{meta["plugin_name"]}-{meta["module_name"]}-{instance_name}' |