| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Check that every Buildkite step the CI generator reads declares a key. |
| 4 | |
| 5 | A step with no `key:` is published under one the generator derives from its |
| 6 | label, which makes the job's address a function of its title: retitling the |
| 7 | step silently renames it and breaks retries, `depends_on:` and build history. |
| 8 | The generator only reads the directories a pipeline config names in |
| 9 | `job_dirs:`, so this walks the same files and fails on any step without a key. |
| 10 | |
| 11 | Usage: |
| 12 | python tools/pre_commit/check_buildkite_step_keys.py |
| 13 | """ |
| 14 | |
| 15 | import sys |
| 16 | from pathlib import Path |
| 17 | |
| 18 | import yaml |
| 19 | |
| 20 | BUILDKITE = Path(".buildkite") |
| 21 | CONFIG_GLOB = "ci_config*.yaml" |
| 22 | JOB_GLOB = "*.yaml" |
| 23 | |
| 24 | |
| 25 | def job_dirs() -> list[Path]: |
| 26 | """Every directory the pipeline generator reads, per the pipeline configs.""" |
| 27 | configs = sorted(BUILDKITE.glob(CONFIG_GLOB)) |
| 28 | if not configs: |
| 29 | raise SystemExit(f"no pipeline config matched {BUILDKITE / CONFIG_GLOB}") |
| 30 | dirs: set[Path] = set() |
| 31 | for config in configs: |
| 32 | with open(config, encoding="utf-8") as f: |
| 33 | doc = yaml.safe_load(f) |
| 34 | declared = doc.get("job_dirs") if isinstance(doc, dict) else None |
| 35 | if not isinstance(declared, list) or not declared: |
| 36 | raise SystemExit(f"{config}: `job_dirs` is missing or not a list") |
| 37 | for name in declared: |
| 38 | job_dir = Path(name) |
| 39 | if not job_dir.is_dir(): |
| 40 | raise SystemExit(f"{config}: job_dir `{name}` does not exist") |
| 41 | dirs.add(job_dir) |
| 42 | return sorted(dirs) |
| 43 | |
| 44 | |
| 45 | def scan(steps: object, source: Path) -> list[str]: |
| 46 | """Label of every step in a `steps:` list that declares no key.""" |
| 47 | if not isinstance(steps, list): |
| 48 | raise SystemExit(f"{source}: `steps:` is not a list") |
| 49 | keyless: list[str] = [] |
| 50 | for i, step in enumerate(steps): |
| 51 | # A scalar shorthand such as `- wait` has nowhere to put a key. Every |
| 52 | # other step does, so there are no other exemptions. A `mirror:` block |
| 53 | # is not a step here: the generator publishes it as `<hw>-<parent key>`. |
| 54 | if not isinstance(step, dict): |
| 55 | continue |
| 56 | label = step.get("label") or step.get("group") or f"step {i}" |
| 57 | if "steps" in step: |
| 58 | raise SystemExit( |
| 59 | f"{source}: `{label}` nests its own `steps:`. The generator has " |
| 60 | "no group step and drops the children without an error, so give " |
| 61 | "each one its own entry." |
| 62 | ) |
| 63 | key = step.get("key") |
| 64 | if not (isinstance(key, str) and key.strip()): |
| 65 | keyless.append(label) |
| 66 | return keyless |
| 67 | |
| 68 | |
| 69 | def keyless_steps(path: Path) -> list[str]: |
| 70 | with open(path, encoding="utf-8") as f: |
| 71 | doc = yaml.safe_load(f) |
| 72 | if not isinstance(doc, dict) or "steps" not in doc: |
| 73 | return [] |
| 74 | return scan(doc["steps"], path) |
| 75 | |
| 76 | |
| 77 | def main() -> int: |
| 78 | files = sorted({p for d in job_dirs() for p in d.rglob(JOB_GLOB)}) |
| 79 | if not files: |
| 80 | raise SystemExit("no pipeline files under the declared job_dirs") |
| 81 | keyless = [(path, label) for path in files for label in keyless_steps(path)] |
| 82 | if not keyless: |
| 83 | return 0 |
| 84 | print(f"{len(keyless)} Buildkite step(s) have no explicit key:\n", file=sys.stderr) |
| 85 | for path, label in keyless: |
| 86 | print(f" {path}: {label}", file=sys.stderr) |
| 87 | print( |
| 88 | "\nAdd a `key:` naming the test each step runs. Without one the pipeline" |
| 89 | "\ngenerator derives the key from the label, so retitling the step renames" |
| 90 | "\nits job and breaks retries and build history.", |
| 91 | file=sys.stderr, |
| 92 | ) |
| 93 | return 1 |
| 94 | |
| 95 | |
| 96 | if __name__ == "__main__": |
| 97 | raise SystemExit(main()) |