@tduncan / Nvidia-Optimization-Stack-VLLM / commits / e9f169d16b

[CI] Add pre-commit check that new tests are tethered to Buildkite jobs (#54867)

Signed-off-by: Turner Jabbour <doubleujabbour@gmail.com> Signed-off-by: Turner <doubleujabbour@gmail.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>

Turner Jabbour committed Sep 21, 2026 at 19:36 UTC e9f169d16b9408bb9ae44f75072b91a5521d733c
8 files changed +1920
.buildkite/test_areas/misc.yaml
+4
@@ -525,6 +525,9 @@ steps:
525 - tests/renderers
526 - tests/standalone_tests/lazy_imports.py
527 - tests/reasoning
528 + - tools/pre_commit/check_test_tethering.py
529 + - tools/pre_commit/test_tethering_allowlist.txt
530 + - tests/tools/test_check_test_tethering.py
531 - tests/tool_parsers
532 - tests/tokenizers_
533 - tests/parser
@@ -552,6 +555,7 @@ steps:
555 commands:
556 - pytest -v -s -m 'cpu_test' multimodal
557 - pytest -v -s config
558 + - pytest -v -s tools/test_check_test_tethering.py
559
560 - label: ":computer: (CPU) Reasoning + Renderers"
561 key: cpu-reasoning-renderers
.pre-commit-config.yaml
+8
@@ -184,6 +184,14 @@ repos:
184 files: ^\.buildkite/.*\.ya?ml$
185 pass_filenames: false
186 additional_dependencies: [PyYAML]
187 + - id: check-test-tethering
188 + name: Check new tests are wired into Buildkite CI
189 + language: python
190 + entry: python tools/pre_commit/check_test_tethering.py
191 + files: ^(tests/.*\.py|\.buildkite/(test_areas|hardware_tests|intel_jobs)/.*\.yaml|\.buildkite/(test-amd|ci_config[^/]*)\.yaml|tools/pre_commit/test_tethering_allowlist\.txt)$
192 + additional_dependencies: [PyYAML, regex]
193 + verbose: true # full-scan `note:` lines are advisory, so the hook passes
194 + require_serial: true # one full scan, not one per pre-commit file batch
195 - id: mypy-3.10 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
196 name: Run mypy for Python 3.10
197 entry: python tools/pre_commit/mypy.py "3.10"
docker/Dockerfile.cpu
+4
@@ -471,6 +471,10 @@ ADD ./benchmarks/ ./benchmarks/
471 ADD ./vllm/collect_env.py .
472 ADD ./docker/ ./docker/
473 ADD ./.buildkite/ ./.buildkite/
474 +# tools/ carries repo-tooling code that a few tests exercise directly
475 +# (e.g. tests/tools/test_check_test_tethering.py imports
476 +# tools.pre_commit.check_test_tethering).
477 +ADD ./tools/ ./tools/
478
479 # install development dependencies (for testing)
480 RUN --mount=type=cache,target=/root/.cache/uv \
tests/tools/test_check_test_tethering.py new
+879
@@ -0,0 +1,879 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Unit tests for the test-tethering pre-commit check.
4 +
5 +The check answers one question - "does any Buildkite job (in
6 +``.buildkite/test_areas/`` or the legacy ``.buildkite/test-amd.yaml``) collect
7 +this test file?" - by parsing job ``commands`` as shell. The risk that
8 +matters is a *false tether*: a misparse that invents coverage makes the gate
9 +pass while the test still never runs, which is the exact failure the check
10 +exists to catch. So the parser cases below pin both directions, and
11 +:func:`test_no_selection_matches_everything` guards the catastrophic version of
12 +it.
13 +"""
14 +
15 +import textwrap
16 +from pathlib import Path
17 +
18 +import pytest
19 +
20 +import tools.pre_commit.check_test_tethering as checker
21 +from tools.pre_commit.check_test_tethering import (
22 + FindSelection,
23 + PytestSelection,
24 + _parse_command,
25 + _to_repo_relative,
26 + all_test_modules,
27 + allowlist_entries_missing_reason,
28 + is_test_module,
29 + is_tethered,
30 + load_allowlist,
31 + load_selections,
32 + main,
33 + normalize_test_path,
34 + run_changed_files_check,
35 + run_full_scan,
36 +)
37 +
38 +REPO_ROOT = Path(__file__).resolve().parents[2]
39 +
40 +
41 +def runs(command: str, test_file: str) -> bool:
42 + """True if `command` collects `test_file` (path relative to ``tests/``)."""
43 + return any(sel.runs(test_file) for sel in _parse_command(command))
44 +
45 +
46 +# --------------------------------------------------------------------------- #
47 +# pytest path arguments
48 +# --------------------------------------------------------------------------- #
49 +
50 +
51 +@pytest.mark.parametrize(
52 + ("command", "test_file", "expected"),
53 + [
54 + # Directory sweeps cover everything nested underneath.
55 + ("pytest -v -s kernels", "kernels/test_attention.py", True),
56 + ("pytest -v -s kernels/", "kernels/moe/test_moe.py", True),
57 + ("pytest -v -s kernels", "lora/test_lora.py", False),
58 + # An exact file argument covers that file and nothing beside it.
59 + ("pytest -v -s kernels/test_a.py", "kernels/test_a.py", True),
60 + ("pytest -v -s kernels/test_a.py", "kernels/test_b.py", False),
61 + # Commands run from the repo root and from tests/ alike.
62 + ("pytest -v -s tests/kernels/test_a.py", "kernels/test_a.py", True),
63 + ("pytest -v -s ./kernels/test_a.py", "kernels/test_a.py", True),
64 + (
65 + "pytest -v -s /vllm-workspace/tests/kernels/test_a.py",
66 + "kernels/test_a.py",
67 + True,
68 + ),
69 + # A ::nodeid selector still collects the file.
70 + ("pytest -v -s kernels/test_a.py::test_one", "kernels/test_a.py", True),
71 + ("pytest -v -s kernels/test_a.py::Cls::test_one", "kernels/test_a.py", True),
72 + # Several positional paths, including a bare root-level file.
73 + ("pytest -v -s test_envs.py test_outputs.py", "test_outputs.py", True),
74 + ("pytest -v -s test_envs.py test_outputs.py", "test_regression.py", False),
75 + # A bare word is a path only when it is really a directory under tests/.
76 + ("pytest -v -s samplers", "samplers/test_sampler.py", True),
77 + # Glob path args do not cross a directory separator, matching the shell.
78 + ("pytest -v -s kernels/test_*.py", "kernels/test_a.py", True),
79 + ("pytest -v -s kernels/test_*.py", "kernels/moe/test_a.py", False),
80 + ("pytest -v -s kernels/test_*.py", "kernels/helper.py", False),
81 + ],
82 +)
83 +def test_pytest_path_arguments(command, test_file, expected):
84 + assert runs(command, test_file) is expected
85 +
86 +
87 +# --------------------------------------------------------------------------- #
88 +# --ignore / --deselect
89 +# --------------------------------------------------------------------------- #
90 +
91 +
92 +@pytest.mark.parametrize(
93 + ("command", "test_file", "expected"),
94 + [
95 + # --ignore removes a file from an otherwise covering sweep, in both the
96 + # `=` and space-separated spellings.
97 + ("pytest -v -s kernels --ignore=kernels/test_a.py", "kernels/test_a.py", False),
98 + ("pytest -v -s kernels --ignore kernels/test_a.py", "kernels/test_a.py", False),
99 + # ...and only that file.
100 + ("pytest -v -s kernels --ignore=kernels/test_a.py", "kernels/test_b.py", True),
101 + # --ignore of a directory removes the whole subtree.
102 + ("pytest -v -s kernels --ignore=kernels/moe", "kernels/moe/test_moe.py", False),
103 + (
104 + "pytest -v -s lora --ignore-glob=lora/*_tp.py",
105 + "lora/test_llama_tp.py",
106 + False,
107 + ),
108 + # --ignore-glob is a whole-path fnmatch (pytest semantics): its `*`
109 + # crosses `/`, so a bare `*_tp.py` drops a nested match too. A positional
110 + # path glob would not - that is the distinction the parser has to keep.
111 + ("pytest -v -s kernels --ignore-glob=*_tp.py", "kernels/test_a_tp.py", False),
112 + (
113 + "pytest -v -s kernels --ignore-glob=*_tp.py",
114 + "kernels/nested/test_a_tp.py",
115 + False,
116 + ),
117 + ("pytest -v -s kernels --ignore-glob '*_tp.py'", "kernels/test_a.py", True),
118 + # A node-id deselect leaves the file collected; a bare-path one does not.
119 + (
120 + "pytest -v -s kernels --deselect kernels/test_a.py::test_one",
121 + "kernels/test_a.py",
122 + True,
123 + ),
124 + (
125 + "pytest -v -s kernels --deselect kernels/test_a.py",
126 + "kernels/test_a.py",
127 + False,
128 + ),
129 + ],
130 +)
131 +def test_ignore_and_deselect(command, test_file, expected):
132 + assert runs(command, test_file) is expected
133 +
134 +
135 +# --------------------------------------------------------------------------- #
136 +# Options, env prefixes and other token noise
137 +# --------------------------------------------------------------------------- #
138 +
139 +
140 +@pytest.mark.parametrize(
141 + ("command", "test_file", "expected"),
142 + [
143 + # A -k / -m filter narrows which tests run but does not un-run the file.
144 + ("pytest -v -s kernels -k 'not slow'", "kernels/test_a.py", True),
145 + ("pytest -v -s kernels -m core_model", "kernels/test_a.py", True),
146 + # A value-taking option's value is never mistaken for a path.
147 + ("pytest -m core_model", "core_model", False),
148 + ("pytest -k kernels", "kernels/test_a.py", False),
149 + ("pytest -n 4 kernels/test_a.py", "kernels/test_a.py", True),
150 + # Buildkite shard flags carry a $$VAR and must not swallow the path.
151 + (
152 + (
153 + "pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB "
154 + "--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT"
155 + ),
156 + "kernels/test_a.py",
157 + True,
158 + ),
159 + # A leading env-var assignment is not a path.
160 + (
161 + "VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s kernels",
162 + "kernels/test_a.py",
163 + True,
164 + ),
165 + # `python -m pytest` is the same invocation.
166 + ("python3 -m pytest -v -s kernels/test_a.py", "kernels/test_a.py", True),
167 + # Quoted paths, wrapper commands and redirects all survive tokenizing.
168 + ("pytest -v -s 'kernels/test_a.py'", "kernels/test_a.py", True),
169 + ("timeout 600 pytest kernels/test_a.py", "kernels/test_a.py", True),
170 + ("pytest kernels/test_a.py > out.log 2>&1", "kernels/test_a.py", True),
171 + # Each sub-command of a compound line is classified on its own.
172 + ("pytest kernels/test_a.py && pytest lora/test_b.py", "lora/test_b.py", True),
173 + ("pytest kernels/test_a.py ; pytest lora/test_b.py", "lora/test_b.py", True),
174 + # An unresolvable $VAR path yields no coverage rather than a guess.
175 + ("pytest -v -s $TEST_TARGET", "kernels/test_a.py", False),
176 + ],
177 +)
178 +def test_option_and_token_handling(command, test_file, expected):
179 + assert runs(command, test_file) is expected
180 +
181 +
182 +# --------------------------------------------------------------------------- #
183 +# find | xargs pytest pipelines
184 +# --------------------------------------------------------------------------- #
185 +
186 +
187 +@pytest.mark.parametrize(
188 + ("command", "test_file", "expected"),
189 + [
190 + ("find kernels -name 'test_*.py' | xargs pytest -v", "kernels/test_a.py", True),
191 + (
192 + "find kernels -name 'test_*.py' | xargs pytest -v",
193 + "kernels/moe/test_b.py",
194 + True,
195 + ),
196 + # A non-matching filename is not collected by the find expression.
197 + (
198 + "find kernels -name 'test_*.py' | xargs pytest -v",
199 + "kernels/helper.py",
200 + False,
201 + ),
202 + # -maxdepth 1 keeps direct children and drops nested ones.
203 + (
204 + "find kernels -maxdepth 1 -name 'test_*.py' | xargs pytest",
205 + "kernels/test_a.py",
206 + True,
207 + ),
208 + (
209 + "find kernels -maxdepth 1 -name 'test_*.py' | xargs pytest",
210 + "kernels/moe/test_b.py",
211 + False,
212 + ),
213 + # -not -name excludes its target and leaves siblings alone.
214 + (
215 + "find kernels -name 'test_*.py' -not -name 'test_skip.py' | xargs pytest",
216 + "kernels/test_skip.py",
217 + False,
218 + ),
219 + (
220 + "find kernels -name 'test_*.py' -not -name 'test_skip.py' | xargs pytest",
221 + "kernels/test_a.py",
222 + True,
223 + ),
224 + # The root confines the sweep.
225 + ("find kernels -name 'test_*.py' | xargs pytest", "lora/test_a.py", False),
226 + # -maxdepth is honored even when the find root is the tests/ root itself
227 + # ("." or "tests/"), not just a subdir.
228 + (
229 + "find . -maxdepth 1 -name 'test_*.py' | xargs pytest",
230 + "test_regression.py",
231 + True,
232 + ),
233 + (
234 + "find . -maxdepth 1 -name 'test_*.py' | xargs pytest",
235 + "v1/test_scheduler.py",
236 + False,
237 + ),
238 + (
239 + "find tests/ -maxdepth 1 -name 'test_*.py' | xargs pytest",
240 + "v1/test_scheduler.py",
241 + False,
242 + ),
243 + # -exec pytest wires the sweep just as `| xargs pytest` does.
244 + (
245 + "find compile -name 'test_*.py' -exec pytest -s -v {} \\;",
246 + "compile/test_a.py",
247 + True,
248 + ),
249 + # A pytest in a *separate* command on the line (not fed by the find) does
250 + # not arm the sweep - the find here is piped to `wc`.
251 + (
252 + "find kernels -name 'test_*.py' | wc -l ; pytest test_regression.py",
253 + "kernels/test_a.py",
254 + False,
255 + ),
256 + (
257 + "find kernels -name 'test_*.py' -delete && pytest test_regression.py",
258 + "kernels/test_a.py",
259 + False,
260 + ),
261 + # ...but only a *non-filtering* intermediate stage. `sort` reorders,
262 + # every match still reaches pytest.
263 + (
264 + "find kernels -name 'test_*.py' | sort | xargs pytest",
265 + "kernels/test_a.py",
266 + True,
267 + ),
268 + # `head` drops all but one match - the sweep is not fully tethered.
269 + (
270 + "find kernels -name 'test_*.py' | head -n 1 | xargs pytest",
271 + "kernels/test_a.py",
272 + False,
273 + ),
274 + (
275 + "find kernels -name 'test_*.py' | grep attention | xargs pytest",
276 + "kernels/test_a.py",
277 + False,
278 + ),
279 + # `xargs` runs `echo` here, not pytest - the trailing `pytest` is echo's
280 + # argument.
281 + (
282 + "find kernels -name 'test_*.py' | xargs echo pytest",
283 + "kernels/test_a.py",
284 + False,
285 + ),
286 + # An `-exec` with no `\;` / `+` terminator is a `find` syntax error - it
287 + # never runs pytest.
288 + (
289 + "find kernels -name 'test_*.py' -exec pytest {}",
290 + "kernels/test_a.py",
291 + False,
292 + ),
293 + # An `-exec pytest` with no `{}` runs only the named file for every
294 + # match - the matched files never reach pytest.
295 + (
296 + "find kernels -name 'test_*.py' -exec pytest test_regression.py \\;",
297 + "kernels/test_a.py",
298 + False,
299 + ),
300 + # `{}` as an option *value* (`--ignore {}`) doesn't reach pytest as a
301 + # test path either - only `test_regression.py` runs.
302 + (
303 + (
304 + "find kernels -name 'test_*.py' -exec pytest test_regression.py "
305 + "--ignore {} \\;"
306 + ),
307 + "kernels/test_a.py",
308 + False,
309 + ),
310 + # xargs value-flags with a space don't swallow the command.
311 + (
312 + "find kernels -name 'test_*.py' | xargs -0 -I {} pytest {}",
313 + "kernels/test_a.py",
314 + True,
315 + ),
316 + # `xargs env FOO=1 pytest` - xargs runs `env`, which runs pytest.
317 + (
318 + "find kernels -name 'test_*.py' | xargs env FOO=1 pytest",
319 + "kernels/test_a.py",
320 + True,
321 + ),
322 + ],
323 +)
324 +def test_find_pipelines(command, test_file, expected):
325 + assert runs(command, test_file) is expected
326 +
327 +
328 +def test_find_pipeline_parses_as_find_selection():
329 + (selection,) = [
330 + s
331 + for s in _parse_command("find kernels -name 'test_*.py' | xargs pytest -v")
332 + if isinstance(s, FindSelection)
333 + ]
334 + assert selection.root == "kernels"
335 + assert selection.name_globs == ["test_*.py"]
336 +
337 +
338 +# --------------------------------------------------------------------------- #
339 +# Direct file runners
340 +# --------------------------------------------------------------------------- #
341 +
342 +
343 +@pytest.mark.parametrize(
344 + ("command", "test_file", "expected"),
345 + [
346 + (
347 + "python3 standalone_tests/lazy_imports.py",
348 + "standalone_tests/lazy_imports.py",
349 + True,
350 + ),
351 + (
352 + "python standalone_tests/lazy_imports.py",
353 + "standalone_tests/lazy_imports.py",
354 + True,
355 + ),
356 + # `timeout -k <grace> <dur>` wrapping a runner: the grace value must not
357 + # be consumed as the duration (which would hide `python`).
358 + (
359 + "timeout -k 5s 30s python standalone_tests/lazy_imports.py",
360 + "standalone_tests/lazy_imports.py",
361 + True,
362 + ),
363 + (
364 + "torchrun --nproc-per-node 2 distributed/test_a.py",
365 + "distributed/test_a.py",
366 + True,
367 + ),
368 + # The option's value is a count, not a path.
369 + ("torchrun --nproc-per-node 2 distributed/test_a.py", "2", False),
370 + (
371 + "VLLM_X=1 python3 standalone_tests/lazy_imports.py | grep -q ok",
372 + "standalone_tests/lazy_imports.py",
373 + True,
374 + ),
375 + # Only the executed script counts; the tokens after it are that script's
376 + # own argv. `--suffix v1` must not tether the whole tests/v1 subtree.
377 + (
378 + (
379 + "python3 features/tensorize_vllm_model.py --model facebook/opt-125m "
380 + "serialize --serialized-directory /tmp/ --suffix v1"
381 + ),
382 + "v1/test_scheduler.py",
383 + False,
384 + ),
385 + (
386 + (
387 + "python3 features/tensorize_vllm_model.py --model facebook/opt-125m "
388 + "serialize --serialized-directory /tmp/ --suffix v1"
389 + ),
390 + "features/tensorize_vllm_model.py",
391 + True,
392 + ),
393 + # A script named by an absolute `$VAR` path still resolves.
394 + (
395 + 'python3 -m pytest -s -x "${GIT_ROOT}/tests/v1/e2e/test_x.py"',
396 + "v1/e2e/test_x.py",
397 + True,
398 + ),
399 + # `python -m <module>` runs the module, not a script - only `-m pytest`
400 + # collects tests. `compileall kernels` byte-compiles, it does not run.
401 + ("python -m compileall kernels", "kernels/test_a.py", False),
402 + ("python -m mypy kernels/test_a.py", "kernels/test_a.py", False),
403 + # `coverage run -m pytest` still resolves through to pytest.
404 + ("coverage run -m pytest kernels/test_a.py", "kernels/test_a.py", True),
405 + # `python -c '<code>' file.py` runs the code string; `file.py` is argv.
406 + ("python -c 'pass' kernels/test_a.py", "kernels/test_a.py", False),
407 + ("python -c 'import sys' kernels/test_a.py", "kernels/test_a.py", False),
408 + # a `-m pytest` *after* the `-c` code operand is that program's argv too.
409 + ("python -c 'pass' -m pytest kernels/test_a.py", "kernels/test_a.py", False),
410 + # ...but `-c` as the *script's* own arg still tethers the script.
411 + ("python kernels/test_a.py -c config.py", "kernels/test_a.py", True),
412 + # ...and pytest's own `-c <cfg>` (after `-m pytest`) is fine.
413 + ("python -m pytest kernels/test_a.py -c setup.cfg", "kernels/test_a.py", True),
414 + ],
415 +)
416 +def test_direct_runners(command, test_file, expected):
417 + assert runs(command, test_file) is expected
418 +
419 +
420 +# --------------------------------------------------------------------------- #
421 +# Commands that must contribute no coverage
422 +# --------------------------------------------------------------------------- #
423 +
424 +
425 +@pytest.mark.parametrize(
426 + "command",
427 + [
428 + "export VLLM_WORKER_MULTIPROC_METHOD=spawn",
429 + "pip install pytest-timeout pytest-forked",
430 + "uv pip install -r requirements/test/cuda.in",
431 + "bash tests/does_not_exist.sh",
432 + "echo 'pytest kernels/test_a.py'",
433 + # `pytest` as an argument to another command - `echo` runs, not pytest.
434 + "echo pytest kernels/test_a.py",
435 + 'pytest -v -s "unbalanced',
436 + # A `.sh` that is only an *argument* (Buildkite printing / copying a
437 + # filename) is not a script invocation.
438 + "echo tests/foo.sh",
439 + "cp scripts/run_tests.sh /tmp/",
440 + # A `find` whose output never reaches pytest is not a test selection.
441 + "find kernels -name 'test_*.py'",
442 + "find kernels -name 'test_*.py' | wc -l",
443 + "find kernels -name 'test_*.py' -delete",
444 + "find kernels -name 'test_*.py' | xargs echo pytest",
445 + "find kernels -name 'test_*.py' -exec pytest {}",
446 + ],
447 +)
448 +def test_non_test_commands_contribute_nothing(command):
449 + assert _parse_command(command) == [] or not any(
450 + sel.runs("kernels/test_a.py") for sel in _parse_command(command)
451 + )
452 +
453 +
454 +# --------------------------------------------------------------------------- #
455 +# Job-level yaml parsing
456 +# --------------------------------------------------------------------------- #
457 +
458 +
459 +def _write_area(tmp_path, monkeypatch, body: str):
460 + mod = checker
461 + area_dir = tmp_path / "test_areas"
462 + area_dir.mkdir(exist_ok=True)
463 + (area_dir / "synthetic.yaml").write_text(textwrap.dedent(body))
464 + monkeypatch.setattr(mod, "TEST_AREAS_DIR", area_dir)
465 + # Keep the real test-amd.yaml out of these isolated parsing tests.
466 + monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "no-such-test-amd.yaml")
467 + # The yaml-error path reports a repo-relative name, so REPO_ROOT has to
468 + # contain the synthetic area too.
469 + monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
470 + return load_selections()
471 +
472 +
473 +def test_plain_step_commands_are_collected(tmp_path, monkeypatch):
474 + selections = _write_area(
475 + tmp_path,
476 + monkeypatch,
477 + """
478 + steps:
479 + - label: ":nvidia: (H200) Kernels"
480 + commands:
481 + - pytest -v -s kernels
482 + """,
483 + )
484 + assert is_tethered("kernels/test_a.py", selections)
485 +
486 +
487 +def test_singular_command_key_is_collected(tmp_path, monkeypatch):
488 + selections = _write_area(
489 + tmp_path,
490 + monkeypatch,
491 + """
492 + steps:
493 + - label: "Singular"
494 + command: pytest -v -s kernels/test_a.py
495 + """,
496 + )
497 + assert is_tethered("kernels/test_a.py", selections)
498 +
499 +
500 +def test_test_amd_yaml_coverage_counts_as_tethered(tmp_path, monkeypatch):
501 + """A test wired only into the legacy test-amd.yaml is still run by CI."""
502 + mod = checker
503 + (tmp_path / "test_areas").mkdir()
504 + amd_yaml = tmp_path / "test-amd.yaml"
505 + amd_yaml.write_text(
506 + textwrap.dedent(
507 + """
508 + steps:
509 + - label: ":amd: (MI300) Quantization"
510 + commands:
511 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/
512 + - pytest -v -s rocm/test_moe_weight_replay.py
513 + """
514 + )
515 + )
516 + monkeypatch.setattr(mod, "TEST_AREAS_DIR", tmp_path / "test_areas")
517 + monkeypatch.setattr(mod, "TEST_AMD_YAML", amd_yaml)
518 + monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
519 + selections = load_selections()
520 + assert is_tethered("rocm/test_moe_weight_replay.py", selections)
521 + assert is_tethered("quantization/test_foo.py", selections)
522 +
523 +
524 +def test_missing_test_amd_yaml_is_not_fatal(tmp_path, monkeypatch):
525 + """When the migration finally deletes test-amd.yaml, the checker still runs."""
526 + mod = checker
527 + (tmp_path / "test_areas").mkdir()
528 + monkeypatch.setattr(mod, "TEST_AREAS_DIR", tmp_path / "test_areas")
529 + monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "gone.yaml")
530 + monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
531 + assert load_selections() == []
532 +
533 +
534 +def test_job_dirs_come_from_ci_config(tmp_path, monkeypatch):
535 + """The pipeline directories are whatever `.buildkite/ci_config*.yaml` lists
536 + under `job_dirs:` - so a test wired only into e.g. `hardware_tests/` counts,
537 + and an unreadable config falls back to `test_areas/` alone."""
538 + mod = checker
539 + bk = tmp_path / ".buildkite"
540 + (bk / "test_areas").mkdir(parents=True)
541 + (bk / "hardware_tests").mkdir()
542 + (bk / "ci_config.yaml").write_text("job_dirs:\n - .buildkite/hardware_tests\n")
543 + (bk / "ci_config_broken.yaml").write_text("job_dirs: [oops\n") # ignored
544 + (bk / "hardware_tests" / "cpu.yaml").write_text(
545 + textwrap.dedent(
546 + """
547 + steps:
548 + - label: "CPU"
549 + commands:
550 + - pytest -v -s v1/e2e/test_cpu_thing.py
551 + """
552 + )
553 + )
554 + monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
555 + monkeypatch.setattr(mod, "TEST_AREAS_DIR", bk / "test_areas")
556 + monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "no-amd.yaml")
557 +
558 + assert (bk / "hardware_tests") in checker._pipeline_job_dirs()
559 + assert is_tethered("v1/e2e/test_cpu_thing.py", load_selections())
560 +
561 +
562 +def test_step_level_ignore_is_honored(tmp_path, monkeypatch):
563 + selections = _write_area(
564 + tmp_path,
565 + monkeypatch,
566 + """
567 + steps:
568 + - label: "Kernels"
569 + commands:
570 + - pytest -v -s kernels --ignore=kernels/test_a.py
571 + """,
572 + )
573 + assert not is_tethered("kernels/test_a.py", selections)
574 + assert is_tethered("kernels/test_b.py", selections)
575 +
576 +
577 +def test_amd_mirror_only_command_is_collected(tmp_path, monkeypatch):
578 + """A mirror that overrides `commands` runs tests the base step never does."""
579 + selections = _write_area(
580 + tmp_path,
581 + monkeypatch,
582 + """
583 + steps:
584 + - label: ":nvidia: (H200) Base"
585 + commands:
586 + - pytest -v -s kernels/test_base.py
587 + mirror:
588 + amd:
589 + label: ":amd: (MI300) Base"
590 + commands:
591 + - pytest -v -s kernels/test_amd_only.py
592 + """,
593 + )
594 + assert is_tethered("kernels/test_base.py", selections)
595 + assert is_tethered("kernels/test_amd_only.py", selections)
596 +
597 +
598 +def test_shell_script_continuation_vars_and_nesting(tmp_path, monkeypatch):
599 + """The v1 P/D integration scripts name their test through a `\\`-continued
600 + line, a literal shell var, and a nested `bash "$SCRIPT"`. Each indirection
601 + has to be followed or the check falsely reports the test untethered."""
602 + scripts = tmp_path / "tests" / "v1" / "kv_connector" / "nixl_integration"
603 + scripts.mkdir(parents=True)
604 + (scripts / "inner.sh").write_text(
605 + "python3 -m pytest -s -v \\\n"
606 + ' "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_inner.py"\n'
607 + )
608 + (scripts / "sweep.sh").write_text(
609 + 'SCRIPT="v1/kv_connector/nixl_integration/inner.sh"\n'
610 + 'IMPORT_CANARY="v1/kv_connector/nixl_integration/test_canary.py"\n'
611 + 'LITERAL="v1/kv_connector/nixl_integration/test_not_run.py"\n'
612 + 'python3 -m pytest -s -x "${IMPORT_CANARY}"\n'
613 + # single-quoted: the shell passes a literal $LITERAL, collecting nothing
614 + "python3 -m pytest -s -x '${LITERAL}'\n"
615 + 'if ! env FOO=1 bash "${SCRIPT}"; then exit 1; fi\n'
616 + )
617 + monkeypatch.setattr(checker, "REPO_ROOT", tmp_path)
618 + monkeypatch.setattr(checker, "TESTS_DIR", tmp_path / "tests")
619 +
620 + selections = _parse_command("bash v1/kv_connector/nixl_integration/sweep.sh")
621 +
622 + def tethered(path):
623 + return any(s.runs(path) for s in selections)
624 +
625 + # `\`-continuation + $VAR-prefixed absolute path, resolved through a nested
626 + # `env FOO=1 bash "$SCRIPT"`:
627 + assert tethered("v1/kv_connector/nixl_integration/test_inner.py")
628 + # literal `VAR=path` + double-quoted `"$VAR"` (the import-canary pattern):
629 + assert tethered("v1/kv_connector/nixl_integration/test_canary.py")
630 + # single-quoted `'$VAR'` is NOT expanded by the shell, so not tethered:
631 + assert not tethered("v1/kv_connector/nixl_integration/test_not_run.py")
632 +
633 +
634 +def test_bash_dash_c_does_not_follow_a_script_operand(tmp_path, monkeypatch):
635 + """`bash -c '<code>' foo.sh` runs the `-c` string; `foo.sh` is `$0` and is
636 + never executed, so its contents must not count as coverage."""
637 + (tmp_path / "tests").mkdir()
638 + (tmp_path / "tests" / "decoy.sh").write_text("pytest -v -s lora/test_decoy.py\n")
639 + monkeypatch.setattr(checker, "REPO_ROOT", tmp_path)
640 + monkeypatch.setattr(checker, "TESTS_DIR", tmp_path / "tests")
641 +
642 + with_c = _parse_command("bash -c 'echo hi' tests/decoy.sh")
643 + assert not any(s.runs("lora/test_decoy.py") for s in with_c)
644 + # ...but a plain `bash tests/decoy.sh` still follows it.
645 + plain = _parse_command("bash tests/decoy.sh")
646 + assert any(s.runs("lora/test_decoy.py") for s in plain)
647 + # ...and a `-c` that is the *script's* own flag (not bash's) is not `bash -c`.
648 + script_flag = _parse_command("bash tests/decoy.sh -c models.txt")
649 + assert any(s.runs("lora/test_decoy.py") for s in script_flag)
650 + # A `bash -c` trailing arg that looks like a command string is `$0`, not run.
651 + cmd_string_arg = _parse_command("bash -c 'echo ok' 'pytest lora/test_x.py'")
652 + assert not any(s.runs("lora/test_x.py") for s in cmd_string_arg)
653 + # ...but the `-c` code string *itself* is parsed - it is what runs.
654 + code_runs = _parse_command("bash -c 'pytest lora/test_x.py'")
655 + assert any(s.runs("lora/test_x.py") for s in code_runs)
656 + # a combined short-option group resolves the `-c` operand with `c` in any
657 + # position (`-ec`, `-ce`), and its trailing args are still `$0`, not run.
658 + for grp in ("-ec", "-ce", "-euxc"):
659 + assert any(
660 + s.runs("lora/test_x.py")
661 + for s in _parse_command(f"bash {grp} 'pytest lora/test_x.py'")
662 + )
663 + assert not any(
664 + s.runs("lora/test_x.py")
665 + for s in _parse_command("bash -ce 'echo ok' 'pytest lora/test_x.py'")
666 + )
667 +
668 +
669 +def test_unparsable_yaml_is_fatal(tmp_path, monkeypatch):
670 + """Silently skipping a bad yaml would drop its coverage and report false
671 + untethered files, so it must fail loudly instead."""
672 + with pytest.raises(SystemExit):
673 + _write_area(tmp_path, monkeypatch, "steps: [oops\n")
674 +
675 +
676 +def test_non_mapping_yaml_is_fatal(tmp_path, monkeypatch):
677 + """A yaml that parses but isn't a pipeline mapping (a bare list or scalar)
678 + must raise the actionable error, not crash on ``.get``."""
679 + with pytest.raises(SystemExit):
680 + _write_area(tmp_path, monkeypatch, "- just\n- a\n- list\n")
681 +
682 +
683 +# --------------------------------------------------------------------------- #
684 +# is_test_module / path normalization
685 +# --------------------------------------------------------------------------- #
686 +
687 +
688 +@pytest.mark.parametrize(
689 + ("path", "expected"),
690 + [
691 + ("tests/kernels/test_a.py", True),
692 + ("tests/kernels/a_test.py", True),
693 + ("tests/kernels/conftest.py", False),
694 + ("tests/kernels/__init__.py", False),
695 + ("tests/kernels/utils.py", False),
696 + ("tests/kernels/test_a.txt", False),
697 + ("vllm/kernels/test_a.py", False),
698 + ],
699 +)
700 +def test_is_test_module(path, expected):
701 + assert is_test_module(path) is expected
702 +
703 +
704 +@pytest.mark.parametrize(
705 + ("raw", "expected"),
706 + [
707 + ("tests/kernels/test_a.py", "kernels/test_a.py"),
708 + ("./kernels/test_a.py", "kernels/test_a.py"),
709 + ("/vllm-workspace/tests/kernels/test_a.py", "kernels/test_a.py"),
710 + ("'kernels/test_a.py'", "kernels/test_a.py"),
711 + ("kernels/test_a.py::test_one", "kernels/test_a.py"),
712 + ("kernels/", "kernels"),
713 + ("${GIT_ROOT}/tests/kernels/test_a.py", "kernels/test_a.py"),
714 + ("$GIT_ROOT/tests/kernels/test_a.py", "kernels/test_a.py"),
715 + ],
716 +)
717 +def test_normalize_test_path(raw, expected):
718 + assert normalize_test_path(raw) == expected
719 +
720 +
721 +# --------------------------------------------------------------------------- #
722 +# Whole-tree invariants
723 +# --------------------------------------------------------------------------- #
724 +
725 +
726 +def test_real_tree_parses_into_selections():
727 + assert len(load_selections()) > 200
728 + assert len(all_test_modules()) > 1000
729 +
730 +
731 +def test_no_selection_matches_everything():
732 + """A path arg that normalizes to '' or '.' would tether every test in the
733 + repo and silently disarm the gate - the worst possible misparse."""
734 + for selection in load_selections():
735 + if isinstance(selection, PytestSelection):
736 + offenders = [
737 + p
738 + for p in selection.included_paths
739 + if normalize_test_path(p) in ("", ".")
740 + ]
741 + assert not offenders, f"match-everything pytest arg: {offenders}"
742 + else:
743 + assert normalize_test_path(selection.root) not in ("", ".")
744 +
745 +
746 +def test_tethering_is_deterministic():
747 + selections = load_selections()
748 + modules = all_test_modules()[:200]
749 + first = [is_tethered(m, selections) for m in modules]
750 + assert first == [is_tethered(m, load_selections()) for m in modules]
751 +
752 +
753 +def test_allowlist_entries_are_well_formed():
754 + allowlist = load_allowlist()
755 + assert allowlist, "allowlist should not be empty while gaps remain"
756 + assert all(is_test_module(path) for path in allowlist)
757 + assert all(not path.startswith("/") for path in allowlist)
758 +
759 +
760 +def test_real_allowlist_entries_all_have_a_reason():
761 + assert allowlist_entries_missing_reason() == []
762 +
763 +
764 +def test_bare_allowlist_entry_is_flagged(tmp_path, monkeypatch):
765 + allowlist = tmp_path / "allowlist.txt"
766 + allowlist.write_text(
767 + "tests/a/test_ok.py # torchrun multi-GPU, no job\n"
768 + "tests/b/test_bare.py\n"
769 + " # a standalone comment line is fine\n"
770 + )
771 + monkeypatch.setattr(checker, "ALLOWLIST_PATH", allowlist)
772 + assert allowlist_entries_missing_reason() == ["tests/b/test_bare.py"]
773 +
774 +
775 +def test_real_tree_has_no_unallowlisted_gaps():
776 + """Every test module is either collected by a job or allowlisted.
777 +
778 + Deliberately non-strict: a stale allowlist entry is advisory by design, so
779 + asserting `strict=True` here would make unrelated allowlist cleanup block
780 + CI - the exact behaviour the two-mode design avoids.
781 + """
782 + assert run_full_scan(load_selections(), load_allowlist(), strict=False) == 0
783 +
784 +
785 +# --------------------------------------------------------------------------- #
786 +# CLI behaviour
787 +# --------------------------------------------------------------------------- #
788 +
789 +
790 +@pytest.fixture
791 +def fake_repo(tmp_path, monkeypatch):
792 + """Point the module at a throwaway tree so the changed-files path can be
793 + exercised without writing test files into the real repo."""
794 + mod = checker
795 + (tmp_path / "tests" / "kernels").mkdir(parents=True)
796 + (tmp_path / "tests" / "kernels" / "test_new.py").write_text("def test_x(): pass\n")
797 + monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
798 + monkeypatch.setattr(mod, "TESTS_DIR", tmp_path / "tests")
799 + monkeypatch.setattr(mod, "ALLOWLIST_PATH", tmp_path / "allowlist.txt")
800 + # The throwaway tree is not a git checkout, so the tracked-file inventory
801 + # has to be stubbed rather than shelled out.
802 + monkeypatch.setattr(mod, "all_test_modules", lambda: ["tests/kernels/test_new.py"])
803 + return tmp_path
804 +
805 +
806 +def test_untethered_changed_file_fails(fake_repo, capsys):
807 + rc = run_changed_files_check(["tests/kernels/test_new.py"], [], set())
808 + assert rc == 1
809 + assert "is not run by any Buildkite job" in capsys.readouterr().out
810 +
811 +
812 +def test_tethered_changed_file_passes(fake_repo):
813 + selections = _parse_command("pytest -v -s kernels")
814 + assert (
815 + run_changed_files_check(["tests/kernels/test_new.py"], selections, set()) == 0
816 + )
817 +
818 +
819 +def test_allowlisted_changed_file_passes(fake_repo):
820 + allowlist = {"tests/kernels/test_new.py"}
821 + assert run_changed_files_check(["tests/kernels/test_new.py"], [], allowlist) == 0
822 +
823 +
824 +def test_non_test_and_missing_paths_are_ignored(fake_repo):
825 + paths = ["tests/kernels/conftest.py", "vllm/config.py", "tests/kernels/gone.py"]
826 + assert run_changed_files_check(paths, [], set()) == 0
827 +
828 +
829 +def test_stale_allowlist_entries_are_advisory_but_reported(fake_repo, capsys):
830 + """Dangling entries must not block an unrelated PR, but they have to be
831 + printed - the hook sets `verbose: true` so a passing run still shows them."""
832 + allowlist = {"tests/kernels/gone.py"}
833 + rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=False)
834 + out = capsys.readouterr().out
835 + assert rc == 0
836 + assert "no longer exists" in out
837 +
838 +
839 +def test_stale_allowlist_entries_are_fatal_under_all(fake_repo, capsys):
840 + allowlist = {"tests/kernels/gone.py"}
841 + rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=True)
842 + assert rc == 1
843 + assert "no longer exists" in capsys.readouterr().out
844 +
845 +
846 +def test_now_tethered_allowlist_entry_is_reported(fake_repo, capsys):
847 + allowlist = {"tests/kernels/test_new.py"}
848 + rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=True)
849 + assert rc == 1
850 + assert "is now tethered" in capsys.readouterr().out
851 +
852 +
853 +def test_main_passes_a_tethered_file(monkeypatch):
854 + """The CLI's normal pre-commit path: a tethered file exits 0.
855 +
856 + `--all` is not asserted here on purpose - it implies `strict=True`, so a
857 + stale allowlist entry would turn unrelated cleanup into a CI failure.
858 + """
859 + monkeypatch.setattr(
860 + "sys.argv",
861 + ["check_test_tethering.py", "tests/tools/test_check_test_tethering.py"],
862 + )
863 + assert main() == 0
864 +
865 +
866 +def test_main_flags_an_untethered_file(monkeypatch):
867 + # Must be named like a test module, or the check correctly skips it.
868 + orphan = REPO_ROOT / "tests" / "test_tethering_probe_orphan.py"
869 + orphan.write_text("def test_x(): pass\n")
870 + monkeypatch.setattr("sys.argv", ["check_test_tethering.py", str(orphan)])
871 + try:
872 + assert main() == 1
873 + finally:
874 + orphan.unlink()
875 +
876 +
877 +def test_to_repo_relative_accepts_absolute_paths():
878 + absolute = str(REPO_ROOT / "tests" / "kernels" / "test_a.py")
879 + assert _to_repo_relative(absolute) == "tests/kernels/test_a.py"
tools/__init__.py
tools/pre_commit/__init__.py
tools/pre_commit/check_test_tethering.py new
+925
@@ -0,0 +1,925 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Check that test files are wired into ("tethered to") the Buildkite CI.
4 +
5 +A test file is *tethered* when at least one Buildkite job actually collects it -
6 +some job's ``commands`` run ``pytest`` (or ``python`` / ``torchrun``, or a
7 +``find ... | xargs pytest`` pipeline) over a path that includes the file. A test
8 +that is collected by no job silently never runs: it passes review, merges, and
9 +rots.
10 +
11 +How it works
12 +------------
13 +1. Parse every job in every Buildkite pipeline yaml (and each ``mirror.amd``
14 + sub-step), and turn each shell command into a :class:`Selection` describing
15 + which test files it runs. The pipeline directories are read from
16 + ``.buildkite/ci_config*.yaml`` (``job_dirs:``) - currently ``test_areas/``,
17 + ``hardware_tests/`` and ``intel_jobs/`` across the CUDA, ROCm and Intel
18 + pipelines - plus the legacy ``.buildkite/test-amd.yaml``.
19 +2. Build the inventory of test files under ``tests/``.
20 +3. A file is a problem if no selection runs it and it is not in the allowlist
21 + (``tools/pre_commit/test_tethering_allowlist.txt`` - the set of pre-existing
22 + gaps, which is only ever meant to shrink). Every allowlist entry must carry a
23 + trailing ``# <reason>``; a bare path line is itself an error.
24 +
25 +Modes
26 +-----
27 +* **changed-files** (default): pre-commit passes the changed test files as
28 + arguments; each one must be tethered or allowlisted.
29 +* **full scan**: ``--all``, or automatically whenever the change set touches a
30 + pipeline yaml or the allowlist itself - because editing a job command can
31 + orphan a test that is not otherwise in the diff. The full scan also reports
32 + allowlist entries that have since become tethered, or whose file is gone, so
33 + the list stays honest.
34 +
35 +Coverage from *any* of those pipelines counts - the question is "does some
36 +Buildkite job run this test", not "which pipeline". ``test-amd.yaml`` is being
37 +folded into ``test_areas/`` a group at a time; when it is gone the checker stops
38 +parsing it automatically.
39 +
40 +Usage::
41 +
42 + python tools/pre_commit/check_test_tethering.py [FILES...]
43 + python tools/pre_commit/check_test_tethering.py --all
44 +"""
45 +
46 +import argparse
47 +import fnmatch
48 +import shlex
49 +import subprocess
50 +import sys
51 +from dataclasses import dataclass, field
52 +from pathlib import Path
53 +
54 +import regex as re
55 +import yaml
56 +
57 +REPO_ROOT = Path(__file__).resolve().parents[2]
58 +BUILDKITE_DIR = REPO_ROOT / ".buildkite"
59 +# Fallback pipeline dir, always parsed even if ``ci_config*.yaml`` can't be read.
60 +TEST_AREAS_DIR = BUILDKITE_DIR / "test_areas"
61 +# The legacy hand-maintained AMD pipeline. It is being migrated into
62 +# ``test_areas/`` piecemeal; until that finishes, a test wired only into this
63 +# file is still genuinely run by CI, so it counts as tethered. When the file is
64 +# removed, ``_pipeline_yaml_paths()`` stops appending it automatically.
65 +TEST_AMD_YAML = BUILDKITE_DIR / "test-amd.yaml"
66 +TESTS_DIR = REPO_ROOT / "tests"
67 +ALLOWLIST_PATH = Path(__file__).resolve().parent / "test_tethering_allowlist.txt"
68 +
69 +# `pytest` options that take a value in the *next* token, so that token must not
70 +# be mistaken for a test path (e.g. `-k expr`, `--shard-id 3`). Options that only
71 +# ever take their value with `=` (`--cov=...`) don't belong here - a bare
72 +# `--cov` takes no value and would wrongly swallow the following test path.
73 +PYTEST_OPTIONS_WITH_VALUE = {
74 + "-k", "-m", "-p", "-n", "-c", "-o", "-W", "-r",
75 + "--shard-id", "--num-shards", "--dist", "--durations", "--maxfail",
76 + "--timeout", "--rootdir", "--junitxml", "--tb",
77 +} # fmt: skip
78 +
79 +# `pytest` options whose value is a path that is *removed* from collection at
80 +# file granularity. ``--ignore`` takes a path prefix; ``--ignore-glob`` takes a
81 +# glob that pytest matches against the whole path with ``fnmatch`` (so its ``*``
82 +# *does* cross a ``/``), which is why the two are modelled separately.
83 +PYTEST_IGNORE_OPTIONS = {"--ignore", "--ignore-glob"}
84 +
85 +# `--deselect` takes a value too, but it removes individual node IDs - a file
86 +# with one parametrization deselected is still collected and still "run".
87 +PYTEST_DESELECT_OPTION = "--deselect"
88 +
89 +PYTEST_COMMANDS = {"pytest", "py.test"}
90 +
91 +# A leading ``$VAR/`` or ``${VAR}/`` path prefix - CI shell scripts routinely
92 +# name a test by an absolute path rooted at a shell var
93 +# (``"${GIT_ROOT}/tests/v1/..."``). The prefix is dropped before the path is
94 +# interpreted; an unresolved ``$`` anywhere else still makes the token unusable.
95 +_LEADING_VAR_PREFIX = re.compile(r"^\$\{?\w+\}?/")
96 +
97 +# Commands that execute a Python file directly. Only the *first* path argument is
98 +# the script that runs; everything after it is that script's own argv, not more
99 +# test files (``python tensorize.py --suffix v1`` runs one file, not ``tests/v1``).
100 +FILE_RUNNER_COMMANDS = {"python", "python3", "torchrun", "coverage"}
101 +
102 +
103 +# --------------------------------------------------------------------------- #
104 +# Shell-token helpers
105 +# --------------------------------------------------------------------------- #
106 +
107 +
108 +def _is_env_assignment(token: str) -> bool:
109 + """True for a leading ``FOO=bar`` shell env-var assignment - the kind some
110 + commands put before ``pytest`` (``VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest``).
111 + """
112 + if "=" not in token:
113 + return False
114 + name = token.split("=", 1)[0]
115 + return bool(name) and name.replace("_", "").isupper()
116 +
117 +
118 +def normalize_test_path(raw: str) -> str:
119 + """Reduce a path as it appears in a command to one comparable form:
120 + relative to ``tests/``, no surrounding quotes, no ``::nodeid`` selector, no
121 + trailing slash.
122 +
123 + Commands run from either the repo root or from ``tests/``, so both a
124 + ``tests/`` and a ``/vllm-workspace/tests/`` prefix are stripped, as is a
125 + leading ``$VAR/`` (``"${GIT_ROOT}/tests/..."``).
126 + """
127 + path = raw.strip().strip("'\"")
128 + path = path.split("::", 1)[0]
129 + path = _LEADING_VAR_PREFIX.sub("", path)
130 + for prefix in ("./", "tests/", "/vllm-workspace/tests/"):
131 + if path.startswith(prefix):
132 + path = path[len(prefix) :]
133 + return path.rstrip("/")
134 +
135 +
136 +def _token_is_test_path(token: str) -> bool:
137 + """True if a shell token is a path argument rather than a flag, an env-var
138 + assignment, a Buildkite ``$VAR``, or a ``-m`` / ``-k`` expression value.
139 +
140 + A token with a ``/`` or a ``.py`` ending is always a path. A bare word
141 + (``samplers``) only counts if it is really a directory under ``tests/``. A
142 + leading ``$VAR/`` is dropped first, but a ``$`` anywhere else means the token
143 + can't be resolved to a known path.
144 + """
145 + if not token or token.startswith("-"):
146 + return False
147 + if _is_env_assignment(token):
148 + return False
149 + token = _LEADING_VAR_PREFIX.sub("", token)
150 + if "$" in token:
151 + return False
152 + if "/" in token or token.endswith(".py"):
153 + return True
154 + return (TESTS_DIR / normalize_test_path(token)).is_dir()
155 +
156 +
157 +def _path_is_within(parent: str, path: str) -> bool:
158 + """True if ``path`` is ``parent`` itself or nested underneath it. A ``''`` /
159 + ``'.'`` parent (a command run with no path arg) contains everything."""
160 + if parent in ("", "."):
161 + return True
162 + return path == parent or path.startswith(parent + "/")
163 +
164 +
165 +def _glob_match_path(pattern: str, path: str) -> bool:
166 + """Like :func:`fnmatch.fnmatch`, but ``*`` and ``?`` do not match across a
167 + ``/`` - the way the shell and ``pytest`` treat an unexpanded glob path
168 + argument (``kernels/test_foo_*.py`` selects one directory, not the tree).
169 + A ``**`` component keeps the cross-separator meaning."""
170 + if not any(ch in pattern for ch in "*?["):
171 + return pattern == path
172 + if "**" in pattern:
173 + return fnmatch.fnmatch(path, pattern)
174 + pattern_parts = pattern.split("/")
175 + path_parts = path.split("/")
176 + return len(pattern_parts) == len(path_parts) and all(
177 + fnmatch.fnmatch(part, glob) for part, glob in zip(path_parts, pattern_parts)
178 + )
179 +
180 +
181 +# --------------------------------------------------------------------------- #
182 +# Selections - "which test files does this command run?"
183 +# --------------------------------------------------------------------------- #
184 +
185 +
186 +@dataclass
187 +class PytestSelection:
188 + """A ``pytest``-style command: run everything under ``included_paths``
189 + except what an ``--ignore`` (path prefix) or ``--ignore-glob`` (whole-path
190 + ``fnmatch``) removes."""
191 +
192 + included_paths: list[str] = field(default_factory=list)
193 + ignored_paths: list[str] = field(default_factory=list)
194 + ignored_globs: list[str] = field(default_factory=list)
195 +
196 + def runs(self, test_file: str) -> bool:
197 + included = any(self._arg_covers(arg, test_file) for arg in self.included_paths)
198 + if not included:
199 + return False
200 + if any(self._arg_covers(arg, test_file) for arg in self.ignored_paths):
201 + return False
202 + # `--ignore-glob` is a plain `fnmatch` over the whole path - `*_tp.py`
203 + # drops `kernels/nested/test_a_tp.py`, unlike a positional glob arg.
204 + return not any(
205 + fnmatch.fnmatch(test_file, normalize_test_path(glob))
206 + for glob in self.ignored_globs
207 + )
208 +
209 + @staticmethod
210 + def _arg_covers(path_arg: str, test_file: str) -> bool:
211 + """True if a single pytest path argument selects ``test_file`` - by
212 + exact match, by being a parent directory, or as a glob."""
213 + path_arg = normalize_test_path(path_arg)
214 + if _path_is_within(path_arg, test_file):
215 + return True
216 + return _glob_match_path(path_arg, test_file)
217 +
218 +
219 +@dataclass
220 +class FindSelection:
221 + """A ``find <root> -name ... [-not -name ...] [-maxdepth N] | xargs pytest``
222 + pipeline. ``pytest`` runs one file per ``find`` hit, so the ``find``
223 + expression alone defines the selected set."""
224 +
225 + root: str
226 + name_globs: list[str] = field(default_factory=list)
227 + exclude_name_globs: list[str] = field(default_factory=list)
228 + max_depth: int | None = None
229 +
230 + def runs(self, test_file: str) -> bool:
231 + root = normalize_test_path(self.root)
232 + if not _path_is_within(root, test_file):
233 + return False
234 +
235 + # `find <root>` puts a direct child at depth 1, whether <root> is a real
236 + # subdir ("compile") or the tests/ root itself ("" / ".").
237 + below = test_file if root in ("", ".") else test_file[len(root) :]
238 + depth_below_root = below.lstrip("/").count("/") + 1
239 + if self.max_depth is not None and depth_below_root > self.max_depth:
240 + return False
241 +
242 + filename = test_file.rsplit("/", 1)[-1]
243 + if self.name_globs and not _glob_any(filename, self.name_globs):
244 + return False
245 + return not _glob_any(filename, self.exclude_name_globs)
246 +
247 +
248 +Selection = PytestSelection | FindSelection
249 +
250 +
251 +def _glob_any(name: str, globs: list[str]) -> bool:
252 + return any(fnmatch.fnmatch(name, glob) for glob in globs)
253 +
254 +
255 +# --------------------------------------------------------------------------- #
256 +# Parsing CI commands into selections
257 +# --------------------------------------------------------------------------- #
258 +
259 +
260 +def _parse_pytest_command(tokens: list[str]) -> PytestSelection:
261 + """Read a ``pytest ...`` invocation's tokens into a :class:`PytestSelection`.
262 +
263 + Positional path args are the included paths; ``--ignore`` values (and a
264 + bare-path ``--deselect``) are ignored paths, ``--ignore-glob`` values are
265 + ignored globs. Value-taking options and their values, and every other flag,
266 + are skipped. ``-k`` / ``-m`` filtering is intentionally *not* modelled - a
267 + file that a marker expression narrows is still "run".
268 + """
269 + selection = PytestSelection()
270 +
271 + # Advance past the pytest token itself (and any env-var / wrapper prefix).
272 + tokens = iter(tokens)
273 + for token in tokens:
274 + if token in PYTEST_COMMANDS:
275 + break
276 +
277 + for token in tokens:
278 + opt, eq, inline = token.partition("=")
279 + if opt in PYTEST_IGNORE_OPTIONS:
280 + value = inline if eq else next(tokens, "")
281 + if opt == "--ignore-glob":
282 + selection.ignored_globs.append(value)
283 + else:
284 + selection.ignored_paths.append(value)
285 + elif opt == PYTEST_DESELECT_OPTION:
286 + value = inline if eq else next(tokens, "")
287 + # A `path::node_id` deselect leaves the file collected; only a
288 + # bare-path deselect removes anything at file granularity.
289 + if value and "::" not in value:
290 + selection.ignored_paths.append(value)
291 + elif token in PYTEST_OPTIONS_WITH_VALUE:
292 + next(tokens, None) # consume and discard the option's value
293 + elif token.startswith("-"):
294 + continue
295 + elif _token_is_test_path(token):
296 + selection.included_paths.append(token)
297 +
298 + return selection
299 +
300 +
301 +def _parse_find_command(tokens: list[str]) -> FindSelection | None:
302 + """Read a ``find <root> -name ...`` command's tokens into a
303 + :class:`FindSelection`, or return ``None`` if there is no usable ``find``."""
304 + if "find" not in tokens:
305 + return None
306 +
307 + root: str | None = None
308 + name_globs: list[str] = []
309 + exclude_name_globs: list[str] = []
310 + max_depth: int | None = None
311 +
312 + tokens = iter(tokens[tokens.index("find") + 1 :])
313 + negated = False
314 + for token in tokens:
315 + if token in ("-not", "!"):
316 + negated = True
317 + continue
318 + if token in ("-name", "-iname"):
319 + glob = next(tokens, "").strip("'\"")
320 + (exclude_name_globs if negated else name_globs).append(glob)
321 + elif token == "-maxdepth":
322 + depth = next(tokens, "")
323 + max_depth = int(depth) if depth.isdigit() else None
324 + elif (
325 + root is None and not token.startswith("-") and token not in PYTEST_COMMANDS
326 + ):
327 + root = token
328 + negated = False
329 +
330 + if root is None:
331 + return None
332 + return FindSelection(root, name_globs, exclude_name_globs, max_depth)
333 +
334 +
335 +def _dash_c_code_operand(tokens: list[str]) -> str | None:
336 + """For ``bash -c '<code>' [$0] [args]`` / ``sh -c ...`` / a combined short
337 + group with ``c`` anywhere in it (``-ec``, ``-ce``, ``-euxc``), the ``<code>``
338 + string the shell runs. ``None`` when this is not a ``-c`` invocation (plain
339 + ``bash script.sh`` / ``bash -x script.sh``). Everything after the code
340 + operand is a positional parameter (``$0``, ``$1``, ...), not a script or
341 + command that executes."""
342 + if not tokens or tokens[0].rsplit("/", 1)[-1] not in ("bash", "sh"):
343 + return None
344 + for i, token in enumerate(tokens[1:], start=1):
345 + if not token.startswith("-"):
346 + return None # script operand reached; a later `-c` is *its* flag
347 + # a short-option group (single dash) with `c` among its flags.
348 + if not token.startswith("--") and "c" in token[1:]:
349 + return tokens[i + 1] if i + 1 < len(tokens) else None
350 + return None
351 +
352 +
353 +def _parse_shell_script(tokens: list[str], visited: set[str]) -> list[Selection]:
354 + """Follow a ``bash tests/foo.sh`` command: read the script and parse the
355 + pytest / runner lines inside it. ``visited`` stops a script that (directly or
356 + otherwise) refers back to itself; nesting is one level deep in practice.
357 + """
358 + if _dash_c_code_operand(tokens) is not None:
359 + return [] # the `-c` code string is parsed by the caller, not a file
360 + script_arg = next((t for t in tokens if t.endswith(".sh")), None)
361 + if script_arg is None:
362 + return []
363 +
364 + rel = script_arg.strip().strip("'\"").replace("/vllm-workspace/", "")
365 + rel = rel.replace("vllm-workspace/", "").removeprefix("./").removeprefix("/")
366 + candidates = [REPO_ROOT / rel, TESTS_DIR / normalize_test_path(rel)]
367 + script_path = next((p for p in candidates if p.is_file()), None)
368 + if script_path is None or str(script_path) in visited:
369 + return []
370 + visited.add(str(script_path))
371 +
372 + # Join `\`-continued lines (a `pytest ... \` with its path on the next line
373 + # is common), then track literal `VAR=value` assignments so a later
374 + # `pytest "$VAR"` - the import-canary pattern - resolves to a real path.
375 + text = script_path.read_text().replace("\\\n", " ")
376 + var_values: dict[str, str] = {}
377 + selections: list[Selection] = []
378 + for line in text.splitlines():
379 + line = line.strip()
380 + if not line or line.startswith("#"):
381 + continue
382 + assignment = re.match(r"^(\w+)=(\S+)$", line)
383 + if assignment:
384 + var_values[assignment.group(1)] = assignment.group(2).strip("'\"")
385 + continue
386 + if var_values:
387 + line = _expand_vars(line, var_values)
388 + selections.extend(_parse_command(line, visited))
389 + return selections
390 +
391 +
392 +def _expand_vars(line: str, var_values: dict[str, str]) -> str:
393 + """Substitute known ``$VAR`` / ``${VAR}`` in a script line - but not inside
394 + single quotes, where the shell does not expand (``pytest '$X'`` passes a
395 + literal ``$X``, collecting nothing)."""
396 +
397 + def sub(segment: str) -> str:
398 + return re.sub(
399 + r"\$\{(\w+)\}|\$(\w+)",
400 + lambda m: var_values.get(m.group(1) or m.group(2), m.group(0)),
401 + segment,
402 + )
403 +
404 + # Splitting on `'`: even-indexed segments are outside single quotes, odd are
405 + # inside. (A `'` within double quotes splits wrongly, but that only ever
406 + # drops a substitution - the safe direction.)
407 + parts = line.split("'")
408 + return "'".join(sub(p) if i % 2 == 0 else p for i, p in enumerate(parts))
409 +
410 +
411 +# Shell tokens (or runs of them) that separate one sub-command from the next.
412 +_SHELL_SEP_CHARS = set("|&;()<>")
413 +_RUNNER_KEYWORDS = PYTEST_COMMANDS | FILE_RUNNER_COMMANDS
414 +
415 +# Tokens that can precede the real command word without being it.
416 +_COMMAND_WRAPPERS = {"if", "!", "then", "time", "exec", "nice", "command", "builtin"}
417 +
418 +
419 +def _effective_command_index(tokens: list[str]) -> int:
420 + """Index of the token that names the command actually executed, past leading
421 + ``FOO=bar`` assignments and simple wrappers - ``if ! env FOO=1 bash x.sh``
422 + resolves to the ``bash``, so ``echo x.sh`` stays ``echo`` and is not
423 + followed. Returns ``len(tokens)`` when nothing executable is left."""
424 + i = 0
425 + while i < len(tokens):
426 + token = tokens[i]
427 + if _is_env_assignment(token) or token in _COMMAND_WRAPPERS:
428 + i += 1
429 + elif token == "env":
430 + i += 1
431 + while i < len(tokens) and (
432 + _is_env_assignment(tokens[i]) or "$" in tokens[i] or tokens[i] == "-"
433 + ):
434 + i += 1
435 + elif token == "timeout" and i + 1 < len(tokens):
436 + i += 1
437 + while i < len(tokens) and tokens[i].startswith("-"):
438 + # `-k` / `--kill-after` / `-s` / `--signal` take a value.
439 + if tokens[i] in ("-k", "--kill-after", "-s", "--signal"):
440 + i += 1
441 + i += 1
442 + i += 1 # the DURATION operand
443 + else:
444 + break
445 + return i
446 +
447 +
448 +def _split_shell_stages(command: str) -> list[tuple[str, list[str]]]:
449 + """Tokenize a shell line - honouring quotes - and split it at the shell
450 + operators, tagging each stage with the operator token that *precedes* it
451 + (``""`` for the first stage, then ``"|"`` / ``"&&"`` / ``"||"`` / ``";"`` /
452 + ``"&"`` / a subshell paren). Knowing which operator joins two stages is what
453 + lets a ``find`` sweep count only when it is genuinely piped into ``pytest``,
454 + not merely on the same line as one. A line that cannot be tokenized
455 + (unbalanced quote, ...) yields nothing: better to miss a selection than to
456 + misparse one. Splitting *after* tokenizing means an operator inside a quoted
457 + argument stays part of that argument."""
458 + lexer = shlex.shlex(command, posix=True, punctuation_chars=True)
459 + lexer.whitespace_split = True
460 + try:
461 + tokens = list(lexer)
462 + except ValueError:
463 + return []
464 +
465 + stages: list[tuple[str, list[str]]] = [("", [])]
466 + in_exec = False # inside a `find -exec ... \;` action - its `;` terminates
467 + for token in tokens:
468 + if in_exec and token in (";", "+"):
469 + stages[-1][1].append(token)
470 + in_exec = False
471 + elif token and set(token) <= _SHELL_SEP_CHARS:
472 + stages.append((token, []))
473 + in_exec = False
474 + else:
475 + stages[-1][1].append(token)
476 + if token in ("-exec", "-execdir"):
477 + in_exec = True
478 + return [(sep, sub) for sep, sub in stages if sub]
479 +
480 +
481 +def _looks_like_command_string(token: str) -> bool:
482 + """True for a single token that is itself a command line - e.g. a quoted
483 + ``"pytest a.py && torchrun b.py"`` passed as an argument to a runner
484 + script."""
485 + return " " in token and any(word in _RUNNER_KEYWORDS for word in token.split())
486 +
487 +
488 +def _classify_subcommand(tokens: list[str], visited: set[str]) -> list[Selection]:
489 + """Map one already-tokenized sub-command to the selections it implies:
490 + find / pytest / ``bash <script>`` / direct file runner. Anything else
491 + contributes nothing - better to miss a real selection than to invent
492 + coverage that isn't there."""
493 + if not tokens:
494 + return []
495 +
496 + is_find = "find" in tokens and any(t in ("-name", "-iname") for t in tokens)
497 + if is_find:
498 + find_selection = _parse_find_command(tokens)
499 + return [find_selection] if find_selection is not None else []
500 +
501 + # A `pytest` token only counts when `pytest` is the command that actually
502 + # runs - `echo pytest kernels` prints text, it does not collect anything.
503 + if _runs_pytest(tokens):
504 + return [_parse_pytest_command(tokens)]
505 +
506 + # Resolve the real command word past `FOO=1` / `env` / `if ! ...` wrappers,
507 + # so `env FOO=1 bash x.sh` is followed but `echo x.sh` (Buildkite just
508 + # printing a filename) is not.
509 + cmd_index = _effective_command_index(tokens)
510 + command_name = (
511 + tokens[cmd_index].rsplit("/", 1)[-1] if cmd_index < len(tokens) else ""
512 + )
513 +
514 + if command_name in ("bash", "sh") or command_name.endswith(".sh"):
515 + run_tokens = tokens[cmd_index:]
516 + code = _dash_c_code_operand(run_tokens)
517 + if code is not None:
518 + # `bash -c '<code>' [$0] [args]` - only the `-c` string runs; the
519 + # trailing args are positional params, not commands.
520 + return _parse_command(code, visited)
521 + selections = _parse_shell_script(run_tokens, visited)
522 + # Runner scripts (run-multi-node-test.sh, ...) take the real test
523 + # commands as quoted string arguments - parse those too.
524 + for arg in run_tokens[1:]:
525 + if _looks_like_command_string(arg):
526 + selections.extend(_parse_command(arg, visited))
527 + return selections
528 +
529 + # `python` / `torchrun` / `coverage <file>.py`.
530 + if command_name in FILE_RUNNER_COMMANDS:
531 + rest = tokens[cmd_index + 1 :]
532 + # `python -c '<code>' [argv...]` runs the code string, not a file - a
533 + # later `-m pytest` or `.py` is that program's argv. Only honour a `-c`
534 + # that is the interpreter's own (nothing script-like precedes it).
535 + if "-c" in rest:
536 + before_c = rest[: rest.index("-c")]
537 + if "-m" not in before_c and not any(t.endswith(".py") for t in before_c):
538 + return []
539 + # `-m <module>` picks the module, not a script: only `-m pytest` runs
540 + # tests. `python -m compileall kernels` collects nothing - the trailing
541 + # `kernels` is compileall's argv, not a test path.
542 + if "-m" in rest:
543 + after_m = rest[rest.index("-m") + 1 :]
544 + if after_m[:1] and after_m[0] in PYTEST_COMMANDS:
545 + return [_parse_pytest_command(after_m)]
546 + return []
547 + # `python foo.py a b/c --suffix v1` executes only `foo.py`; the tokens
548 + # after it are that script's argv - treating them as test paths tethers
549 + # whatever directory an argument happens to name (`v1` -> `tests/v1`).
550 + script = next(
551 + (t for t in rest if t.endswith(".py") and _token_is_test_path(t)), None
552 + )
553 + if script is not None:
554 + return [PytestSelection(included_paths=[script])]
555 +
556 + return []
557 +
558 +
559 +# `xargs` options that consume the following token as their value.
560 +_XARGS_VALUE_FLAGS = {
561 + "-n", "-L", "-P", "-s", "-I", "-i", "-J", "-E", "-d",
562 + "--max-args", "--max-lines", "--max-procs", "--replace", "--delimiter",
563 +} # fmt: skip
564 +
565 +
566 +def _xargs_command_tokens(tokens: list[str]) -> list[str]:
567 + """The command (plus its args) that ``xargs`` will exec, with ``xargs``' own
568 + options stripped: ``xargs -0 -n1 -I{} env FOO=1 pytest`` -> ``['env',
569 + 'FOO=1', 'pytest']``."""
570 + i = 0
571 + while i < len(tokens):
572 + if tokens[i] in _XARGS_VALUE_FLAGS:
573 + i += 2
574 + elif tokens[i].startswith("-"):
575 + i += 1
576 + else:
577 + return tokens[i:]
578 + return []
579 +
580 +
581 +def _runs_pytest(tokens: list[str]) -> bool:
582 + """True if ``tokens`` is a command that executes ``pytest``, seeing through
583 + ``env FOO=1`` / ``timeout`` / ``if !`` wrappers (``xargs echo pytest`` is
584 + not - ``echo`` is what runs)."""
585 + idx = _effective_command_index(tokens)
586 + return idx < len(tokens) and tokens[idx] in PYTEST_COMMANDS
587 +
588 +
589 +# Pipe stages that reorder or echo the stream without dropping any line, so
590 +# every `find` match still reaches a downstream `xargs pytest`. Anything else
591 +# (`head`, `grep`, `shuf`, `sed -n`, ...) filters, and then only some matches
592 +# run - not enough to claim the whole sweep is tethered.
593 +_PIPE_PASSTHROUGH = {"sort", "tee", "cat", "tac"}
594 +
595 +
596 +def _find_feeds_pytest(stages: list[tuple[str, list[str]]], find_index: int) -> bool:
597 + """True if the ``find`` stage at ``find_index`` actually runs its matches
598 + through ``pytest``: a *terminated* ``-exec pytest ... \\;`` / ``... +`` in the
599 + ``find`` itself, or an ``xargs`` whose command operand is ``pytest``, reached
600 + through ``|`` pipes of non-filtering stages only. A ``find`` piped to
601 + ``wc`` / ``head`` / an ``xargs echo``, an unterminated ``-exec``, or a
602 + ``pytest`` in a separate ``;`` / ``&&`` command, runs nothing (or not the
603 + whole sweep)."""
604 + find_tokens = stages[find_index][1]
605 + for i, token in enumerate(find_tokens):
606 + if token in ("-exec", "-execdir"):
607 + action = find_tokens[i + 1 :]
608 + if _exec_runs_matches_through_pytest(action):
609 + return True
610 + for sep, tokens in stages[find_index + 1 :]:
611 + if sep != "|":
612 + break
613 + if tokens[:1] == ["xargs"] and _runs_pytest(_xargs_command_tokens(tokens[1:])):
614 + return True
615 + if tokens[:1] and tokens[0] in _PIPE_PASSTHROUGH:
616 + continue
617 + break
618 + return False
619 +
620 +
621 +def _terminated(exec_action: list[str]) -> bool:
622 + """A ``find -exec`` action only runs if it ends with a ``;`` or ``+`` token
623 + (from ``\\;`` / ``+`` on the command line)."""
624 + return ";" in exec_action or "+" in exec_action
625 +
626 +
627 +def _exec_runs_matches_through_pytest(action: list[str]) -> bool:
628 + """True if a ``find -exec``/``-execdir`` action runs ``pytest`` over each
629 + *matched* file: ``pytest`` is the command, the action is terminated
630 + (``\\;`` / ``+``), and the ``{}`` placeholder lands as a positional test-path
631 + argument - not as the value of an option (``-exec pytest --ignore {} \\;``
632 + only ever runs whatever positional file is also on the line)."""
633 + if not action or action[0] not in PYTEST_COMMANDS or not _terminated(action):
634 + return False
635 + # Substitute a real-looking path for `{}` and see whether pytest's own arg
636 + # parser keeps it as an included path rather than eating it as an option value.
637 + probe = [t.replace("{}", "_find_match_.py") for t in action[:-1]]
638 + selection = _parse_pytest_command(probe)
639 + return any("_find_match_" in path for path in selection.included_paths)
640 +
641 +
642 +def _parse_command(command: str, visited: set[str] | None = None) -> list[Selection]:
643 + """Turn one yaml ``commands:`` entry into the selections it implies.
644 +
645 + The line is tokenized (respecting quotes) and split into stages at the shell
646 + operators, and each stage is classified independently so that, e.g., a
647 + ``find`` and a ``pytest`` on the same line are both handled. A ``find -name``
648 + sweep is kept only when :func:`_find_feeds_pytest` confirms its output
649 + actually reaches ``pytest`` - otherwise it would invent coverage.
650 + """
651 + visited = visited if visited is not None else set()
652 + stages = _split_shell_stages(command)
653 + selections: list[Selection] = []
654 + for index, (_, tokens) in enumerate(stages):
655 + for selection in _classify_subcommand(tokens, visited):
656 + if isinstance(selection, FindSelection) and not _find_feeds_pytest(
657 + stages, index
658 + ):
659 + continue
660 + selections.append(selection)
661 + return selections
662 +
663 +
664 +def _iter_job_steps(yaml_doc: dict):
665 + """Yield each job step in a test_areas yaml, plus a merged view of each
666 + ``mirror.*`` sub-step. The AMD mirror can override ``commands``, so it has to
667 + be checked as its own step."""
668 + for step in yaml_doc.get("steps") or []:
669 + if not isinstance(step, dict):
670 + continue
671 + yield step
672 +
673 + for mirror_step in (step.get("mirror") or {}).values():
674 + if isinstance(mirror_step, dict):
675 + merged = {**step, **mirror_step}
676 + merged.pop("mirror", None)
677 + yield merged
678 +
679 +
680 +def _pipeline_job_dirs() -> list[Path]:
681 + """The Buildkite job directories, read from every ``.buildkite/ci_config*.yaml``
682 + (``job_dirs:``). Buildkite itself uses these files to assemble each pipeline,
683 + so a test wired into any job under any of them is genuinely run by CI. Falls
684 + back to ``test_areas/`` alone if no config is readable."""
685 + job_dirs: set[Path] = {TEST_AREAS_DIR}
686 + for config in sorted((REPO_ROOT / ".buildkite").glob("ci_config*.yaml")):
687 + try:
688 + doc = yaml.safe_load(config.read_text()) or {}
689 + except yaml.YAMLError:
690 + continue
691 + for rel in doc.get("job_dirs") or []:
692 + job_dirs.add(REPO_ROOT / rel)
693 + return sorted(job_dirs)
694 +
695 +
696 +def _pipeline_yaml_paths() -> list[Path]:
697 + """Every Buildkite pipeline yaml whose jobs count as CI coverage: the yamls
698 + in every :func:`_pipeline_job_dirs` directory plus the legacy ``test-amd.yaml``
699 + (see ``TEST_AMD_YAML``)."""
700 + paths: list[Path] = []
701 + for job_dir in _pipeline_job_dirs():
702 + paths.extend(sorted(job_dir.glob("*.yaml")))
703 + if TEST_AMD_YAML.is_file():
704 + paths.append(TEST_AMD_YAML)
705 + return paths
706 +
707 +
708 +def load_selections() -> list[Selection]:
709 + """Parse every CI pipeline yaml into the full list of selections - the CI's
710 + complete picture of which test files it runs.
711 +
712 + A yaml that doesn't parse is fatal: silently skipping it would drop whatever
713 + coverage it defines and produce false "untethered" reports.
714 + """
715 + selections: list[Selection] = []
716 + for yaml_path in _pipeline_yaml_paths():
717 + try:
718 + yaml_doc = yaml.safe_load(yaml_path.read_text()) or {}
719 + except yaml.YAMLError as e:
720 + rel = yaml_path.relative_to(REPO_ROOT)
721 + raise SystemExit(f"error: {rel} is not valid YAML: {e}") from None
722 + if not isinstance(yaml_doc, dict):
723 + rel = yaml_path.relative_to(REPO_ROOT)
724 + raise SystemExit(
725 + f"error: {rel} is not a Buildkite pipeline "
726 + f"(top-level {type(yaml_doc).__name__}, expected a mapping)"
727 + )
728 + for step in _iter_job_steps(yaml_doc):
729 + commands = list(step.get("commands") or [])
730 + if step.get("command"): # some steps use the singular form
731 + commands.append(step["command"])
732 + for command in commands:
733 + if isinstance(command, str):
734 + selections.extend(_parse_command(command))
735 + return selections
736 +
737 +
738 +def is_tethered(test_file: str, selections: list[Selection]) -> bool:
739 + """True if any CI selection runs ``test_file`` (path relative to ``tests/``
740 + or to the repo root - both are accepted)."""
741 + normalized = normalize_test_path(test_file)
742 + return any(selection.runs(normalized) for selection in selections)
743 +
744 +
745 +# --------------------------------------------------------------------------- #
746 +# Test-file inventory and allowlist
747 +# --------------------------------------------------------------------------- #
748 +
749 +
750 +def is_test_module(repo_path: str) -> bool:
751 + """True if ``repo_path`` is a pytest test module CI is expected to run - a
752 + ``tests/**/test_*.py`` or ``*_test.py``, but not ``conftest.py`` /
753 + ``__init__.py`` or other helper modules."""
754 + if not repo_path.startswith("tests/") or not repo_path.endswith(".py"):
755 + return False
756 + filename = repo_path.rsplit("/", 1)[-1]
757 + if filename in ("conftest.py", "__init__.py"):
758 + return False
759 + return filename.startswith("test_") or filename.endswith("_test.py")
760 +
761 +
762 +def all_test_modules() -> list[str]:
763 + """Every test module under ``tests/``, sorted. Prefers ``git ls-files`` so a
764 + developer's local scratch files are never flagged; falls back to a
765 + filesystem walk when run outside a git checkout - the CI test images bake in
766 + ``tests/`` with no ``.git``, and there are no untracked scratch files there.
767 + """
768 + try:
769 + tracked = subprocess.run(
770 + ["git", "-C", str(REPO_ROOT), "ls-files", "tests/"],
771 + capture_output=True,
772 + text=True,
773 + check=True,
774 + ).stdout
775 + paths = tracked.splitlines()
776 + except (OSError, subprocess.CalledProcessError):
777 + paths = [p.relative_to(REPO_ROOT).as_posix() for p in TESTS_DIR.rglob("*.py")]
778 + return sorted(path for path in paths if is_test_module(path))
779 +
780 +
781 +def load_allowlist() -> set[str]:
782 + """Read the allowlist file into a set of repo-relative paths. Blank lines and
783 + ``#`` comments (whole-line or trailing) are ignored."""
784 + if not ALLOWLIST_PATH.exists():
785 + return set()
786 +
787 + paths = set()
788 + for line in ALLOWLIST_PATH.read_text().splitlines():
789 + path = line.split("#", 1)[0].strip()
790 + if path:
791 + paths.add(path)
792 + return paths
793 +
794 +
795 +def allowlist_entries_missing_reason() -> list[str]:
796 + """Allowlist paths with no trailing ``# reason`` on their own line. Every gap
797 + has to say why the test can't be wired into a job, so a bare ``path`` line
798 + (the easy way to silence the gate) is itself an error."""
799 + if not ALLOWLIST_PATH.exists():
800 + return []
801 +
802 + offenders = []
803 + for line in ALLOWLIST_PATH.read_text().splitlines():
804 + path, _, comment = line.partition("#")
805 + if path.strip() and not comment.strip():
806 + offenders.append(path.strip())
807 + return offenders
808 +
809 +
810 +# --------------------------------------------------------------------------- #
811 +# The two check modes
812 +# --------------------------------------------------------------------------- #
813 +
814 +
815 +def run_full_scan(
816 + selections: list[Selection], allowlist: set[str], *, strict: bool
817 +) -> int:
818 + """Scan every test module. Untethered files that aren't allowlisted are
819 + always errors. Allowlist entries that have become tethered, or whose file no
820 + longer exists, are reported either way but are only errors when ``strict``
821 + (an explicit ``--all`` run, as opposed to a PR that happened to touch CI
822 + config - which shouldn't be blocked by unrelated allowlist cleanup).
823 + """
824 + untethered = [
825 + path
826 + for path in all_test_modules()
827 + if path not in allowlist and not is_tethered(path, selections)
828 + ]
829 + now_tethered = sorted(path for path in allowlist if is_tethered(path, selections))
830 + deleted = sorted(path for path in allowlist if not (REPO_ROOT / path).exists())
831 +
832 + for path in untethered:
833 + print(f"error: {path} is not run by any Buildkite job")
834 + for path in now_tethered:
835 + print(f"note: {path} is now tethered - remove it from the allowlist")
836 + for path in deleted:
837 + print(f"note: {path} no longer exists - remove it from the allowlist")
838 +
839 + if untethered:
840 + print(
841 + f"\n{len(untethered)} test file(s) exist but run in no CI job. Wire "
842 + "each into a job's `commands` in a .buildkite/ pipeline yaml, or add "
843 + f"it to {ALLOWLIST_PATH.relative_to(REPO_ROOT)} with a reason."
844 + )
845 +
846 + error_count = len(untethered)
847 + if strict:
848 + error_count += len(now_tethered) + len(deleted)
849 + return 1 if error_count else 0
850 +
851 +
852 +def run_changed_files_check(
853 + paths: list[str], selections: list[Selection], allowlist: set[str]
854 +) -> int:
855 + """Check only the given paths (what pre-commit passes). Non-test files, and
856 + paths that don't exist on disk, are ignored."""
857 + untethered = []
858 + for raw_path in paths:
859 + repo_path = _to_repo_relative(raw_path)
860 + if not is_test_module(repo_path) or not (REPO_ROOT / repo_path).is_file():
861 + continue
862 + if repo_path in allowlist or is_tethered(repo_path, selections):
863 + continue
864 + untethered.append(repo_path)
865 +
866 + for path in untethered:
867 + print(f"error: {path} is not run by any Buildkite job")
868 + if untethered:
869 + print(
870 + "\nAdd the test to a job's `commands` (and `source_file_dependencies`)"
871 + " in a .buildkite/ pipeline yaml, or, if it is intentionally not run "
872 + f"in CI, add it to {ALLOWLIST_PATH.relative_to(REPO_ROOT)} with a "
873 + "comment explaining why."
874 + )
875 + return 1 if untethered else 0
876 +
877 +
878 +def _to_repo_relative(path: str) -> str:
879 + if Path(path).is_absolute():
880 + path = str(Path(path).resolve().relative_to(REPO_ROOT))
881 + return path.replace("\\", "/")
882 +
883 +
884 +def _change_set_touches_ci_config(paths: list[str]) -> bool:
885 + """A change to a pipeline yaml, a ci_config, or the allowlist can orphan a
886 + test that is not itself in the change set, so those changes force a full
887 + scan."""
888 + return any(
889 + ".buildkite/test_areas/" in path
890 + or ".buildkite/hardware_tests/" in path
891 + or ".buildkite/intel_jobs/" in path
892 + or ".buildkite/ci_config" in path
893 + or path.endswith("test-amd.yaml")
894 + or path.endswith("test_tethering_allowlist.txt")
895 + for path in paths
896 + )
897 +
898 +
899 +def main() -> int:
900 + parser = argparse.ArgumentParser(description=__doc__)
901 + parser.add_argument("files", nargs="*", help="test files to check")
902 + parser.add_argument(
903 + "--all", action="store_true", help="scan every test file in the tree"
904 + )
905 + args = parser.parse_args()
906 +
907 + selections = load_selections()
908 + allowlist = load_allowlist()
909 +
910 + unreasoned = allowlist_entries_missing_reason()
911 + for path in unreasoned:
912 + print(
913 + f"error: {ALLOWLIST_PATH.relative_to(REPO_ROOT)}: {path} has no "
914 + "trailing '# <reason>' - say why it can't be wired into a job"
915 + )
916 +
917 + if args.all or _change_set_touches_ci_config(args.files):
918 + rc = run_full_scan(selections, allowlist, strict=args.all)
919 + else:
920 + rc = run_changed_files_check(args.files, selections, allowlist)
921 + return 1 if (unreasoned or rc) else 0
922 +
923 +
924 +if __name__ == "__main__":
925 + sys.exit(main())
tools/pre_commit/test_tethering_allowlist.txt new
+100
@@ -0,0 +1,100 @@
1 +# Allowlist for tools/pre_commit/check_test_tethering.py
2 +#
3 +# Every path below is a test file that is collected by NO Buildkite job in any
4 +# pipeline yaml the checker parses (the job_dirs of .buildkite/ci_config*.yaml -
5 +# test_areas/, hardware_tests/, intel_jobs/ - plus .buildkite/test-amd.yaml).
6 +# These are pre-existing gaps, captured when the check was first added so that
7 +# the gate could start green. The list is meant to only ever shrink.
8 +#
9 +# Rules:
10 +# * Every entry needs a trailing `# <reason>` on its own line saying why the
11 +# test isn't wired into a job. A bare path is rejected by the check.
12 +# * Do not add a new entry to unblock a PR without a real justification.
13 +# * To remove an entry: add the test to a job's `commands` (and matching
14 +# `source_file_dependencies`) in a .buildkite/ pipeline yaml, then delete the
15 +# line.
16 +# * `check_test_tethering.py --all` reports entries here that have since
17 +# become tethered, and entries whose file no longer exists.
18 +
19 +# tests/distributed/ - multi-GPU / multi-node suites, launched via torchrun or a
20 +# custom runner rather than a directory-level pytest invocation.
21 +tests/distributed/aux_output_connector/test_store.py # added by #45635 (2026-09-21), no job collects this path
22 +tests/distributed/test_dcp_a2a.py # multi-GPU DCP all-to-all; no dir-level pytest job
23 +tests/distributed/test_dcp_direct_a2a_lse_reduce.py # multi-GPU DCP all-to-all; no dir-level pytest job
24 +tests/distributed/test_eplb_events.py # multi-GPU EPLB; no dir-level pytest job
25 +tests/distributed/test_eplb_fused_moe_layer.py # multi-GPU EPLB; no dir-level pytest job
26 +tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py # multi-GPU EPLB nvfp4; no dir-level pytest job
27 +tests/distributed/test_eplb_quant_scale_consistency.py # multi-GPU EPLB; no dir-level pytest job
28 +tests/distributed/test_expert_parallel.py # multi-GPU expert parallel; no dir-level pytest job
29 +tests/distributed/test_expert_placement.py # multi-GPU expert placement; no dir-level pytest job
30 +tests/distributed/test_file_store.py # no distributed.yaml job collects this path
31 +tests/distributed/test_flashinfer_pcie_ipc_all_reduce.py # multi-GPU PCIe IPC all-reduce (#53576); no job
32 +tests/distributed/test_kimi_linear_context_parallel.py # multi-GPU context parallel; no dir-level pytest job
33 +tests/distributed/test_kv_cache_events.py # needs multi-proc KV-events setup; no job
34 +tests/distributed/test_kvlayout.py # no distributed.yaml job collects this path
35 +tests/distributed/test_mq_connect_ip.py # no distributed.yaml job collects this path
36 +tests/distributed/test_pipeline_partition.py # no distributed.yaml job collects this path
37 +tests/distributed/test_split_group.py # no distributed.yaml job collects this path
38 +tests/distributed/test_weight_transfer_nccl_uid.py # sibling test_weight_transfer.py is tethered; not added to that job (#53784)
39 +
40 +# tests/entrypoints/cli/
41 +tests/entrypoints/cli/test_serve.py # added by #53124 (2026-09-15); no job collects tests/entrypoints/cli/
42 +
43 +# tests/kernels/
44 +tests/kernels/test_shuffle_rows.py # explicitly --ignore'd in kernels.yaml with no follow-up job
45 +
46 +# tests/lora/ - explicitly --ignore'd from the sharded LoRA run, with no
47 +# dedicated *_tp job (unlike the other ignored *_tp tests).
48 +tests/lora/test_deepseekv2_tp.py # --ignore'd from sharded LoRA run; no dedicated *_tp job
49 +tests/lora/test_qwen3moe_tp.py # --ignore'd from sharded LoRA run; no dedicated *_tp job
50 +
51 +# tests/models/ - the models_*.yaml jobs run specific paths, not all of models/.
52 +tests/models/deepseek_v32/test_sequence_parallel.py # no models_*.yaml job collects this path
53 +tests/models/test_deepseek_v4_dspark_rocm.py # added by #52362 (2026-09-21), no models_*.yaml job collects this path
54 +tests/models/test_deepseek_v4_fi_moe_ep.py # no models_*.yaml job collects this path
55 +tests/models/test_deepseek_v4_rocm_compressor_gemm_fusion.py # no models_*.yaml job collects this path
56 +tests/models/test_deepseek_v4_rocm_wo_a.py # added by #54894 (2026-09-20), no models_*.yaml job collects this path
57 +tests/models/test_deepseek_v41_replay_start.py # added by #56227 (2026-09-18), no models_*.yaml job collects this path
58 +tests/models/test_dspark_mla.py # no models_*.yaml job collects this path
59 +tests/models/test_language_model_cache_is_weak.py # no models_*.yaml job collects this path
60 +tests/models/transformers/test_layer_registry.py # no models_*.yaml job collects this path
61 +
62 +# tests/ root-level - misc.yaml runs a hand-picked set of these by name
63 +# (test_regression.py, test_envs.py, test_outputs.py, test_pooling_params.py);
64 +# the rest are unreferenced.
65 +tests/test_access_log_filter.py # root test_*.py not in misc.yaml's by-name list
66 +tests/test_attention_backend_registry.py # root test_*.py not in misc.yaml's by-name list
67 +tests/test_audio_media_size_precheck.py # root test_*.py not in misc.yaml's by-name list
68 +tests/test_cmake_utils.py # root test_*.py not in misc.yaml's by-name list
69 +tests/test_connections.py # root test_*.py not in misc.yaml's by-name list
70 +tests/test_embedded_commit.py # root test_*.py not in misc.yaml's by-name list
71 +tests/test_force_first_config.py # root test_*.py not in misc.yaml's by-name list
72 +tests/test_fxgraphcache_pickle_patch.py # root test_*.py not in misc.yaml's by-name list
73 +tests/test_logprobs.py # root test_*.py not in misc.yaml's by-name list
74 +tests/test_quantization_revision_pin.py # root test_*.py not in misc.yaml's by-name list
75 +tests/test_ray_env_utils.py # root test_*.py not in misc.yaml's by-name list
76 +tests/test_request_input_bounds.py # root test_*.py not in misc.yaml's by-name list
77 +tests/test_scalartype.py # root test_*.py not in misc.yaml's by-name list
78 +tests/test_tensorizer_revision_pin.py # root test_*.py not in misc.yaml's by-name list
79 +tests/test_triton_utils.py # root test_*.py not in misc.yaml's by-name list
80 +tests/test_version.py # root test_*.py not in misc.yaml's by-name list
81 +tests/test_zen_cpu_platform_detection.py # root test_*.py not in misc.yaml's by-name list
82 +
83 +# tests/v1/ - the v1 subtree is otherwise covered by many `pytest v1/...` jobs.
84 +tests/v1/e2e/test_replayssm_decode.py # ReplaySSM decode test (#52506) wired into no job
85 +tests/v1/ec_connector/integration/test_epd_correctness.py # run_epd_correctness_test.sh exists but no Buildkite job invokes it
86 +tests/v1/ec_connector/integration/test_nixl_failure.py # 2-GPU + NIXL E/PD integration test (#55290); no Buildkite job collects ec_connector/integration/
87 +tests/v1/kv_connector/nixl_integration/test_disagg_accuracy.py # run by intel_jobs/misc_intel.yaml's run_xpu_disagg_accuracy_test.sh, which names it via $EXP_ROOT (the script's own dir) - not statically resolvable
88 +
89 +# tests/platforms/
90 +tests/platforms/test_cpu.py # CPU-only unit test; no job targets tests/platforms/
91 +
92 +# tests/tools/
93 +tests/tools/test_config_validator.py # logic covered by the validate-config pre-commit hook; no test_areas job
94 +
95 +# tests/watermarking/ - added by #54053 (2026-09-09), no Buildkite job collects the dir yet
96 +tests/watermarking/test_detection.py # watermarking feature (#54053); no job collects tests/watermarking/
97 +tests/watermarking/test_gumbel.py # watermarking feature (#54053); no job collects tests/watermarking/
98 +tests/watermarking/test_prf.py # watermarking feature (#54053); no job collects tests/watermarking/
99 +tests/watermarking/test_watermarking.py # watermarking feature (#54053); no job collects tests/watermarking/
100 +tests/watermarking/test_watermarking_e2e.py # watermarking hardening (#56309); no job collects tests/watermarking/