| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Unit tests for the test-tethering pre-commit check. |
| 4 | |
| 5 | The check answers one question - "does any Buildkite job (in |
| 6 | ``.buildkite/test_areas/`` or the legacy ``.buildkite/test-amd.yaml``) collect |
| 7 | this test file?" - by parsing job ``commands`` as shell. The risk that |
| 8 | matters is a *false tether*: a misparse that invents coverage makes the gate |
| 9 | pass while the test still never runs, which is the exact failure the check |
| 10 | exists to catch. So the parser cases below pin both directions, and |
| 11 | :func:`test_no_selection_matches_everything` guards the catastrophic version of |
| 12 | it. |
| 13 | """ |
| 14 | |
| 15 | import textwrap |
| 16 | from pathlib import Path |
| 17 | |
| 18 | import pytest |
| 19 | |
| 20 | import tools.pre_commit.check_test_tethering as checker |
| 21 | from tools.pre_commit.check_test_tethering import ( |
| 22 | FindSelection, |
| 23 | PytestSelection, |
| 24 | _parse_command, |
| 25 | _to_repo_relative, |
| 26 | all_test_modules, |
| 27 | allowlist_entries_missing_reason, |
| 28 | is_test_module, |
| 29 | is_tethered, |
| 30 | load_allowlist, |
| 31 | load_selections, |
| 32 | main, |
| 33 | normalize_test_path, |
| 34 | run_changed_files_check, |
| 35 | run_full_scan, |
| 36 | ) |
| 37 | |
| 38 | REPO_ROOT = Path(__file__).resolve().parents[2] |
| 39 | |
| 40 | |
| 41 | def runs(command: str, test_file: str) -> bool: |
| 42 | """True if `command` collects `test_file` (path relative to ``tests/``).""" |
| 43 | return any(sel.runs(test_file) for sel in _parse_command(command)) |
| 44 | |
| 45 | |
| 46 | # --------------------------------------------------------------------------- # |
| 47 | # pytest path arguments |
| 48 | # --------------------------------------------------------------------------- # |
| 49 | |
| 50 | |
| 51 | @pytest.mark.parametrize( |
| 52 | ("command", "test_file", "expected"), |
| 53 | [ |
| 54 | # Directory sweeps cover everything nested underneath. |
| 55 | ("pytest -v -s kernels", "kernels/test_attention.py", True), |
| 56 | ("pytest -v -s kernels/", "kernels/moe/test_moe.py", True), |
| 57 | ("pytest -v -s kernels", "lora/test_lora.py", False), |
| 58 | # An exact file argument covers that file and nothing beside it. |
| 59 | ("pytest -v -s kernels/test_a.py", "kernels/test_a.py", True), |
| 60 | ("pytest -v -s kernels/test_a.py", "kernels/test_b.py", False), |
| 61 | # Commands run from the repo root and from tests/ alike. |
| 62 | ("pytest -v -s tests/kernels/test_a.py", "kernels/test_a.py", True), |
| 63 | ("pytest -v -s ./kernels/test_a.py", "kernels/test_a.py", True), |
| 64 | ( |
| 65 | "pytest -v -s /vllm-workspace/tests/kernels/test_a.py", |
| 66 | "kernels/test_a.py", |
| 67 | True, |
| 68 | ), |
| 69 | # A ::nodeid selector still collects the file. |
| 70 | ("pytest -v -s kernels/test_a.py::test_one", "kernels/test_a.py", True), |
| 71 | ("pytest -v -s kernels/test_a.py::Cls::test_one", "kernels/test_a.py", True), |
| 72 | # Several positional paths, including a bare root-level file. |
| 73 | ("pytest -v -s test_envs.py test_outputs.py", "test_outputs.py", True), |
| 74 | ("pytest -v -s test_envs.py test_outputs.py", "test_regression.py", False), |
| 75 | # A bare word is a path only when it is really a directory under tests/. |
| 76 | ("pytest -v -s samplers", "samplers/test_sampler.py", True), |
| 77 | # Glob path args do not cross a directory separator, matching the shell. |
| 78 | ("pytest -v -s kernels/test_*.py", "kernels/test_a.py", True), |
| 79 | ("pytest -v -s kernels/test_*.py", "kernels/moe/test_a.py", False), |
| 80 | ("pytest -v -s kernels/test_*.py", "kernels/helper.py", False), |
| 81 | ], |
| 82 | ) |
| 83 | def test_pytest_path_arguments(command, test_file, expected): |
| 84 | assert runs(command, test_file) is expected |
| 85 | |
| 86 | |
| 87 | # --------------------------------------------------------------------------- # |
| 88 | # --ignore / --deselect |
| 89 | # --------------------------------------------------------------------------- # |
| 90 | |
| 91 | |
| 92 | @pytest.mark.parametrize( |
| 93 | ("command", "test_file", "expected"), |
| 94 | [ |
| 95 | # --ignore removes a file from an otherwise covering sweep, in both the |
| 96 | # `=` and space-separated spellings. |
| 97 | ("pytest -v -s kernels --ignore=kernels/test_a.py", "kernels/test_a.py", False), |
| 98 | ("pytest -v -s kernels --ignore kernels/test_a.py", "kernels/test_a.py", False), |
| 99 | # ...and only that file. |
| 100 | ("pytest -v -s kernels --ignore=kernels/test_a.py", "kernels/test_b.py", True), |
| 101 | # --ignore of a directory removes the whole subtree. |
| 102 | ("pytest -v -s kernels --ignore=kernels/moe", "kernels/moe/test_moe.py", False), |
| 103 | ( |
| 104 | "pytest -v -s lora --ignore-glob=lora/*_tp.py", |
| 105 | "lora/test_llama_tp.py", |
| 106 | False, |
| 107 | ), |
| 108 | # --ignore-glob is a whole-path fnmatch (pytest semantics): its `*` |
| 109 | # crosses `/`, so a bare `*_tp.py` drops a nested match too. A positional |
| 110 | # path glob would not - that is the distinction the parser has to keep. |
| 111 | ("pytest -v -s kernels --ignore-glob=*_tp.py", "kernels/test_a_tp.py", False), |
| 112 | ( |
| 113 | "pytest -v -s kernels --ignore-glob=*_tp.py", |
| 114 | "kernels/nested/test_a_tp.py", |
| 115 | False, |
| 116 | ), |
| 117 | ("pytest -v -s kernels --ignore-glob '*_tp.py'", "kernels/test_a.py", True), |
| 118 | # A node-id deselect leaves the file collected; a bare-path one does not. |
| 119 | ( |
| 120 | "pytest -v -s kernels --deselect kernels/test_a.py::test_one", |
| 121 | "kernels/test_a.py", |
| 122 | True, |
| 123 | ), |
| 124 | ( |
| 125 | "pytest -v -s kernels --deselect kernels/test_a.py", |
| 126 | "kernels/test_a.py", |
| 127 | False, |
| 128 | ), |
| 129 | ], |
| 130 | ) |
| 131 | def test_ignore_and_deselect(command, test_file, expected): |
| 132 | assert runs(command, test_file) is expected |
| 133 | |
| 134 | |
| 135 | # --------------------------------------------------------------------------- # |
| 136 | # Options, env prefixes and other token noise |
| 137 | # --------------------------------------------------------------------------- # |
| 138 | |
| 139 | |
| 140 | @pytest.mark.parametrize( |
| 141 | ("command", "test_file", "expected"), |
| 142 | [ |
| 143 | # A -k / -m filter narrows which tests run but does not un-run the file. |
| 144 | ("pytest -v -s kernels -k 'not slow'", "kernels/test_a.py", True), |
| 145 | ("pytest -v -s kernels -m core_model", "kernels/test_a.py", True), |
| 146 | # A value-taking option's value is never mistaken for a path. |
| 147 | ("pytest -m core_model", "core_model", False), |
| 148 | ("pytest -k kernels", "kernels/test_a.py", False), |
| 149 | ("pytest -n 4 kernels/test_a.py", "kernels/test_a.py", True), |
| 150 | # Buildkite shard flags carry a $$VAR and must not swallow the path. |
| 151 | ( |
| 152 | ( |
| 153 | "pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB " |
| 154 | "--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT" |
| 155 | ), |
| 156 | "kernels/test_a.py", |
| 157 | True, |
| 158 | ), |
| 159 | # A leading env-var assignment is not a path. |
| 160 | ( |
| 161 | "VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s kernels", |
| 162 | "kernels/test_a.py", |
| 163 | True, |
| 164 | ), |
| 165 | # `python -m pytest` is the same invocation. |
| 166 | ("python3 -m pytest -v -s kernels/test_a.py", "kernels/test_a.py", True), |
| 167 | # Quoted paths, wrapper commands and redirects all survive tokenizing. |
| 168 | ("pytest -v -s 'kernels/test_a.py'", "kernels/test_a.py", True), |
| 169 | ("timeout 600 pytest kernels/test_a.py", "kernels/test_a.py", True), |
| 170 | ("pytest kernels/test_a.py > out.log 2>&1", "kernels/test_a.py", True), |
| 171 | # Each sub-command of a compound line is classified on its own. |
| 172 | ("pytest kernels/test_a.py && pytest lora/test_b.py", "lora/test_b.py", True), |
| 173 | ("pytest kernels/test_a.py ; pytest lora/test_b.py", "lora/test_b.py", True), |
| 174 | # An unresolvable $VAR path yields no coverage rather than a guess. |
| 175 | ("pytest -v -s $TEST_TARGET", "kernels/test_a.py", False), |
| 176 | ], |
| 177 | ) |
| 178 | def test_option_and_token_handling(command, test_file, expected): |
| 179 | assert runs(command, test_file) is expected |
| 180 | |
| 181 | |
| 182 | # --------------------------------------------------------------------------- # |
| 183 | # find | xargs pytest pipelines |
| 184 | # --------------------------------------------------------------------------- # |
| 185 | |
| 186 | |
| 187 | @pytest.mark.parametrize( |
| 188 | ("command", "test_file", "expected"), |
| 189 | [ |
| 190 | ("find kernels -name 'test_*.py' | xargs pytest -v", "kernels/test_a.py", True), |
| 191 | ( |
| 192 | "find kernels -name 'test_*.py' | xargs pytest -v", |
| 193 | "kernels/moe/test_b.py", |
| 194 | True, |
| 195 | ), |
| 196 | # A non-matching filename is not collected by the find expression. |
| 197 | ( |
| 198 | "find kernels -name 'test_*.py' | xargs pytest -v", |
| 199 | "kernels/helper.py", |
| 200 | False, |
| 201 | ), |
| 202 | # -maxdepth 1 keeps direct children and drops nested ones. |
| 203 | ( |
| 204 | "find kernels -maxdepth 1 -name 'test_*.py' | xargs pytest", |
| 205 | "kernels/test_a.py", |
| 206 | True, |
| 207 | ), |
| 208 | ( |
| 209 | "find kernels -maxdepth 1 -name 'test_*.py' | xargs pytest", |
| 210 | "kernels/moe/test_b.py", |
| 211 | False, |
| 212 | ), |
| 213 | # -not -name excludes its target and leaves siblings alone. |
| 214 | ( |
| 215 | "find kernels -name 'test_*.py' -not -name 'test_skip.py' | xargs pytest", |
| 216 | "kernels/test_skip.py", |
| 217 | False, |
| 218 | ), |
| 219 | ( |
| 220 | "find kernels -name 'test_*.py' -not -name 'test_skip.py' | xargs pytest", |
| 221 | "kernels/test_a.py", |
| 222 | True, |
| 223 | ), |
| 224 | # The root confines the sweep. |
| 225 | ("find kernels -name 'test_*.py' | xargs pytest", "lora/test_a.py", False), |
| 226 | # -maxdepth is honored even when the find root is the tests/ root itself |
| 227 | # ("." or "tests/"), not just a subdir. |
| 228 | ( |
| 229 | "find . -maxdepth 1 -name 'test_*.py' | xargs pytest", |
| 230 | "test_regression.py", |
| 231 | True, |
| 232 | ), |
| 233 | ( |
| 234 | "find . -maxdepth 1 -name 'test_*.py' | xargs pytest", |
| 235 | "v1/test_scheduler.py", |
| 236 | False, |
| 237 | ), |
| 238 | ( |
| 239 | "find tests/ -maxdepth 1 -name 'test_*.py' | xargs pytest", |
| 240 | "v1/test_scheduler.py", |
| 241 | False, |
| 242 | ), |
| 243 | # -exec pytest wires the sweep just as `| xargs pytest` does. |
| 244 | ( |
| 245 | "find compile -name 'test_*.py' -exec pytest -s -v {} \\;", |
| 246 | "compile/test_a.py", |
| 247 | True, |
| 248 | ), |
| 249 | # A pytest in a *separate* command on the line (not fed by the find) does |
| 250 | # not arm the sweep - the find here is piped to `wc`. |
| 251 | ( |
| 252 | "find kernels -name 'test_*.py' | wc -l ; pytest test_regression.py", |
| 253 | "kernels/test_a.py", |
| 254 | False, |
| 255 | ), |
| 256 | ( |
| 257 | "find kernels -name 'test_*.py' -delete && pytest test_regression.py", |
| 258 | "kernels/test_a.py", |
| 259 | False, |
| 260 | ), |
| 261 | # ...but only a *non-filtering* intermediate stage. `sort` reorders, |
| 262 | # every match still reaches pytest. |
| 263 | ( |
| 264 | "find kernels -name 'test_*.py' | sort | xargs pytest", |
| 265 | "kernels/test_a.py", |
| 266 | True, |
| 267 | ), |
| 268 | # `head` drops all but one match - the sweep is not fully tethered. |
| 269 | ( |
| 270 | "find kernels -name 'test_*.py' | head -n 1 | xargs pytest", |
| 271 | "kernels/test_a.py", |
| 272 | False, |
| 273 | ), |
| 274 | ( |
| 275 | "find kernels -name 'test_*.py' | grep attention | xargs pytest", |
| 276 | "kernels/test_a.py", |
| 277 | False, |
| 278 | ), |
| 279 | # `xargs` runs `echo` here, not pytest - the trailing `pytest` is echo's |
| 280 | # argument. |
| 281 | ( |
| 282 | "find kernels -name 'test_*.py' | xargs echo pytest", |
| 283 | "kernels/test_a.py", |
| 284 | False, |
| 285 | ), |
| 286 | # An `-exec` with no `\;` / `+` terminator is a `find` syntax error - it |
| 287 | # never runs pytest. |
| 288 | ( |
| 289 | "find kernels -name 'test_*.py' -exec pytest {}", |
| 290 | "kernels/test_a.py", |
| 291 | False, |
| 292 | ), |
| 293 | # An `-exec pytest` with no `{}` runs only the named file for every |
| 294 | # match - the matched files never reach pytest. |
| 295 | ( |
| 296 | "find kernels -name 'test_*.py' -exec pytest test_regression.py \\;", |
| 297 | "kernels/test_a.py", |
| 298 | False, |
| 299 | ), |
| 300 | # `{}` as an option *value* (`--ignore {}`) doesn't reach pytest as a |
| 301 | # test path either - only `test_regression.py` runs. |
| 302 | ( |
| 303 | ( |
| 304 | "find kernels -name 'test_*.py' -exec pytest test_regression.py " |
| 305 | "--ignore {} \\;" |
| 306 | ), |
| 307 | "kernels/test_a.py", |
| 308 | False, |
| 309 | ), |
| 310 | # xargs value-flags with a space don't swallow the command. |
| 311 | ( |
| 312 | "find kernels -name 'test_*.py' | xargs -0 -I {} pytest {}", |
| 313 | "kernels/test_a.py", |
| 314 | True, |
| 315 | ), |
| 316 | # `xargs env FOO=1 pytest` - xargs runs `env`, which runs pytest. |
| 317 | ( |
| 318 | "find kernels -name 'test_*.py' | xargs env FOO=1 pytest", |
| 319 | "kernels/test_a.py", |
| 320 | True, |
| 321 | ), |
| 322 | ], |
| 323 | ) |
| 324 | def test_find_pipelines(command, test_file, expected): |
| 325 | assert runs(command, test_file) is expected |
| 326 | |
| 327 | |
| 328 | def test_find_pipeline_parses_as_find_selection(): |
| 329 | (selection,) = [ |
| 330 | s |
| 331 | for s in _parse_command("find kernels -name 'test_*.py' | xargs pytest -v") |
| 332 | if isinstance(s, FindSelection) |
| 333 | ] |
| 334 | assert selection.root == "kernels" |
| 335 | assert selection.name_globs == ["test_*.py"] |
| 336 | |
| 337 | |
| 338 | # --------------------------------------------------------------------------- # |
| 339 | # Direct file runners |
| 340 | # --------------------------------------------------------------------------- # |
| 341 | |
| 342 | |
| 343 | @pytest.mark.parametrize( |
| 344 | ("command", "test_file", "expected"), |
| 345 | [ |
| 346 | ( |
| 347 | "python3 standalone_tests/lazy_imports.py", |
| 348 | "standalone_tests/lazy_imports.py", |
| 349 | True, |
| 350 | ), |
| 351 | ( |
| 352 | "python standalone_tests/lazy_imports.py", |
| 353 | "standalone_tests/lazy_imports.py", |
| 354 | True, |
| 355 | ), |
| 356 | # `timeout -k <grace> <dur>` wrapping a runner: the grace value must not |
| 357 | # be consumed as the duration (which would hide `python`). |
| 358 | ( |
| 359 | "timeout -k 5s 30s python standalone_tests/lazy_imports.py", |
| 360 | "standalone_tests/lazy_imports.py", |
| 361 | True, |
| 362 | ), |
| 363 | ( |
| 364 | "torchrun --nproc-per-node 2 distributed/test_a.py", |
| 365 | "distributed/test_a.py", |
| 366 | True, |
| 367 | ), |
| 368 | # The option's value is a count, not a path. |
| 369 | ("torchrun --nproc-per-node 2 distributed/test_a.py", "2", False), |
| 370 | ( |
| 371 | "VLLM_X=1 python3 standalone_tests/lazy_imports.py | grep -q ok", |
| 372 | "standalone_tests/lazy_imports.py", |
| 373 | True, |
| 374 | ), |
| 375 | # Only the executed script counts; the tokens after it are that script's |
| 376 | # own argv. `--suffix v1` must not tether the whole tests/v1 subtree. |
| 377 | ( |
| 378 | ( |
| 379 | "python3 features/tensorize_vllm_model.py --model facebook/opt-125m " |
| 380 | "serialize --serialized-directory /tmp/ --suffix v1" |
| 381 | ), |
| 382 | "v1/test_scheduler.py", |
| 383 | False, |
| 384 | ), |
| 385 | ( |
| 386 | ( |
| 387 | "python3 features/tensorize_vllm_model.py --model facebook/opt-125m " |
| 388 | "serialize --serialized-directory /tmp/ --suffix v1" |
| 389 | ), |
| 390 | "features/tensorize_vllm_model.py", |
| 391 | True, |
| 392 | ), |
| 393 | # A script named by an absolute `$VAR` path still resolves. |
| 394 | ( |
| 395 | 'python3 -m pytest -s -x "${GIT_ROOT}/tests/v1/e2e/test_x.py"', |
| 396 | "v1/e2e/test_x.py", |
| 397 | True, |
| 398 | ), |
| 399 | # `python -m <module>` runs the module, not a script - only `-m pytest` |
| 400 | # collects tests. `compileall kernels` byte-compiles, it does not run. |
| 401 | ("python -m compileall kernels", "kernels/test_a.py", False), |
| 402 | ("python -m mypy kernels/test_a.py", "kernels/test_a.py", False), |
| 403 | # `coverage run -m pytest` still resolves through to pytest. |
| 404 | ("coverage run -m pytest kernels/test_a.py", "kernels/test_a.py", True), |
| 405 | # `python -c '<code>' file.py` runs the code string; `file.py` is argv. |
| 406 | ("python -c 'pass' kernels/test_a.py", "kernels/test_a.py", False), |
| 407 | ("python -c 'import sys' kernels/test_a.py", "kernels/test_a.py", False), |
| 408 | # a `-m pytest` *after* the `-c` code operand is that program's argv too. |
| 409 | ("python -c 'pass' -m pytest kernels/test_a.py", "kernels/test_a.py", False), |
| 410 | # ...but `-c` as the *script's* own arg still tethers the script. |
| 411 | ("python kernels/test_a.py -c config.py", "kernels/test_a.py", True), |
| 412 | # ...and pytest's own `-c <cfg>` (after `-m pytest`) is fine. |
| 413 | ("python -m pytest kernels/test_a.py -c setup.cfg", "kernels/test_a.py", True), |
| 414 | ], |
| 415 | ) |
| 416 | def test_direct_runners(command, test_file, expected): |
| 417 | assert runs(command, test_file) is expected |
| 418 | |
| 419 | |
| 420 | # --------------------------------------------------------------------------- # |
| 421 | # Commands that must contribute no coverage |
| 422 | # --------------------------------------------------------------------------- # |
| 423 | |
| 424 | |
| 425 | @pytest.mark.parametrize( |
| 426 | "command", |
| 427 | [ |
| 428 | "export VLLM_WORKER_MULTIPROC_METHOD=spawn", |
| 429 | "pip install pytest-timeout pytest-forked", |
| 430 | "uv pip install -r requirements/test/cuda.in", |
| 431 | "bash tests/does_not_exist.sh", |
| 432 | "echo 'pytest kernels/test_a.py'", |
| 433 | # `pytest` as an argument to another command - `echo` runs, not pytest. |
| 434 | "echo pytest kernels/test_a.py", |
| 435 | 'pytest -v -s "unbalanced', |
| 436 | # A `.sh` that is only an *argument* (Buildkite printing / copying a |
| 437 | # filename) is not a script invocation. |
| 438 | "echo tests/foo.sh", |
| 439 | "cp scripts/run_tests.sh /tmp/", |
| 440 | # A `find` whose output never reaches pytest is not a test selection. |
| 441 | "find kernels -name 'test_*.py'", |
| 442 | "find kernels -name 'test_*.py' | wc -l", |
| 443 | "find kernels -name 'test_*.py' -delete", |
| 444 | "find kernels -name 'test_*.py' | xargs echo pytest", |
| 445 | "find kernels -name 'test_*.py' -exec pytest {}", |
| 446 | ], |
| 447 | ) |
| 448 | def test_non_test_commands_contribute_nothing(command): |
| 449 | assert _parse_command(command) == [] or not any( |
| 450 | sel.runs("kernels/test_a.py") for sel in _parse_command(command) |
| 451 | ) |
| 452 | |
| 453 | |
| 454 | # --------------------------------------------------------------------------- # |
| 455 | # Job-level yaml parsing |
| 456 | # --------------------------------------------------------------------------- # |
| 457 | |
| 458 | |
| 459 | def _write_area(tmp_path, monkeypatch, body: str): |
| 460 | mod = checker |
| 461 | area_dir = tmp_path / "test_areas" |
| 462 | area_dir.mkdir(exist_ok=True) |
| 463 | (area_dir / "synthetic.yaml").write_text(textwrap.dedent(body)) |
| 464 | monkeypatch.setattr(mod, "TEST_AREAS_DIR", area_dir) |
| 465 | # Keep the real test-amd.yaml out of these isolated parsing tests. |
| 466 | monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "no-such-test-amd.yaml") |
| 467 | # The yaml-error path reports a repo-relative name, so REPO_ROOT has to |
| 468 | # contain the synthetic area too. |
| 469 | monkeypatch.setattr(mod, "REPO_ROOT", tmp_path) |
| 470 | return load_selections() |
| 471 | |
| 472 | |
| 473 | def test_plain_step_commands_are_collected(tmp_path, monkeypatch): |
| 474 | selections = _write_area( |
| 475 | tmp_path, |
| 476 | monkeypatch, |
| 477 | """ |
| 478 | steps: |
| 479 | - label: ":nvidia: (H200) Kernels" |
| 480 | commands: |
| 481 | - pytest -v -s kernels |
| 482 | """, |
| 483 | ) |
| 484 | assert is_tethered("kernels/test_a.py", selections) |
| 485 | |
| 486 | |
| 487 | def test_singular_command_key_is_collected(tmp_path, monkeypatch): |
| 488 | selections = _write_area( |
| 489 | tmp_path, |
| 490 | monkeypatch, |
| 491 | """ |
| 492 | steps: |
| 493 | - label: "Singular" |
| 494 | command: pytest -v -s kernels/test_a.py |
| 495 | """, |
| 496 | ) |
| 497 | assert is_tethered("kernels/test_a.py", selections) |
| 498 | |
| 499 | |
| 500 | def test_test_amd_yaml_coverage_counts_as_tethered(tmp_path, monkeypatch): |
| 501 | """A test wired only into the legacy test-amd.yaml is still run by CI.""" |
| 502 | mod = checker |
| 503 | (tmp_path / "test_areas").mkdir() |
| 504 | amd_yaml = tmp_path / "test-amd.yaml" |
| 505 | amd_yaml.write_text( |
| 506 | textwrap.dedent( |
| 507 | """ |
| 508 | steps: |
| 509 | - label: ":amd: (MI300) Quantization" |
| 510 | commands: |
| 511 | - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ |
| 512 | - pytest -v -s rocm/test_moe_weight_replay.py |
| 513 | """ |
| 514 | ) |
| 515 | ) |
| 516 | monkeypatch.setattr(mod, "TEST_AREAS_DIR", tmp_path / "test_areas") |
| 517 | monkeypatch.setattr(mod, "TEST_AMD_YAML", amd_yaml) |
| 518 | monkeypatch.setattr(mod, "REPO_ROOT", tmp_path) |
| 519 | selections = load_selections() |
| 520 | assert is_tethered("rocm/test_moe_weight_replay.py", selections) |
| 521 | assert is_tethered("quantization/test_foo.py", selections) |
| 522 | |
| 523 | |
| 524 | def test_missing_test_amd_yaml_is_not_fatal(tmp_path, monkeypatch): |
| 525 | """When the migration finally deletes test-amd.yaml, the checker still runs.""" |
| 526 | mod = checker |
| 527 | (tmp_path / "test_areas").mkdir() |
| 528 | monkeypatch.setattr(mod, "TEST_AREAS_DIR", tmp_path / "test_areas") |
| 529 | monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "gone.yaml") |
| 530 | monkeypatch.setattr(mod, "REPO_ROOT", tmp_path) |
| 531 | assert load_selections() == [] |
| 532 | |
| 533 | |
| 534 | def test_job_dirs_come_from_ci_config(tmp_path, monkeypatch): |
| 535 | """The pipeline directories are whatever `.buildkite/ci_config*.yaml` lists |
| 536 | under `job_dirs:` - so a test wired only into e.g. `hardware_tests/` counts, |
| 537 | and an unreadable config falls back to `test_areas/` alone.""" |
| 538 | mod = checker |
| 539 | bk = tmp_path / ".buildkite" |
| 540 | (bk / "test_areas").mkdir(parents=True) |
| 541 | (bk / "hardware_tests").mkdir() |
| 542 | (bk / "ci_config.yaml").write_text("job_dirs:\n - .buildkite/hardware_tests\n") |
| 543 | (bk / "ci_config_broken.yaml").write_text("job_dirs: [oops\n") # ignored |
| 544 | (bk / "hardware_tests" / "cpu.yaml").write_text( |
| 545 | textwrap.dedent( |
| 546 | """ |
| 547 | steps: |
| 548 | - label: "CPU" |
| 549 | commands: |
| 550 | - pytest -v -s v1/e2e/test_cpu_thing.py |
| 551 | """ |
| 552 | ) |
| 553 | ) |
| 554 | monkeypatch.setattr(mod, "REPO_ROOT", tmp_path) |
| 555 | monkeypatch.setattr(mod, "TEST_AREAS_DIR", bk / "test_areas") |
| 556 | monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "no-amd.yaml") |
| 557 | |
| 558 | assert (bk / "hardware_tests") in checker._pipeline_job_dirs() |
| 559 | assert is_tethered("v1/e2e/test_cpu_thing.py", load_selections()) |
| 560 | |
| 561 | |
| 562 | def test_step_level_ignore_is_honored(tmp_path, monkeypatch): |
| 563 | selections = _write_area( |
| 564 | tmp_path, |
| 565 | monkeypatch, |
| 566 | """ |
| 567 | steps: |
| 568 | - label: "Kernels" |
| 569 | commands: |
| 570 | - pytest -v -s kernels --ignore=kernels/test_a.py |
| 571 | """, |
| 572 | ) |
| 573 | assert not is_tethered("kernels/test_a.py", selections) |
| 574 | assert is_tethered("kernels/test_b.py", selections) |
| 575 | |
| 576 | |
| 577 | def test_amd_mirror_only_command_is_collected(tmp_path, monkeypatch): |
| 578 | """A mirror that overrides `commands` runs tests the base step never does.""" |
| 579 | selections = _write_area( |
| 580 | tmp_path, |
| 581 | monkeypatch, |
| 582 | """ |
| 583 | steps: |
| 584 | - label: ":nvidia: (H200) Base" |
| 585 | commands: |
| 586 | - pytest -v -s kernels/test_base.py |
| 587 | mirror: |
| 588 | amd: |
| 589 | label: ":amd: (MI300) Base" |
| 590 | commands: |
| 591 | - pytest -v -s kernels/test_amd_only.py |
| 592 | """, |
| 593 | ) |
| 594 | assert is_tethered("kernels/test_base.py", selections) |
| 595 | assert is_tethered("kernels/test_amd_only.py", selections) |
| 596 | |
| 597 | |
| 598 | def test_shell_script_continuation_vars_and_nesting(tmp_path, monkeypatch): |
| 599 | """The v1 P/D integration scripts name their test through a `\\`-continued |
| 600 | line, a literal shell var, and a nested `bash "$SCRIPT"`. Each indirection |
| 601 | has to be followed or the check falsely reports the test untethered.""" |
| 602 | scripts = tmp_path / "tests" / "v1" / "kv_connector" / "nixl_integration" |
| 603 | scripts.mkdir(parents=True) |
| 604 | (scripts / "inner.sh").write_text( |
| 605 | "python3 -m pytest -s -v \\\n" |
| 606 | ' "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_inner.py"\n' |
| 607 | ) |
| 608 | (scripts / "sweep.sh").write_text( |
| 609 | 'SCRIPT="v1/kv_connector/nixl_integration/inner.sh"\n' |
| 610 | 'IMPORT_CANARY="v1/kv_connector/nixl_integration/test_canary.py"\n' |
| 611 | 'LITERAL="v1/kv_connector/nixl_integration/test_not_run.py"\n' |
| 612 | 'python3 -m pytest -s -x "${IMPORT_CANARY}"\n' |
| 613 | # single-quoted: the shell passes a literal $LITERAL, collecting nothing |
| 614 | "python3 -m pytest -s -x '${LITERAL}'\n" |
| 615 | 'if ! env FOO=1 bash "${SCRIPT}"; then exit 1; fi\n' |
| 616 | ) |
| 617 | monkeypatch.setattr(checker, "REPO_ROOT", tmp_path) |
| 618 | monkeypatch.setattr(checker, "TESTS_DIR", tmp_path / "tests") |
| 619 | |
| 620 | selections = _parse_command("bash v1/kv_connector/nixl_integration/sweep.sh") |
| 621 | |
| 622 | def tethered(path): |
| 623 | return any(s.runs(path) for s in selections) |
| 624 | |
| 625 | # `\`-continuation + $VAR-prefixed absolute path, resolved through a nested |
| 626 | # `env FOO=1 bash "$SCRIPT"`: |
| 627 | assert tethered("v1/kv_connector/nixl_integration/test_inner.py") |
| 628 | # literal `VAR=path` + double-quoted `"$VAR"` (the import-canary pattern): |
| 629 | assert tethered("v1/kv_connector/nixl_integration/test_canary.py") |
| 630 | # single-quoted `'$VAR'` is NOT expanded by the shell, so not tethered: |
| 631 | assert not tethered("v1/kv_connector/nixl_integration/test_not_run.py") |
| 632 | |
| 633 | |
| 634 | def test_bash_dash_c_does_not_follow_a_script_operand(tmp_path, monkeypatch): |
| 635 | """`bash -c '<code>' foo.sh` runs the `-c` string; `foo.sh` is `$0` and is |
| 636 | never executed, so its contents must not count as coverage.""" |
| 637 | (tmp_path / "tests").mkdir() |
| 638 | (tmp_path / "tests" / "decoy.sh").write_text("pytest -v -s lora/test_decoy.py\n") |
| 639 | monkeypatch.setattr(checker, "REPO_ROOT", tmp_path) |
| 640 | monkeypatch.setattr(checker, "TESTS_DIR", tmp_path / "tests") |
| 641 | |
| 642 | with_c = _parse_command("bash -c 'echo hi' tests/decoy.sh") |
| 643 | assert not any(s.runs("lora/test_decoy.py") for s in with_c) |
| 644 | # ...but a plain `bash tests/decoy.sh` still follows it. |
| 645 | plain = _parse_command("bash tests/decoy.sh") |
| 646 | assert any(s.runs("lora/test_decoy.py") for s in plain) |
| 647 | # ...and a `-c` that is the *script's* own flag (not bash's) is not `bash -c`. |
| 648 | script_flag = _parse_command("bash tests/decoy.sh -c models.txt") |
| 649 | assert any(s.runs("lora/test_decoy.py") for s in script_flag) |
| 650 | # A `bash -c` trailing arg that looks like a command string is `$0`, not run. |
| 651 | cmd_string_arg = _parse_command("bash -c 'echo ok' 'pytest lora/test_x.py'") |
| 652 | assert not any(s.runs("lora/test_x.py") for s in cmd_string_arg) |
| 653 | # ...but the `-c` code string *itself* is parsed - it is what runs. |
| 654 | code_runs = _parse_command("bash -c 'pytest lora/test_x.py'") |
| 655 | assert any(s.runs("lora/test_x.py") for s in code_runs) |
| 656 | # a combined short-option group resolves the `-c` operand with `c` in any |
| 657 | # position (`-ec`, `-ce`), and its trailing args are still `$0`, not run. |
| 658 | for grp in ("-ec", "-ce", "-euxc"): |
| 659 | assert any( |
| 660 | s.runs("lora/test_x.py") |
| 661 | for s in _parse_command(f"bash {grp} 'pytest lora/test_x.py'") |
| 662 | ) |
| 663 | assert not any( |
| 664 | s.runs("lora/test_x.py") |
| 665 | for s in _parse_command("bash -ce 'echo ok' 'pytest lora/test_x.py'") |
| 666 | ) |
| 667 | |
| 668 | |
| 669 | def test_unparsable_yaml_is_fatal(tmp_path, monkeypatch): |
| 670 | """Silently skipping a bad yaml would drop its coverage and report false |
| 671 | untethered files, so it must fail loudly instead.""" |
| 672 | with pytest.raises(SystemExit): |
| 673 | _write_area(tmp_path, monkeypatch, "steps: [oops\n") |
| 674 | |
| 675 | |
| 676 | def test_non_mapping_yaml_is_fatal(tmp_path, monkeypatch): |
| 677 | """A yaml that parses but isn't a pipeline mapping (a bare list or scalar) |
| 678 | must raise the actionable error, not crash on ``.get``.""" |
| 679 | with pytest.raises(SystemExit): |
| 680 | _write_area(tmp_path, monkeypatch, "- just\n- a\n- list\n") |
| 681 | |
| 682 | |
| 683 | # --------------------------------------------------------------------------- # |
| 684 | # is_test_module / path normalization |
| 685 | # --------------------------------------------------------------------------- # |
| 686 | |
| 687 | |
| 688 | @pytest.mark.parametrize( |
| 689 | ("path", "expected"), |
| 690 | [ |
| 691 | ("tests/kernels/test_a.py", True), |
| 692 | ("tests/kernels/a_test.py", True), |
| 693 | ("tests/kernels/conftest.py", False), |
| 694 | ("tests/kernels/__init__.py", False), |
| 695 | ("tests/kernels/utils.py", False), |
| 696 | ("tests/kernels/test_a.txt", False), |
| 697 | ("vllm/kernels/test_a.py", False), |
| 698 | ], |
| 699 | ) |
| 700 | def test_is_test_module(path, expected): |
| 701 | assert is_test_module(path) is expected |
| 702 | |
| 703 | |
| 704 | @pytest.mark.parametrize( |
| 705 | ("raw", "expected"), |
| 706 | [ |
| 707 | ("tests/kernels/test_a.py", "kernels/test_a.py"), |
| 708 | ("./kernels/test_a.py", "kernels/test_a.py"), |
| 709 | ("/vllm-workspace/tests/kernels/test_a.py", "kernels/test_a.py"), |
| 710 | ("'kernels/test_a.py'", "kernels/test_a.py"), |
| 711 | ("kernels/test_a.py::test_one", "kernels/test_a.py"), |
| 712 | ("kernels/", "kernels"), |
| 713 | ("${GIT_ROOT}/tests/kernels/test_a.py", "kernels/test_a.py"), |
| 714 | ("$GIT_ROOT/tests/kernels/test_a.py", "kernels/test_a.py"), |
| 715 | ], |
| 716 | ) |
| 717 | def test_normalize_test_path(raw, expected): |
| 718 | assert normalize_test_path(raw) == expected |
| 719 | |
| 720 | |
| 721 | # --------------------------------------------------------------------------- # |
| 722 | # Whole-tree invariants |
| 723 | # --------------------------------------------------------------------------- # |
| 724 | |
| 725 | |
| 726 | def test_real_tree_parses_into_selections(): |
| 727 | assert len(load_selections()) > 200 |
| 728 | assert len(all_test_modules()) > 1000 |
| 729 | |
| 730 | |
| 731 | def test_no_selection_matches_everything(): |
| 732 | """A path arg that normalizes to '' or '.' would tether every test in the |
| 733 | repo and silently disarm the gate - the worst possible misparse.""" |
| 734 | for selection in load_selections(): |
| 735 | if isinstance(selection, PytestSelection): |
| 736 | offenders = [ |
| 737 | p |
| 738 | for p in selection.included_paths |
| 739 | if normalize_test_path(p) in ("", ".") |
| 740 | ] |
| 741 | assert not offenders, f"match-everything pytest arg: {offenders}" |
| 742 | else: |
| 743 | assert normalize_test_path(selection.root) not in ("", ".") |
| 744 | |
| 745 | |
| 746 | def test_tethering_is_deterministic(): |
| 747 | selections = load_selections() |
| 748 | modules = all_test_modules()[:200] |
| 749 | first = [is_tethered(m, selections) for m in modules] |
| 750 | assert first == [is_tethered(m, load_selections()) for m in modules] |
| 751 | |
| 752 | |
| 753 | def test_allowlist_entries_are_well_formed(): |
| 754 | allowlist = load_allowlist() |
| 755 | assert allowlist, "allowlist should not be empty while gaps remain" |
| 756 | assert all(is_test_module(path) for path in allowlist) |
| 757 | assert all(not path.startswith("/") for path in allowlist) |
| 758 | |
| 759 | |
| 760 | def test_real_allowlist_entries_all_have_a_reason(): |
| 761 | assert allowlist_entries_missing_reason() == [] |
| 762 | |
| 763 | |
| 764 | def test_bare_allowlist_entry_is_flagged(tmp_path, monkeypatch): |
| 765 | allowlist = tmp_path / "allowlist.txt" |
| 766 | allowlist.write_text( |
| 767 | "tests/a/test_ok.py # torchrun multi-GPU, no job\n" |
| 768 | "tests/b/test_bare.py\n" |
| 769 | " # a standalone comment line is fine\n" |
| 770 | ) |
| 771 | monkeypatch.setattr(checker, "ALLOWLIST_PATH", allowlist) |
| 772 | assert allowlist_entries_missing_reason() == ["tests/b/test_bare.py"] |
| 773 | |
| 774 | |
| 775 | def test_real_tree_has_no_unallowlisted_gaps(): |
| 776 | """Every test module is either collected by a job or allowlisted. |
| 777 | |
| 778 | Deliberately non-strict: a stale allowlist entry is advisory by design, so |
| 779 | asserting `strict=True` here would make unrelated allowlist cleanup block |
| 780 | CI - the exact behaviour the two-mode design avoids. |
| 781 | """ |
| 782 | assert run_full_scan(load_selections(), load_allowlist(), strict=False) == 0 |
| 783 | |
| 784 | |
| 785 | # --------------------------------------------------------------------------- # |
| 786 | # CLI behaviour |
| 787 | # --------------------------------------------------------------------------- # |
| 788 | |
| 789 | |
| 790 | @pytest.fixture |
| 791 | def fake_repo(tmp_path, monkeypatch): |
| 792 | """Point the module at a throwaway tree so the changed-files path can be |
| 793 | exercised without writing test files into the real repo.""" |
| 794 | mod = checker |
| 795 | (tmp_path / "tests" / "kernels").mkdir(parents=True) |
| 796 | (tmp_path / "tests" / "kernels" / "test_new.py").write_text("def test_x(): pass\n") |
| 797 | monkeypatch.setattr(mod, "REPO_ROOT", tmp_path) |
| 798 | monkeypatch.setattr(mod, "TESTS_DIR", tmp_path / "tests") |
| 799 | monkeypatch.setattr(mod, "ALLOWLIST_PATH", tmp_path / "allowlist.txt") |
| 800 | # The throwaway tree is not a git checkout, so the tracked-file inventory |
| 801 | # has to be stubbed rather than shelled out. |
| 802 | monkeypatch.setattr(mod, "all_test_modules", lambda: ["tests/kernels/test_new.py"]) |
| 803 | return tmp_path |
| 804 | |
| 805 | |
| 806 | def test_untethered_changed_file_fails(fake_repo, capsys): |
| 807 | rc = run_changed_files_check(["tests/kernels/test_new.py"], [], set()) |
| 808 | assert rc == 1 |
| 809 | assert "is not run by any Buildkite job" in capsys.readouterr().out |
| 810 | |
| 811 | |
| 812 | def test_tethered_changed_file_passes(fake_repo): |
| 813 | selections = _parse_command("pytest -v -s kernels") |
| 814 | assert ( |
| 815 | run_changed_files_check(["tests/kernels/test_new.py"], selections, set()) == 0 |
| 816 | ) |
| 817 | |
| 818 | |
| 819 | def test_allowlisted_changed_file_passes(fake_repo): |
| 820 | allowlist = {"tests/kernels/test_new.py"} |
| 821 | assert run_changed_files_check(["tests/kernels/test_new.py"], [], allowlist) == 0 |
| 822 | |
| 823 | |
| 824 | def test_non_test_and_missing_paths_are_ignored(fake_repo): |
| 825 | paths = ["tests/kernels/conftest.py", "vllm/config.py", "tests/kernels/gone.py"] |
| 826 | assert run_changed_files_check(paths, [], set()) == 0 |
| 827 | |
| 828 | |
| 829 | def test_stale_allowlist_entries_are_advisory_but_reported(fake_repo, capsys): |
| 830 | """Dangling entries must not block an unrelated PR, but they have to be |
| 831 | printed - the hook sets `verbose: true` so a passing run still shows them.""" |
| 832 | allowlist = {"tests/kernels/gone.py"} |
| 833 | rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=False) |
| 834 | out = capsys.readouterr().out |
| 835 | assert rc == 0 |
| 836 | assert "no longer exists" in out |
| 837 | |
| 838 | |
| 839 | def test_stale_allowlist_entries_are_fatal_under_all(fake_repo, capsys): |
| 840 | allowlist = {"tests/kernels/gone.py"} |
| 841 | rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=True) |
| 842 | assert rc == 1 |
| 843 | assert "no longer exists" in capsys.readouterr().out |
| 844 | |
| 845 | |
| 846 | def test_now_tethered_allowlist_entry_is_reported(fake_repo, capsys): |
| 847 | allowlist = {"tests/kernels/test_new.py"} |
| 848 | rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=True) |
| 849 | assert rc == 1 |
| 850 | assert "is now tethered" in capsys.readouterr().out |
| 851 | |
| 852 | |
| 853 | def test_main_passes_a_tethered_file(monkeypatch): |
| 854 | """The CLI's normal pre-commit path: a tethered file exits 0. |
| 855 | |
| 856 | `--all` is not asserted here on purpose - it implies `strict=True`, so a |
| 857 | stale allowlist entry would turn unrelated cleanup into a CI failure. |
| 858 | """ |
| 859 | monkeypatch.setattr( |
| 860 | "sys.argv", |
| 861 | ["check_test_tethering.py", "tests/tools/test_check_test_tethering.py"], |
| 862 | ) |
| 863 | assert main() == 0 |
| 864 | |
| 865 | |
| 866 | def test_main_flags_an_untethered_file(monkeypatch): |
| 867 | # Must be named like a test module, or the check correctly skips it. |
| 868 | orphan = REPO_ROOT / "tests" / "test_tethering_probe_orphan.py" |
| 869 | orphan.write_text("def test_x(): pass\n") |
| 870 | monkeypatch.setattr("sys.argv", ["check_test_tethering.py", str(orphan)]) |
| 871 | try: |
| 872 | assert main() == 1 |
| 873 | finally: |
| 874 | orphan.unlink() |
| 875 | |
| 876 | |
| 877 | def test_to_repo_relative_accepts_absolute_paths(): |
| 878 | absolute = str(REPO_ROOT / "tests" / "kernels" / "test_a.py") |
| 879 | assert _to_repo_relative(absolute) == "tests/kernels/test_a.py" |