[CI] Wire untethered test files into CI jobs (#49340)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Nick Hill committed
Jul 28, 2026 at 07:02 UTC
94100b5915d449aed0685cc5c6fc8949fcf5fe40
9 files changed
+88
-13
.buildkite/test_areas/cuda.yaml
+1
@@ -16,6 +16,7 @@ steps:
16
commands:
17
- pytest -v -s cuda/test_cuda_context.py
18
- pytest -v -s cuda/test_platform_no_cuda_init.py
19
+ - pytest -v -s cuda/test_cuda_compatibility_path.py
20
21
- label: Cudagraph
22
device: h200_35gb
.buildkite/test_areas/disaggregated.yaml
+16
@@ -131,6 +131,22 @@ steps:
131
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
132
- HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
133
134
+- label: NixlConnector PD edge case test (2 GPUs)
135
+ key: nixlconnector-pd-edge-cases-2-gpus
136
+ timeout_in_minutes: 40
137
+ working_dir: "/vllm-workspace/tests"
138
+ num_devices: 2
139
+ source_file_dependencies:
140
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
141
+ - vllm/v1/core/sched/
142
+ - tests/v1/kv_connector/nixl_integration/
143
+ env:
144
+ PREFILL_GPU_ID: "0"
145
+ DECODE_GPU_ID: "1"
146
+ commands:
147
+ - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
148
+ - bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
149
+
150
- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs)
151
key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
152
timeout_in_minutes: 25
.buildkite/test_areas/engine.yaml
+2
@@ -40,9 +40,11 @@ steps:
40
source_file_dependencies:
41
- vllm/v1/engine/
42
- tests/v1/engine/
43
+ - tests/v1/test_tensor_ipc_queue.py
44
commands:
45
- pytest -v -s v1/engine/test_preprocess_error_handling.py
46
- pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
47
+ - pytest -v -s v1/test_tensor_ipc_queue.py
48
mirror:
49
amd:
50
device: mi250_1
.buildkite/test_areas/kernels.yaml
+36
@@ -61,9 +61,45 @@ steps:
61
source_file_dependencies:
62
- csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
63
- vllm/models/deepseek_v4/common/ops/
64
+ - vllm/models/deepseek_v4/nvidia/
65
- tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
66
+ - tests/models/test_deepseek_v4_mega_moe.py
67
commands:
68
- pytest -v -s kernels/test_fused_deepseek_v4_*.py
69
+ - pytest -v -s models/test_deepseek_v4_mega_moe.py
70
+
71
+# Catch-all for test files at the tests/kernels root. This job collects
72
+# the whole root so new files are wired by default.
73
+# Files with dedicated jobs elsewhere in this file are excluded via --ignore
74
+# (test_kda, test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in
75
+# their own jobs / Kernels (B200)).
76
+- label: Kernels Root Misc Test (B200)
77
+ key: kernels-root-misc-test-b200
78
+ timeout_in_minutes: 45
79
+ device: b200-k8s
80
+ source_file_dependencies:
81
+ - csrc/
82
+ - vllm/
83
+ - tests/kernels/
84
+ commands:
85
+ - pytest -v -s kernels/
86
+ --ignore=kernels/attention
87
+ --ignore=kernels/core
88
+ --ignore=kernels/helion
89
+ --ignore=kernels/ir
90
+ --ignore=kernels/mamba
91
+ --ignore=kernels/moe
92
+ --ignore=kernels/quantization
93
+ --ignore=kernels/test_concat_mla_q.py
94
+ --ignore=kernels/test_fused_qk_norm_rope_gate.py
95
+ --ignore=kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
96
+ --ignore=kernels/test_top_k_per_row.py
97
+ --ignore=kernels/test_kda.py
98
+ --ignore=kernels/test_bf16x3_router_gemm_cutedsl.py
99
+ --ignore=kernels/test_ll_bf16_gemm.py
100
+ --ignore=kernels/test_shuffle_rows.py
101
+ # BROKEN on main, pending kernel fixes (B200):
102
+ # test_shuffle_rows.py (1: test_shuffle_rows_edge_cases)
103
104
- label: Kernels Attention Test %N
105
key: kernels-attention-test
.buildkite/test_areas/misc.yaml
+8
-2
@@ -148,6 +148,7 @@ steps:
148
- pytest -v -s -m 'cpu_test' v1/core
149
- pytest -v -s v1/structured_output
150
- pytest -v -s v1/test_serial_utils.py
151
+ - pytest -v -s v1/test_kv_cache_spec_registry.py
152
- pytest -v -s v1/cudagraph/test_cudagraph_manager.py
153
- pytest -v -s -m 'cpu_test' v1/kv_connector/unit
154
- pytest -v -s -m 'cpu_test' v1/metrics
@@ -265,6 +266,7 @@ steps:
266
- vllm/utils/
267
- vllm/v1/
268
- tests/v1/tracing
269
+ - tests/tracing/
270
commands:
271
- "pip install \
272
'opentelemetry-sdk>=1.26.0' \
@@ -272,6 +274,7 @@ steps:
274
'opentelemetry-exporter-otlp>=1.26.0' \
275
'opentelemetry-semantic-conventions-ai>=0.4.1'"
276
- pytest -v -s v1/tracing
277
+ - pytest -v -s tracing
278
mirror:
279
amd:
280
dind: false
@@ -425,7 +428,7 @@ steps:
428
429
- label: Batch Invariance (B200)
430
key: batch-invariance-b200
428
- timeout_in_minutes: 35
431
+ timeout_in_minutes: 45
432
device: b200-k8s
433
source_file_dependencies:
434
- vllm/v1/attention
@@ -440,7 +443,10 @@ steps:
443
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
444
- pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
445
- pytest -v -s v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py
443
-
446
+ - pytest -v -s v1/determinism/test_matmul_batch_invariant.py
447
+ - pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
448
+ - pytest -v -s v1/determinism/test_online_batch_invariance.py
449
+
450
- label: Acceptance Length Test (Large Models) # optional
451
device: h200_35gb
452
key: acceptance-length-test-large-models
.buildkite/test_areas/models_basic.yaml
+2
-1
@@ -82,7 +82,8 @@ steps:
82
- vllm/
83
- tests/models/test_utils.py
84
- tests/models/test_vision.py
85
+ - tests/models/test_adapters.py
86
- tests/models/transformers/fusers/
87
device: cpu-small
88
commands:
88
- - pytest -v -s models/test_utils.py models/test_vision.py models/transformers/fusers/
89
+ - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/transformers/fusers/
.buildkite/test_areas/spec_decode.yaml
+2
@@ -90,8 +90,10 @@ steps:
90
- vllm/v1/spec_decode/
91
- vllm/v1/worker/gpu/spec_decode/
92
- tests/v1/e2e/spec_decode/
93
+ - tests/spec_decode/
94
commands:
95
- pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
96
+ - python3 spec_decode/test_custom_proposer.py
97
mirror:
98
amd:
99
dind: false
tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py
+15
-6
@@ -147,8 +147,11 @@ def test_dense_norm_rope(num_tokens, num_heads, num_kv_heads):
147
eps,
148
).view(num_tokens, kvsz)
149
150
- torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
151
- torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
150
+ # The fused kernel keeps an fp32 intermediate across norm->rope, while the
151
+ # reference materializes bf16 after the norm (the unfused boundary), so
152
+ # rounding-boundary elements can differ by ~1 bf16 ulp.
153
+ torch.testing.assert_close(q_out, q_ref, rtol=2e-2, atol=2e-2)
154
+ torch.testing.assert_close(k_out, k_ref, rtol=2e-2, atol=2e-2)
155
# V is untouched.
156
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
157
@@ -255,8 +258,11 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
258
ik_orig.view(num_tokens, 1, HEAD_DIM), ik_w, positions, cos_sin, eps
259
).view(num_tokens, HEAD_DIM)
260
258
- torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
259
- torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
261
+ # The fused kernel keeps an fp32 intermediate across norm->rope, while the
262
+ # reference materializes bf16 after the norm (the unfused boundary), so
263
+ # rounding-boundary elements can differ by ~1 bf16 ulp.
264
+ torch.testing.assert_close(q_out, q_ref, rtol=2e-2, atol=2e-2)
265
+ torch.testing.assert_close(k_out, k_ref, rtol=2e-2, atol=2e-2)
266
torch.testing.assert_close(index_q, iq_ref, rtol=1e-2, atol=1e-2)
267
torch.testing.assert_close(index_k, ik_ref, rtol=1e-2, atol=1e-2)
268
@@ -376,8 +382,11 @@ def test_sparse_skip_index_branch(num_tokens, block_size, kv_cache_dtype):
382
eps,
383
).view(num_tokens, kvsz)
384
379
- torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
380
- torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
385
+ # The fused kernel keeps an fp32 intermediate across norm->rope, while the
386
+ # reference materializes bf16 after the norm (the unfused boundary), so
387
+ # rounding-boundary elements can differ by ~1 bf16 ulp.
388
+ torch.testing.assert_close(q_out, q_ref, rtol=2e-2, atol=2e-2)
389
+ torch.testing.assert_close(k_out, k_ref, rtol=2e-2, atol=2e-2)
390
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
391
torch.testing.assert_close(index_q_out, index_q_in, rtol=0, atol=0)
392
torch.testing.assert_close(index_k_out, index_k_in, rtol=0, atol=0)
tests/v1/kv_connector/nixl_integration/run_edge_case_test.sh
+6
-4
@@ -3,8 +3,8 @@ set -xe
3
4
# Parse command line arguments
5
KV_BUFFER_DEVICE="cuda" # Default to cuda
6
-PREFILL_GPU_ID=4 # Default GPU IDs
7
-DECODE_GPU_ID=5
6
+PREFILL_GPU_ID="${PREFILL_GPU_ID:-4}" # Default GPU IDs
7
+DECODE_GPU_ID="${DECODE_GPU_ID:-5}"
8
while [[ $# -gt 0 ]]; do
9
case $1 in
10
--kv_buffer_device)
@@ -70,6 +70,7 @@ run_tests_for_model() {
70
--port $PREFILL_PORT \
71
--enforce-eager \
72
--gpu-memory-utilization 0.2 \
73
+ --max-model-len 8192 \
74
--kv-transfer-config '$KV_CONFIG'"
75
76
FULL_CMD="$BASE_CMD"
@@ -84,6 +85,7 @@ run_tests_for_model() {
85
--port $DECODE_PORT \
86
--enforce-eager \
87
--gpu-memory-utilization 0.2 \
88
+ --max-model-len 8192 \
89
--kv-transfer-config '$KV_CONFIG'"
90
91
FULL_CMD="$BASE_CMD"
@@ -98,7 +100,7 @@ run_tests_for_model() {
100
101
# Build the command for the proxy server with all the hosts and ports
102
PROXY_PORT=8192
101
- PROXY_CMD="python ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --port $PROXY_PORT"
103
+ PROXY_CMD="python3 ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --port $PROXY_PORT"
104
PROXY_CMD+=" --prefiller-ports ${PREFILL_PORT}"
105
PROXY_CMD+=" --decoder-ports ${DECODE_PORT}"
106
# Start the proxy server
@@ -110,7 +112,7 @@ run_tests_for_model() {
112
113
# Run lm eval for this model
114
echo "Running tests for $model_name"
113
- PREFILL_PORT=$PREFILL_PORT DECODE_PORT=$DECODE_PORT PROXY_PORT=$PROXY_PORT python -m pytest -s -v "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
115
+ PREFILL_PORT=$PREFILL_PORT DECODE_PORT=$DECODE_PORT PROXY_PORT=$PROXY_PORT python3 -m pytest -s -v "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
116
117
# Clean up before running next model
118
cleanup_instances