| 1 | group: Miscellaneous |
| 2 | depends_on: |
| 3 | - image-build |
| 4 | steps: |
| 5 | - label: ":nvidia: (H200 MIG 18GB) V1 Sample" |
| 6 | key: v1-sample |
| 7 | timeout_in_minutes: 35 |
| 8 | device: h200_18gb |
| 9 | source_file_dependencies: &v1-sample-logits-deps |
| 10 | - vllm/config/ |
| 11 | - vllm/distributed/ |
| 12 | - "!vllm/distributed/kv_transfer/" |
| 13 | - vllm/engine/ |
| 14 | - vllm/inputs/ |
| 15 | - vllm/logger.py |
| 16 | - vllm/model_executor/ |
| 17 | - vllm/platforms/ |
| 18 | - vllm/sampling_params.py |
| 19 | - vllm/transformers_utils/ |
| 20 | - vllm/utils/ |
| 21 | - vllm/v1/ |
| 22 | - tests/v1/sample |
| 23 | - tests/v1/logits_processors |
| 24 | - tests/v1/test_oracle.py |
| 25 | - tests/v1/test_request.py |
| 26 | - tests/v1/test_outputs.py |
| 27 | commands: |
| 28 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 29 | - pytest -v -s v1/sample |
| 30 | mirror: |
| 31 | amd: |
| 32 | label: ":amd: (MI355 DPX) V1 Sample" |
| 33 | dind: false |
| 34 | device: mi355_dpx |
| 35 | timeout_in_minutes: 70 |
| 36 | depends_on: |
| 37 | - image-build-amd |
| 38 | |
| 39 | - label: ":nvidia: (H200 MIG 18GB) V1 Logits + Oracle" |
| 40 | key: v1-logits-oracle |
| 41 | timeout_in_minutes: 35 |
| 42 | device: h200_18gb |
| 43 | source_file_dependencies: *v1-sample-logits-deps |
| 44 | commands: |
| 45 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 46 | # Keep these in separate pytest processes because the CUDA-initializing |
| 47 | # correctness tests interfere with the fork-based logits processor tests. |
| 48 | - pytest -v -s v1/logits_processors |
| 49 | - pytest -v -s v1/test_oracle.py |
| 50 | - pytest -v -s v1/test_request.py |
| 51 | - pytest -v -s v1/test_outputs.py |
| 52 | mirror: |
| 53 | amd: |
| 54 | label: ":amd: (MI355 DPX) V1 Logits + Oracle" |
| 55 | dind: false |
| 56 | device: mi355_dpx |
| 57 | timeout_in_minutes: 70 |
| 58 | depends_on: |
| 59 | - image-build-amd |
| 60 | |
| 61 | - label: ":nvidia: (H200 MIG 35GB) V1 Core" |
| 62 | device: h200_35gb |
| 63 | key: v1-core |
| 64 | timeout_in_minutes: 45 |
| 65 | source_file_dependencies: &v1-core-kv-metrics-deps |
| 66 | - vllm/config/ |
| 67 | - vllm/distributed/ |
| 68 | - "!vllm/distributed/kv_transfer/" |
| 69 | - vllm/engine/ |
| 70 | - vllm/entrypoints/pooling/ |
| 71 | - vllm/inputs/ |
| 72 | - vllm/lora/ |
| 73 | - vllm/model_executor/ |
| 74 | - vllm/multimodal/ |
| 75 | - vllm/outputs.py |
| 76 | - vllm/platforms/ |
| 77 | - vllm/pooling_params.py |
| 78 | - vllm/profiler/ |
| 79 | - vllm/sampling_params.py |
| 80 | - vllm/tokenizers/ |
| 81 | - vllm/transformers_utils/ |
| 82 | - vllm/utils/ |
| 83 | - vllm/v1/ |
| 84 | - tests/v1/core |
| 85 | - tests/v1/executor |
| 86 | - tests/v1/kv_offload |
| 87 | - tests/v1/simple_kv_offload |
| 88 | - tests/v1/worker |
| 89 | - tests/v1/streaming_input |
| 90 | - tests/v1/kv_connector/unit |
| 91 | - tests/v1/ec_connector/unit |
| 92 | - tests/v1/metrics |
| 93 | - tests/entrypoints/openai/correctness/test_lmeval.py |
| 94 | commands: |
| 95 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 96 | - pytest -v -s -m 'not cpu_test' v1/core |
| 97 | mirror: |
| 98 | amd: |
| 99 | label: ":amd: (MI355 DPX) V1 Core" |
| 100 | dind: false |
| 101 | device: mi355_dpx |
| 102 | timeout_in_minutes: 60 |
| 103 | depends_on: |
| 104 | - image-build-amd |
| 105 | |
| 106 | - label: ":nvidia: (H200 MIG 18GB) V1 Executor + Worker" |
| 107 | device: h200_18gb |
| 108 | key: v1-executor-worker |
| 109 | timeout_in_minutes: 45 |
| 110 | source_file_dependencies: *v1-core-kv-metrics-deps |
| 111 | commands: |
| 112 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 113 | # split the test to avoid interference |
| 114 | - pytest -v -s v1/executor |
| 115 | - pytest -v -s v1/worker |
| 116 | - pytest -v -s v1/streaming_input |
| 117 | mirror: |
| 118 | amd: |
| 119 | label: ":amd: (MI355 DPX) V1 Executor + Worker" |
| 120 | dind: false |
| 121 | device: mi355_dpx |
| 122 | timeout_in_minutes: 60 |
| 123 | depends_on: |
| 124 | - image-build-amd |
| 125 | |
| 126 | - label: ":nvidia: (H200 MIG 35GB) V1 KV Offload" |
| 127 | device: h200_35gb |
| 128 | key: v1-kv-offload |
| 129 | timeout_in_minutes: 45 |
| 130 | source_file_dependencies: *v1-core-kv-metrics-deps |
| 131 | commands: |
| 132 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 133 | - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh |
| 134 | - pytest -v -s v1/kv_offload |
| 135 | - pytest -v -s v1/simple_kv_offload |
| 136 | mirror: |
| 137 | amd: |
| 138 | label: ":amd: (MI355 DPX) V1 KV Offload" |
| 139 | dind: false |
| 140 | device: mi355_dpx |
| 141 | timeout_in_minutes: 60 |
| 142 | depends_on: |
| 143 | - image-build-amd |
| 144 | |
| 145 | - label: ":nvidia: (H200 MIG 35GB) V1 KV Connectors Shard %N" |
| 146 | device: h200_35gb |
| 147 | key: v1-kv-connectors |
| 148 | timeout_in_minutes: 45 |
| 149 | parallelism: 4 |
| 150 | source_file_dependencies: *v1-core-kv-metrics-deps |
| 151 | commands: |
| 152 | - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh |
| 153 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 154 | - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB |
| 155 | - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB |
| 156 | mirror: |
| 157 | amd: |
| 158 | label: ":amd: (MI355 DPX) V1 KV Connectors Shard %N" |
| 159 | dind: false |
| 160 | device: mi355_dpx |
| 161 | timeout_in_minutes: 60 |
| 162 | depends_on: |
| 163 | - image-build-amd |
| 164 | |
| 165 | - label: ":nvidia: (H200 MIG 35GB) V1 Metrics + LM Eval" |
| 166 | device: h200_35gb |
| 167 | key: v1-metrics-lmeval |
| 168 | timeout_in_minutes: 45 |
| 169 | env: |
| 170 | # GitHub intermittently rejects HTTP/2 upload-pack requests from the H200 |
| 171 | # CI image. Force Git's smart protocol to use HTTP/1.1 for source installs. |
| 172 | GIT_CONFIG_COUNT: "1" |
| 173 | GIT_CONFIG_KEY_0: http.version |
| 174 | GIT_CONFIG_VALUE_0: HTTP/1.1 |
| 175 | GIT_TERMINAL_PROMPT: "0" |
| 176 | source_file_dependencies: *v1-core-kv-metrics-deps |
| 177 | commands: |
| 178 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 179 | - pytest -v -s -m 'not cpu_test' v1/metrics |
| 180 | # Integration test for streaming correctness (requires special branch). |
| 181 | - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api |
| 182 | - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine |
| 183 | mirror: |
| 184 | amd: |
| 185 | label: ":amd: (MI355 DPX) V1 Metrics + LM Eval" |
| 186 | dind: false |
| 187 | device: mi355_dpx |
| 188 | timeout_in_minutes: 65 |
| 189 | depends_on: |
| 190 | - image-build-amd |
| 191 | |
| 192 | - label: ":computer: (CPU) V1 Others" |
| 193 | key: v1-others-cpu |
| 194 | depends_on: |
| 195 | - image-build-cpu |
| 196 | source_file_dependencies: |
| 197 | - vllm/config/ |
| 198 | - vllm/distributed/ |
| 199 | - "!vllm/distributed/kv_transfer/" |
| 200 | - vllm/engine/ |
| 201 | - vllm/inputs/ |
| 202 | - vllm/lora/ |
| 203 | - vllm/multimodal/ |
| 204 | - vllm/outputs.py |
| 205 | - vllm/platforms/ |
| 206 | - vllm/pooling_params.py |
| 207 | - vllm/profiler/ |
| 208 | - vllm/sampling_params.py |
| 209 | - vllm/tokenizers/ |
| 210 | - vllm/transformers_utils/ |
| 211 | - vllm/utils/ |
| 212 | - vllm/v1/ |
| 213 | - tests/v1 |
| 214 | - tests/watermarking |
| 215 | device: cpu-small |
| 216 | commands: |
| 217 | # split the test to avoid interference |
| 218 | - pytest -v -s -m 'cpu_test' v1/core |
| 219 | - pytest -v -s v1/structured_output |
| 220 | - pytest -v -s v1/test_serial_utils.py |
| 221 | - pytest -v -s v1/test_kv_cache_spec_registry.py |
| 222 | - pytest -v -s v1/cudagraph/test_cudagraph_manager.py |
| 223 | - pytest -v -s -m 'cpu_test' v1/kv_connector/unit |
| 224 | - pytest -v -s -m 'cpu_test' v1/ec_connector/unit |
| 225 | - pytest -v -s -m 'cpu_test' v1/metrics |
| 226 | - pytest -v -s watermarking |
| 227 | |
| 228 | - label: ":nvidia: (H200 MIG 18GB) Extract Hidden States Integration" |
| 229 | key: extract-hidden-states-integration |
| 230 | timeout_in_minutes: 20 |
| 231 | device: h200_18gb |
| 232 | working_dir: "/vllm-workspace" |
| 233 | source_file_dependencies: |
| 234 | - vllm/v1/spec_decode/extract_hidden_states.py |
| 235 | - vllm/model_executor/models/extract_hidden_states.py |
| 236 | - vllm/transformers_utils/configs/extract_hidden_states.py |
| 237 | - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py |
| 238 | - tests/v1/kv_connector/extract_hidden_states_integration |
| 239 | commands: |
| 240 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 241 | - pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration |
| 242 | mirror: |
| 243 | amd: |
| 244 | label: ":amd: (MI355 DPX) Extract Hidden States Integration Single GPU" |
| 245 | dind: false |
| 246 | device: mi355_dpx |
| 247 | num_devices: 1 |
| 248 | timeout_in_minutes: 45 |
| 249 | working_dir: /vllm-workspace |
| 250 | depends_on: |
| 251 | - image-build-amd |
| 252 | source_file_dependencies: |
| 253 | - vllm/v1/spec_decode/extract_hidden_states.py |
| 254 | - vllm/model_executor/models/extract_hidden_states.py |
| 255 | - vllm/transformers_utils/configs/extract_hidden_states.py |
| 256 | - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py |
| 257 | - tests/v1/kv_connector/extract_hidden_states_integration |
| 258 | - vllm/platforms/rocm.py |
| 259 | env: |
| 260 | VLLM_WORKER_MULTIPROC_METHOD: "spawn" |
| 261 | commands: |
| 262 | - pytest -v -s -m 'not distributed' tests/v1/kv_connector/extract_hidden_states_integration |
| 263 | |
| 264 | - label: ":nvidia: (L4) Extract Hidden States Integration" |
| 265 | key: extract-hidden-states-integration-2-gpus |
| 266 | timeout_in_minutes: 20 |
| 267 | device: l4 |
| 268 | num_devices: 2 |
| 269 | working_dir: "/vllm-workspace" |
| 270 | source_file_dependencies: |
| 271 | - vllm/v1/spec_decode/extract_hidden_states.py |
| 272 | - vllm/model_executor/models/extract_hidden_states.py |
| 273 | - vllm/transformers_utils/configs/extract_hidden_states.py |
| 274 | - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py |
| 275 | - tests/v1/kv_connector/extract_hidden_states_integration |
| 276 | commands: |
| 277 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 278 | - pytest -v -s -m 'distributed' tests/v1/kv_connector/extract_hidden_states_integration |
| 279 | mirror: |
| 280 | amd: |
| 281 | label: ":amd: (MI355) Extract Hidden States Integration" |
| 282 | dind: false |
| 283 | device: mi355_2 |
| 284 | timeout_in_minutes: 30 |
| 285 | depends_on: |
| 286 | - image-build-amd |
| 287 | source_file_dependencies: |
| 288 | - vllm/config/speculative.py |
| 289 | - vllm/distributed/kv_transfer/kv_connector/ |
| 290 | - vllm/model_executor/layers/attention/ |
| 291 | - vllm/model_executor/layers/mamba/ |
| 292 | - vllm/model_executor/model_loader/ |
| 293 | - vllm/model_executor/models/extract_hidden_states.py |
| 294 | - vllm/model_executor/models/llama.py |
| 295 | - vllm/model_executor/models/qwen3_5.py |
| 296 | - vllm/model_executor/models/qwen3_next.py |
| 297 | - vllm/model_executor/models/registry.py |
| 298 | - vllm/transformers_utils/configs/extract_hidden_states.py |
| 299 | - vllm/transformers_utils/configs/qwen3_5.py |
| 300 | - vllm/v1/attention/backends/ |
| 301 | - vllm/v1/attention/selector.py |
| 302 | - vllm/v1/kv_cache_interface.py |
| 303 | - vllm/v1/spec_decode/extract_hidden_states.py |
| 304 | - vllm/v1/worker/gpu_model_runner.py |
| 305 | - vllm/_aiter_ops.py |
| 306 | - tests/v1/kv_connector/extract_hidden_states_integration/ |
| 307 | - vllm/platforms/rocm.py |
| 308 | |
| 309 | - label: ":nvidia: (H200 MIG 18GB) Regression" |
| 310 | key: regression |
| 311 | timeout_in_minutes: 30 |
| 312 | device: h200_18gb |
| 313 | source_file_dependencies: |
| 314 | - vllm/config/ |
| 315 | - vllm/distributed/ |
| 316 | - "!vllm/distributed/kv_transfer/" |
| 317 | - vllm/engine/ |
| 318 | - vllm/inputs/ |
| 319 | - vllm/model_executor/ |
| 320 | - vllm/multimodal/ |
| 321 | - vllm/platforms/ |
| 322 | - vllm/sampling_params.py |
| 323 | - vllm/transformers_utils/ |
| 324 | - vllm/utils/ |
| 325 | - vllm/v1/ |
| 326 | - tests/test_regression |
| 327 | commands: |
| 328 | - pip install 'modelscope<1.38' |
| 329 | - pytest -v -s test_regression.py |
| 330 | working_dir: "/vllm-workspace/tests" # optional |
| 331 | mirror: |
| 332 | amd: |
| 333 | label: ":amd: (MI355 DPX) Regression" |
| 334 | dind: false |
| 335 | device: mi355_dpx |
| 336 | timeout_in_minutes: 25 |
| 337 | depends_on: |
| 338 | - image-build-amd |
| 339 | working_dir: /vllm-workspace/tests |
| 340 | |
| 341 | - label: ":nvidia: (H200 MIG 35GB) Examples" |
| 342 | device: h200_35gb |
| 343 | key: examples |
| 344 | timeout_in_minutes: 40 |
| 345 | working_dir: "/vllm-workspace/examples" |
| 346 | source_file_dependencies: |
| 347 | - vllm/entrypoints |
| 348 | - vllm/multimodal |
| 349 | - examples/ |
| 350 | commands: |
| 351 | - pip install --no-deps tensorizer # for tensorizer test |
| 352 | # for basic |
| 353 | - python3 basic/offline_inference/chat.py |
| 354 | - python3 basic/offline_inference/generate.py --model facebook/opt-125m |
| 355 | - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 |
| 356 | - python3 basic/offline_inference/classify.py |
| 357 | - python3 basic/offline_inference/embed.py |
| 358 | - python3 basic/offline_inference/score.py |
| 359 | # for multi-modal models |
| 360 | - python3 generate/multimodal/audio_language_offline.py --seed 0 |
| 361 | - python3 generate/multimodal/vision_language_offline.py --seed 0 |
| 362 | - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 |
| 363 | - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 |
| 364 | # for pooling models |
| 365 | - python3 pooling/embed/vision_embedding_offline.py --seed 0 |
| 366 | # for features demo |
| 367 | - python3 features/automatic_prefix_caching/prefix_caching_offline.py |
| 368 | - python3 deployment/llm_engine_example.py |
| 369 | - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors |
| 370 | - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 |
| 371 | # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU |
| 372 | - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 |
| 373 | mirror: |
| 374 | amd: |
| 375 | label: ":amd: (MI355 DPX) Examples" |
| 376 | dind: false |
| 377 | device: mi355_dpx |
| 378 | timeout_in_minutes: 75 |
| 379 | source_file_dependencies: |
| 380 | - vllm/entrypoints |
| 381 | - vllm/multimodal |
| 382 | - examples/ |
| 383 | - vllm/platforms/rocm.py |
| 384 | depends_on: |
| 385 | - image-build-amd |
| 386 | |
| 387 | - label: ":nvidia: (L4) Metrics, Tracing" |
| 388 | key: metrics-tracing-2-gpus |
| 389 | timeout_in_minutes: 25 |
| 390 | device: l4 |
| 391 | num_devices: 2 |
| 392 | source_file_dependencies: |
| 393 | - vllm/config/ |
| 394 | - vllm/distributed/ |
| 395 | - "!vllm/distributed/kv_transfer/" |
| 396 | - vllm/engine/ |
| 397 | - vllm/inputs/ |
| 398 | - vllm/model_executor/ |
| 399 | - vllm/multimodal/ |
| 400 | - vllm/platforms/ |
| 401 | - vllm/sampling_params.py |
| 402 | - vllm/tracing/ |
| 403 | - vllm/transformers_utils/ |
| 404 | - vllm/utils/ |
| 405 | - vllm/v1/ |
| 406 | - tests/v1/tracing |
| 407 | - tests/tracing/ |
| 408 | commands: |
| 409 | - "pip install \ |
| 410 | 'opentelemetry-sdk>=1.26.0' \ |
| 411 | 'opentelemetry-api>=1.26.0' \ |
| 412 | 'opentelemetry-exporter-otlp>=1.26.0' \ |
| 413 | 'opentelemetry-semantic-conventions-ai>=0.4.1'" |
| 414 | - pytest -v -s v1/tracing |
| 415 | - pytest -v -s tracing |
| 416 | mirror: |
| 417 | amd: |
| 418 | label: ":amd: (MI300) Metrics, Tracing" |
| 419 | dind: false |
| 420 | device: mi300_2 |
| 421 | timeout_in_minutes: 30 |
| 422 | depends_on: |
| 423 | - image-build-amd |
| 424 | |
| 425 | - label: ":nvidia: (H200 MIG 18GB) Python-only Installation" |
| 426 | key: python-only-installation |
| 427 | depends_on: ~ |
| 428 | optional: true |
| 429 | timeout_in_minutes: 20 |
| 430 | device: h200_18gb |
| 431 | source_file_dependencies: |
| 432 | - tests/standalone_tests/python_only_compile.sh |
| 433 | - setup.py |
| 434 | commands: |
| 435 | - bash standalone_tests/python_only_compile.sh |
| 436 | mirror: |
| 437 | amd: |
| 438 | label: ":amd: (MI300) Python-only Installation" |
| 439 | dind: false |
| 440 | device: mi300_1 |
| 441 | timeout_in_minutes: 55 |
| 442 | depends_on: |
| 443 | - image-build-amd |
| 444 | source_file_dependencies: |
| 445 | - tests/standalone_tests/python_only_compile.sh |
| 446 | - setup.py |
| 447 | - vllm/platforms/rocm.py |
| 448 | |
| 449 | - label: ":nvidia: (H200 MIG 35GB) Async Engine, Inputs, Utils, Worker" |
| 450 | device: h200_35gb |
| 451 | key: async-engine-inputs-utils-worker |
| 452 | timeout_in_minutes: 25 |
| 453 | source_file_dependencies: |
| 454 | - vllm/assets/ |
| 455 | - vllm/config/ |
| 456 | - vllm/distributed/ |
| 457 | - "!vllm/distributed/kv_transfer/" |
| 458 | - vllm/engine/ |
| 459 | - vllm/inputs/ |
| 460 | - vllm/model_executor/ |
| 461 | - vllm/multimodal/ |
| 462 | - vllm/platforms/ |
| 463 | - vllm/sampling_params.py |
| 464 | - vllm/tokenizers/ |
| 465 | - vllm/transformers_utils/ |
| 466 | - vllm/utils/ |
| 467 | - vllm/v1/ |
| 468 | - tests/detokenizer |
| 469 | - tests/multimodal |
| 470 | - tests/utils_ |
| 471 | commands: |
| 472 | - pytest -v -s detokenizer |
| 473 | - pytest -v -s -m 'not cpu_test' multimodal |
| 474 | - pytest -v -s utils_ |
| 475 | mirror: |
| 476 | amd: |
| 477 | label: ":amd: (MI355 DPX) Async Engine, Inputs, Utils, Worker" |
| 478 | dind: false |
| 479 | device: mi355_dpx |
| 480 | timeout_in_minutes: 50 |
| 481 | depends_on: |
| 482 | - image-build-amd |
| 483 | commands: |
| 484 | - pytest -v -s detokenizer |
| 485 | - pytest -v -s -m 'not cpu_test' multimodal |
| 486 | # test_cache.py marks every test cpu_test, including this GPU regression. |
| 487 | - pytest -v -s multimodal/test_cache.py::test_sleep_wake_preserves_mm_cache_consistency |
| 488 | - pytest -v -s utils_ |
| 489 | |
| 490 | - label: ":computer: (CPU) Params, Env, Tokenizers, Parser" |
| 491 | key: cpu-params-env-tokenizers-parser |
| 492 | depends_on: |
| 493 | - image-build-cpu |
| 494 | timeout_in_minutes: 35 |
| 495 | source_file_dependencies: &cpu-async-engine-deps |
| 496 | - vllm/assets/ |
| 497 | - vllm/config/ |
| 498 | - vllm/device_allocator/ |
| 499 | - vllm/connections.py |
| 500 | - vllm/engine/arg_utils.py |
| 501 | - vllm/entrypoints/chat_utils.py |
| 502 | - vllm/entrypoints/mcp/ |
| 503 | - vllm/entrypoints/openai/chat_completion/protocol.py |
| 504 | - vllm/entrypoints/openai/completion/protocol.py |
| 505 | - vllm/entrypoints/openai/responses/protocol.py |
| 506 | - vllm/entrypoints/generate/base/protocol.py |
| 507 | - vllm/env_override.py |
| 508 | - vllm/envs.py |
| 509 | - vllm/exceptions.py |
| 510 | - vllm/inputs/ |
| 511 | - vllm/logging_utils/ |
| 512 | - vllm/logprobs.py |
| 513 | - vllm/model_executor/layers/quantization/quark/ |
| 514 | - vllm/model_executor/model_loader/tensorizer.py |
| 515 | - vllm/multimodal/ |
| 516 | - vllm/outputs.py |
| 517 | - vllm/parser/ |
| 518 | - vllm/platforms/ |
| 519 | - vllm/pooling_params.py |
| 520 | - vllm/ray/ |
| 521 | - vllm/reasoning/ |
| 522 | - vllm/renderers/ |
| 523 | - vllm/sampling_params.py |
| 524 | - vllm/scalar_type.py |
| 525 | - vllm/tokenizers/ |
| 526 | - vllm/tool_parsers/ |
| 527 | - vllm/transformers_utils/ |
| 528 | - vllm/triton_utils/ |
| 529 | - vllm/utils/ |
| 530 | - vllm/v1/ |
| 531 | - vllm/version.py |
| 532 | - cmake/utils.cmake |
| 533 | - tests/test_uvicorn_logging.py |
| 534 | - tests/test_attention_backend_registry.py |
| 535 | - tests/test_audio_media_size_precheck.py |
| 536 | - tests/test_cmake_utils.py |
| 537 | - tests/test_connections.py |
| 538 | - tests/test_embedded_commit.py |
| 539 | - tests/test_envs.py |
| 540 | - tests/test_force_first_config.py |
| 541 | - tests/test_fxgraphcache_pickle_patch.py |
| 542 | - tests/test_logprobs.py |
| 543 | - tests/test_outputs.py |
| 544 | - tests/test_pcp_dp.py |
| 545 | - tests/test_pooling_params.py |
| 546 | - tests/test_quantization_revision_pin.py |
| 547 | - tests/test_ray_env.py |
| 548 | - tests/test_ray_env_utils.py |
| 549 | - tests/test_request_input_bounds.py |
| 550 | - tests/test_sampling_params.py |
| 551 | - tests/test_scalartype.py |
| 552 | - tests/test_tensorizer_revision_pin.py |
| 553 | - tests/test_triton_utils.py |
| 554 | - tests/test_version.py |
| 555 | - tests/test_zen_cpu_platform_detection.py |
| 556 | - tests/multimodal |
| 557 | - tests/renderers |
| 558 | - tests/standalone_tests/lazy_imports.py |
| 559 | - tests/reasoning |
| 560 | - tools/pre_commit/check_test_tethering.py |
| 561 | - tools/pre_commit/test_tethering_allowlist.txt |
| 562 | - tests/tools/test_check_test_tethering.py |
| 563 | - tests/tool_parsers |
| 564 | - tests/tokenizers_ |
| 565 | - tests/parser |
| 566 | - tests/transformers_utils |
| 567 | - tests/config |
| 568 | - tests/device_allocator |
| 569 | device: cpu-small |
| 570 | commands: |
| 571 | - pip install cmake # test_cmake_utils.py shells out to cmake; the CPU image has none |
| 572 | - python3 standalone_tests/lazy_imports.py |
| 573 | - pytest -v -s device_allocator |
| 574 | - pytest -v -s test_uvicorn_logging.py |
| 575 | - pytest -v -s test_attention_backend_registry.py |
| 576 | - pytest -v -s test_audio_media_size_precheck.py |
| 577 | - pytest -v -s test_cmake_utils.py |
| 578 | - pytest -v -s test_connections.py |
| 579 | - pytest -v -s test_embedded_commit.py |
| 580 | - pytest -v -s test_envs.py |
| 581 | - pytest -v -s test_force_first_config.py |
| 582 | - pytest -v -s test_fxgraphcache_pickle_patch.py |
| 583 | - pytest -v -s test_logprobs.py |
| 584 | - pytest -v -s test_outputs.py |
| 585 | - pytest -v -s test_pcp_dp.py |
| 586 | - pytest -v -s test_pooling_params.py |
| 587 | - pytest -v -s test_quantization_revision_pin.py |
| 588 | - pytest -v -s test_ray_env.py |
| 589 | - pytest -v -s test_ray_env_utils.py |
| 590 | - pytest -v -s test_request_input_bounds.py |
| 591 | - pytest -v -s test_sampling_params.py |
| 592 | - pytest -v -s test_scalartype.py |
| 593 | - pytest -v -s test_tensorizer_revision_pin.py |
| 594 | - pytest -v -s test_triton_utils.py |
| 595 | - pytest -v -s test_version.py |
| 596 | - pytest -v -s test_zen_cpu_platform_detection.py |
| 597 | - pytest -v -s tokenizers_ |
| 598 | - pytest -v -s parser |
| 599 | - pytest -v -s transformers_utils |
| 600 | |
| 601 | - label: ":computer: (CPU) Multimodal + Config" |
| 602 | key: cpu-multimodal-config |
| 603 | depends_on: |
| 604 | - image-build-cpu |
| 605 | timeout_in_minutes: 35 |
| 606 | source_file_dependencies: *cpu-async-engine-deps |
| 607 | device: cpu-small |
| 608 | commands: |
| 609 | - pytest -v -s -m 'cpu_test' multimodal |
| 610 | - pytest -v -s config |
| 611 | - pytest -v -s tools/test_check_test_tethering.py |
| 612 | |
| 613 | - label: ":computer: (CPU) Reasoning + Renderers" |
| 614 | key: cpu-reasoning-renderers |
| 615 | depends_on: |
| 616 | - image-build-cpu |
| 617 | timeout_in_minutes: 35 |
| 618 | source_file_dependencies: *cpu-async-engine-deps |
| 619 | device: cpu-small |
| 620 | commands: |
| 621 | - pytest -v -s reasoning |
| 622 | - pytest -v -s renderers |
| 623 | |
| 624 | - label: ":computer: (CPU) Tool Parsers" |
| 625 | key: cpu-tool-parsers |
| 626 | depends_on: |
| 627 | - image-build-cpu |
| 628 | timeout_in_minutes: 35 |
| 629 | source_file_dependencies: *cpu-async-engine-deps |
| 630 | device: cpu-small |
| 631 | commands: |
| 632 | - pytest -v -s tool_parsers |
| 633 | |
| 634 | - label: ":nvidia: (A100) Batch Invariance" |
| 635 | key: batch-invariance-a100 |
| 636 | timeout_in_minutes: 60 |
| 637 | device: a100 |
| 638 | source_file_dependencies: |
| 639 | - vllm/v1/attention |
| 640 | - vllm/model_executor/layers |
| 641 | - tests/v1/determinism/ |
| 642 | commands: |
| 643 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 644 | - pip install pytest-timeout pytest-forked |
| 645 | - pytest -v -s v1/determinism/test_batch_invariance.py |
| 646 | - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA |
| 647 | |
| 648 | - label: ":nvidia: (H100) Batch Invariance" |
| 649 | key: batch-invariance-h100 |
| 650 | timeout_in_minutes: 60 |
| 651 | device: h100 |
| 652 | source_file_dependencies: |
| 653 | - vllm/v1/attention |
| 654 | - vllm/model_executor/layers |
| 655 | - tests/v1/determinism/ |
| 656 | commands: |
| 657 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 658 | - pip install pytest-timeout pytest-forked |
| 659 | - pytest -v -s v1/determinism/test_batch_invariance.py |
| 660 | - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py |
| 661 | - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA |
| 662 | - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN |
| 663 | mirror: |
| 664 | amd: |
| 665 | label: ":amd: (MI250) Batch Invariance" |
| 666 | dind: false |
| 667 | device: mi250_1 |
| 668 | timeout_in_minutes: 35 |
| 669 | depends_on: |
| 670 | - image-build-amd |
| 671 | source_file_dependencies: |
| 672 | - vllm/v1/attention |
| 673 | - vllm/model_executor/layers |
| 674 | - tests/v1/determinism/ |
| 675 | - vllm/_aiter_ops.py |
| 676 | - vllm/platforms/rocm.py |
| 677 | commands: |
| 678 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 679 | - pip install pytest-timeout pytest-forked |
| 680 | - pytest -v -s v1/determinism/test_batch_invariance.py |
| 681 | - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py |
| 682 | |
| 683 | - label: ":nvidia: (B200) Batch Invariance" |
| 684 | key: batch-invariance-b200 |
| 685 | timeout_in_minutes: 45 |
| 686 | device: b200-k8s |
| 687 | source_file_dependencies: |
| 688 | - vllm/v1/attention |
| 689 | - vllm/model_executor/layers |
| 690 | - tests/v1/determinism/ |
| 691 | commands: |
| 692 | - export VLLM_WORKER_MULTIPROC_METHOD=spawn |
| 693 | - pip install pytest-timeout pytest-forked |
| 694 | - pytest -v -s v1/determinism/test_batch_invariance.py |
| 695 | - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py |
| 696 | - pytest -v -s v1/determinism/test_batch_invariance_vlm.py |
| 697 | - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA |
| 698 | - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN |
| 699 | - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py |
| 700 | - pytest -v -s v1/determinism/test_matmul_batch_invariant.py |
| 701 | - pytest -v -s v1/determinism/test_cutlass_batch_invariance.py |
| 702 | - pytest -v -s v1/determinism/test_online_batch_invariance.py |
| 703 | |
| 704 | - label: ":nvidia: (H200 MIG 35GB) Acceptance Length (Large Models)" |
| 705 | device: h200_35gb |
| 706 | key: acceptance-length-test-large-models |
| 707 | timeout_in_minutes: 20 |
| 708 | gpu: h100 |
| 709 | optional: true |
| 710 | num_gpus: 1 |
| 711 | working_dir: "/vllm-workspace/tests" |
| 712 | source_file_dependencies: |
| 713 | - vllm/v1/spec_decode/ |
| 714 | - vllm/model_executor/models/mlp_speculator.py |
| 715 | - tests/v1/spec_decode/test_acceptance_length.py |
| 716 | commands: |
| 717 | - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 |
| 718 | - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test |
| 719 | mirror: |
| 720 | amd: |
| 721 | label: ":amd: (MI355 DPX) Acceptance Length (Large Models)" |
| 722 | dind: false |
| 723 | device: mi355_dpx |
| 724 | timeout_in_minutes: 45 |
| 725 | depends_on: |
| 726 | - image-build-amd |
| 727 | working_dir: /vllm-workspace/tests |
| 728 | source_file_dependencies: |
| 729 | - vllm/v1/spec_decode/ |
| 730 | - vllm/model_executor/models/mlp_speculator.py |
| 731 | - tests/v1/spec_decode/test_acceptance_length.py |
| 732 | - vllm/platforms/rocm.py |