@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cc8f1f4764
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 19:15 UTC

[ROCm][CI] Preparing gfx90a mirroring (#36210)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>

Andreas Karatzas committed Mar 12, 2026 at 15:42 UTC cc8f1f47644868869d5a7fb4c55cebbf91fb9943
2 files changed +1330 -1
.buildkite/hardware_tests/amd.yaml
+1 -1
@@ -10,7 +10,7 @@ steps:
10 docker build
11 --build-arg max_jobs=16
12 --build-arg REMOTE_VLLM=1
13 - --build-arg ARG_PYTORCH_ROCM_ARCH='gfx942;gfx950'
13 + --build-arg ARG_PYTORCH_ROCM_ARCH='gfx90a;gfx942;gfx950'
14 --build-arg VLLM_BRANCH=$BUILDKITE_COMMIT
15 --tag "rocm/vllm-ci:${BUILDKITE_COMMIT}"
16 -f docker/Dockerfile.rocm
.buildkite/test-amd.yaml
+1329
@@ -33,6 +33,1335 @@
33 # Note that all steps execute in parallel.
34
35 steps:
36 +
37 +
38 +#####################################################################################################################################
39 +# #
40 +# MI250 test definitions ( currently the test set is completely mirrored // TBD which tests are to be routed there ultimately) #
41 +# #
42 +#####################################################################################################################################
43 +
44 +- label: Pytorch Nightly Dependency Override Check # 2min
45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
46 + agent_pool: mi250_1
47 + source_file_dependencies:
48 + - requirements/nightly_torch_test.txt
49 + commands:
50 + - bash standalone_tests/pytorch_nightly_dependency.sh
51 +
52 +- label: Async Engine, Inputs, Utils, Worker Test # 10min
53 + timeout_in_minutes: 15
54 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
55 + agent_pool: mi250_1
56 + source_file_dependencies:
57 + - vllm/
58 + - tests/detokenizer
59 + - tests/multimodal
60 + - tests/utils_
61 + commands:
62 + - pytest -v -s detokenizer
63 + - pytest -v -s -m 'not cpu_test' multimodal
64 + - pytest -v -s utils_
65 +
66 +- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 20min
67 + timeout_in_minutes: 30
68 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
69 + agent_pool: mi250_1
70 + source_file_dependencies:
71 + - vllm/
72 + - tests/test_inputs.py
73 + - tests/test_outputs.py
74 + - tests/test_pooling_params.py
75 + - tests/multimodal
76 + - tests/renderers
77 + - tests/standalone_tests/lazy_imports.py
78 + - tests/tokenizers_
79 + - tests/tool_parsers
80 + - tests/transformers_utils
81 + - tests/config
82 + no_gpu: true
83 + commands:
84 + - python3 standalone_tests/lazy_imports.py
85 + - pytest -v -s test_inputs.py
86 + - pytest -v -s test_outputs.py
87 + - pytest -v -s test_pooling_params.py
88 + - pytest -v -s -m 'cpu_test' multimodal
89 + - pytest -v -s renderers
90 + - pytest -v -s tokenizers_
91 + - pytest -v -s tool_parsers
92 + - pytest -v -s transformers_utils
93 + - pytest -v -s config
94 +
95 +- label: Python-only Installation Test # 10min
96 + timeout_in_minutes: 20
97 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
98 + agent_pool: mi250_1
99 + source_file_dependencies:
100 + - tests/standalone_tests/python_only_compile.sh
101 + - setup.py
102 + commands:
103 + - bash standalone_tests/python_only_compile.sh
104 +
105 +- label: Basic Correctness Test # 20min
106 + timeout_in_minutes: 30
107 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
108 + agent_pool: mi250_1
109 + fast_check: true
110 + torch_nightly: true
111 + source_file_dependencies:
112 + - vllm/
113 + - tests/basic_correctness/test_basic_correctness
114 + - tests/basic_correctness/test_cpu_offload
115 + - tests/basic_correctness/test_cumem.py
116 + commands:
117 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
118 + - pytest -v -s basic_correctness/test_cumem.py
119 + - pytest -v -s basic_correctness/test_basic_correctness.py
120 + - pytest -v -s basic_correctness/test_cpu_offload.py
121 +
122 +- label: Entrypoints Unit Tests # 5min
123 + timeout_in_minutes: 10
124 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
125 + agent_pool: mi250_1
126 + working_dir: "/vllm-workspace/tests"
127 + fast_check: true
128 + source_file_dependencies:
129 + - vllm/entrypoints
130 + - tests/entrypoints/
131 + commands:
132 + - pytest -v -s entrypoints/openai/tool_parsers
133 + - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/rpc --ignore=entrypoints/instrumentator --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
134 +
135 +- label: Entrypoints Integration Test (LLM) # 30min
136 + timeout_in_minutes: 40
137 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
138 + agent_pool: mi250_1
139 + fast_check: true
140 + torch_nightly: true
141 + working_dir: "/vllm-workspace/tests"
142 + source_file_dependencies:
143 + - vllm/
144 + - tests/entrypoints/llm
145 + - tests/entrypoints/offline_mode
146 + commands:
147 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
148 + - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
149 + - pytest -v -s entrypoints/llm/test_generate.py
150 + - pytest -v -s entrypoints/offline_mode
151 +
152 +- label: Entrypoints Integration Test (API Server 1) # 100min
153 + timeout_in_minutes: 130
154 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
155 + agent_pool: mi250_1
156 + working_dir: "/vllm-workspace/tests"
157 + fast_check: true
158 + torch_nightly: true
159 + source_file_dependencies:
160 + - vllm/
161 + - tests/entrypoints/openai
162 + - tests/entrypoints/test_chat_utils
163 + commands:
164 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
165 + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
166 + - pytest -v -s entrypoints/test_chat_utils.py
167 +
168 +- label: Entrypoints Integration Test (API Server 2)
169 + timeout_in_minutes: 50
170 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
171 + agent_pool: mi250_1
172 + optional: true
173 + working_dir: "/vllm-workspace/tests"
174 + fast_check: true
175 + torch_nightly: true
176 + source_file_dependencies:
177 + - vllm/
178 + - tests/entrypoints/rpc
179 + - tests/entrypoints/instrumentator
180 + - tests/tool_use
181 + commands:
182 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
183 + - pytest -v -s entrypoints/instrumentator
184 + - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
185 + - pytest -v -s tool_use
186 +
187 +- label: Entrypoints Integration Test (Pooling)
188 + timeout_in_minutes: 50
189 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
190 + agent_pool: mi250_1
191 + optional: true
192 + fast_check: true
193 + torch_nightly: true
194 + working_dir: "/vllm-workspace/tests"
195 + source_file_dependencies:
196 + - vllm/
197 + - tests/entrypoints/pooling
198 + commands:
199 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
200 + - pytest -v -s entrypoints/pooling
201 +
202 +- label: Entrypoints Integration Test (Responses API)
203 + timeout_in_minutes: 50
204 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
205 + agent_pool: mi250_1
206 + optional: true
207 + working_dir: "/vllm-workspace/tests"
208 + fast_check: true
209 + torch_nightly: true
210 + source_file_dependencies:
211 + - vllm/
212 + - tests/entrypoints/openai/responses
213 + commands:
214 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
215 + - pytest -v -s entrypoints/openai/responses
216 +
217 +- label: Distributed Tests (4 GPUs) # 35min
218 + timeout_in_minutes: 50
219 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
220 + agent_pool: mi250_4
221 + optional: true
222 + num_gpus: 4
223 + working_dir: "/vllm-workspace/tests"
224 + source_file_dependencies:
225 + - vllm/distributed/
226 + - tests/distributed/test_utils
227 + - tests/distributed/test_pynccl
228 + - tests/distributed/test_events
229 + - tests/compile/fullgraph/test_basic_correctness.py
230 + - examples/offline_inference/rlhf.py
231 + - examples/offline_inference/rlhf_colocate.py
232 + - examples/offline_inference/new_weight_syncing/
233 + - tests/examples/offline_inference/data_parallel.py
234 + - tests/v1/distributed
235 + - tests/v1/engine/test_engine_core_client.py
236 + - tests/distributed/test_symm_mem_allreduce.py
237 + commands:
238 + - export TORCH_NCCL_BLOCKING_WAIT=1
239 + - torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
240 + - PP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
241 + - TP_SIZE=4 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
242 + - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
243 + - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
244 + - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
245 + - python3 ../examples/offline_inference/data_parallel.py --enforce-eager
246 + - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
247 + - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
248 + - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
249 + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
250 + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
251 + - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
252 + - pytest -v -s distributed/test_utils.py
253 + - pytest -v -s compile/fullgraph/test_basic_correctness.py
254 + - pytest -v -s distributed/test_pynccl.py
255 + - pytest -v -s distributed/test_events.py
256 + - pytest -v -s distributed/test_symm_mem_allreduce.py
257 + - pushd ../examples/offline_inference
258 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf.py
259 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py
260 + - popd
261 + - pushd ../examples/offline_inference/new_weight_syncing
262 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_nccl.py
263 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_ipc.py
264 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_async_new_apis.py
265 + - popd
266 +
267 +- label: Distributed Tests (8 GPUs) # 4min
268 + timeout_in_minutes: 10
269 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
270 + agent_pool: mi250_8
271 + optional: true
272 + num_gpus: 8
273 + working_dir: "/vllm-workspace/tests"
274 + source_file_dependencies:
275 + - examples/offline_inference/torchrun_dp_example.py
276 + - vllm/config/parallel.py
277 + - vllm/distributed/
278 + - vllm/v1/engine/llm_engine.py
279 + - vllm/v1/executor/uniproc_executor.py
280 + - vllm/v1/worker/gpu_worker.py
281 + commands:
282 + - export TORCH_NCCL_BLOCKING_WAIT=1
283 + - torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
284 +
285 +- label: EPLB Algorithm Test # 5min
286 + timeout_in_minutes: 15
287 + mirror_hardwares: [amdexperimental, amdproduction, amdtentative, amdgfx90a]
288 + agent_pool: mi250_1
289 + working_dir: "/vllm-workspace/tests"
290 + source_file_dependencies:
291 + - vllm/distributed/eplb
292 + - tests/distributed/test_eplb_algo.py
293 + commands:
294 + - pytest -v -s distributed/test_eplb_algo.py
295 +
296 +- label: EPLB Execution Test # 10min
297 + timeout_in_minutes: 20
298 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
299 + agent_pool: mi250_4
300 + num_gpus: 4
301 + working_dir: "/vllm-workspace/tests"
302 + source_file_dependencies:
303 + - vllm/distributed/eplb
304 + - tests/distributed/test_eplb_execute.py
305 + commands:
306 + - pytest -v -s distributed/test_eplb_execute.py
307 + - pytest -v -s distributed/test_eplb_spec_decode.py
308 +
309 +- label: Metrics, Tracing Test # 12min
310 + timeout_in_minutes: 20
311 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
312 + agent_pool: mi250_2
313 + num_gpus: 2
314 + source_file_dependencies:
315 + - vllm/
316 + - tests/v1/tracing
317 + commands:
318 + - "pip install \
319 + 'opentelemetry-sdk>=1.26.0' \
320 + 'opentelemetry-api>=1.26.0' \
321 + 'opentelemetry-exporter-otlp>=1.26.0' \
322 + 'opentelemetry-semantic-conventions-ai>=0.4.1'"
323 + - pytest -v -s v1/tracing
324 +
325 +- label: Regression Test # 7min
326 + timeout_in_minutes: 20
327 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
328 + agent_pool: mi250_1
329 + working_dir: "/vllm-workspace/tests"
330 + source_file_dependencies:
331 + - vllm/
332 + - tests/test_regression
333 + commands:
334 + - pip install modelscope
335 + - pytest -v -s test_regression.py
336 +
337 +- label: Engine Test # 9min
338 + timeout_in_minutes: 15
339 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
340 + agent_pool: mi250_1
341 + source_file_dependencies:
342 + - vllm/
343 + - tests/engine
344 + - tests/test_sequence
345 + - tests/test_config
346 + - tests/test_logger
347 + - tests/test_vllm_port
348 + commands:
349 + - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
350 +
351 +- label: V1 Test e2e + engine # 65min
352 + timeout_in_minutes: 90
353 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
354 + agent_pool: mi250_1
355 + optional: true
356 + source_file_dependencies:
357 + - vllm/
358 + - tests/v1
359 + commands:
360 + - pytest -v -s v1/e2e
361 + - pytest -v -s v1/engine
362 +
363 +- label: V1 Test e2e (2 GPUs) # 65min
364 + timeout_in_minutes: 90
365 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
366 + agent_pool: mi250_2
367 + optional: true
368 + source_file_dependencies:
369 + - vllm/
370 + - tests/v1
371 + commands:
372 + - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism"
373 +
374 +- label: V1 Test e2e (4 GPUs) # 65min
375 + timeout_in_minutes: 90
376 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
377 + agent_pool: mi250_4
378 + optional: true
379 + source_file_dependencies:
380 + - vllm/
381 + - tests/v1
382 + commands:
383 + - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy"
384 +
385 +- label: V1 Test entrypoints # 35min
386 + timeout_in_minutes: 50
387 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
388 + agent_pool: mi250_1
389 + source_file_dependencies:
390 + - vllm/
391 + - tests/v1
392 + commands:
393 + - pytest -v -s v1/entrypoints
394 +
395 +- label: V1 Test others # 42min
396 + timeout_in_minutes: 60
397 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
398 + agent_pool: mi250_1
399 + optional: true
400 + source_file_dependencies:
401 + - vllm/
402 + - tests/v1
403 + commands:
404 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
405 + - pytest -v -s -m 'not cpu_test' v1/core
406 + - pytest -v -s v1/executor
407 + - pytest -v -s v1/kv_offload
408 + - pytest -v -s v1/sample
409 + - pytest -v -s v1/logits_processors
410 + - pytest -v -s v1/worker
411 + - pytest -v -s v1/spec_decode
412 + - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
413 + - pytest -v -s -m 'not cpu_test' v1/metrics
414 + - pytest -v -s v1/test_oracle.py
415 + - pytest -v -s v1/test_request.py
416 + - pytest -v -s v1/test_outputs.py
417 + - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
418 + - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
419 +
420 +- label: V1 Test attention (H100) # 10min
421 + timeout_in_minutes: 30
422 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
423 + agent_pool: mi250_1
424 + optional: true
425 + source_file_dependencies:
426 + - vllm/config/attention.py
427 + - vllm/model_executor/layers/attention
428 + - vllm/v1/attention
429 + - tests/v1/attention
430 + commands:
431 + - pytest -v -s v1/attention
432 +
433 +- label: Batch Invariance Tests (H100) # 10min
434 + timeout_in_minutes: 25
435 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
436 + agent_pool: mi250_1
437 + source_file_dependencies:
438 + - vllm/v1/attention
439 + - vllm/model_executor/layers
440 + - tests/v1/determinism/
441 + commands:
442 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
443 + - pip install pytest-timeout pytest-forked
444 + - pytest -v -s v1/determinism/test_batch_invariance.py
445 + - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
446 +
447 +- label: V1 Test others (CPU) # 5 mins
448 + timeout_in_minutes: 15
449 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
450 + agent_pool: mi250_1
451 + no_gpu: true
452 + source_file_dependencies:
453 + - vllm/
454 + - tests/v1
455 + commands:
456 + - pytest -v -s -m 'cpu_test' v1/core
457 + - pytest -v -s v1/structured_output
458 + - pytest -v -s v1/test_serial_utils.py
459 + - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
460 + - pytest -v -s -m 'cpu_test' v1/metrics
461 +
462 +
463 +- label: Examples Test # 30min
464 + timeout_in_minutes: 45
465 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
466 + agent_pool: mi250_1
467 + working_dir: "/vllm-workspace/examples"
468 + source_file_dependencies:
469 + - vllm/entrypoints
470 + - vllm/multimodal
471 + - examples/
472 + commands:
473 + - pip install tensorizer
474 + - python3 offline_inference/basic/chat.py
475 + - python3 offline_inference/basic/generate.py --model facebook/opt-125m
476 + - python3 offline_inference/basic/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
477 + - python3 offline_inference/basic/classify.py
478 + - python3 offline_inference/basic/embed.py
479 + - python3 offline_inference/basic/score.py
480 + - python3 offline_inference/audio_language.py --seed 0
481 + - python3 offline_inference/vision_language.py --seed 0
482 + - python3 offline_inference/vision_language_multi_image.py --seed 0
483 + - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
484 + - python3 pooling/embed/vision_embedding_offline.py --seed 0
485 + - python3 offline_inference/prefix_caching.py
486 + - python3 offline_inference/llm_engine_example.py
487 + - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
488 + - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
489 + - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
490 +
491 +- label: Platform Tests (CUDA) # 4min
492 + timeout_in_minutes: 15
493 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
494 + agent_pool: mi250_1
495 + source_file_dependencies:
496 + - vllm/
497 + - tests/cuda
498 + commands:
499 + - pytest -v -s cuda/test_cuda_context.py
500 + - pytest -v -s cuda/test_platform_no_cuda_init.py
501 +
502 +- label: Samplers Test # 56min
503 + timeout_in_minutes: 75
504 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
505 + agent_pool: mi250_1
506 + optional: true
507 + source_file_dependencies:
508 + - vllm/model_executor/layers
509 + - vllm/sampling_metadata.py
510 + - tests/samplers
511 + - tests/conftest.py
512 + commands:
513 + - pytest -v -s samplers
514 +
515 +- label: LoRA Test %N # 20min
516 + timeout_in_minutes: 30
517 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
518 + agent_pool: mi250_1
519 + optional: true
520 + parallelism: 4
521 + source_file_dependencies:
522 + - vllm/lora
523 + - tests/lora
524 + commands:
525 + - pytest -v -s lora \
526 + --shard-id=$$BUILDKITE_PARALLEL_JOB \
527 + --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \
528 + --ignore=lora/test_chatglm3_tp.py \
529 + --ignore=lora/test_llama_tp.py \
530 + --ignore=lora/test_llm_with_multi_loras.py \
531 + --ignore=lora/test_olmoe_tp.py \
532 + --ignore=lora/test_deepseekv2_tp.py \
533 + --ignore=lora/test_gptoss_tp.py \
534 + --ignore=lora/test_qwen3moe_tp.py
535 +
536 +- label: PyTorch Compilation Unit Tests # 15min
537 + timeout_in_minutes: 30
538 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
539 + agent_pool: mi250_1
540 + torch_nightly: true
541 + source_file_dependencies:
542 + - vllm/
543 + - tests/compile
544 + commands:
545 + - "find compile/ -maxdepth 1 -name 'test_*.py' -exec pytest -s -v {} \\\\;"
546 +
547 +- label: PyTorch Compilation Passes Unit Tests
548 + timeout_in_minutes: 20
549 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
550 + agent_pool: mi250_1
551 + source_file_dependencies:
552 + - vllm/
553 + - tests/compile/passes
554 + commands:
555 + - "find compile/passes -maxdepth 1 -name 'test_*.py' -exec pytest -s -v {} \\\\;"
556 +
557 +- label: PyTorch Fullgraph Smoke Test # 15min
558 + timeout_in_minutes: 30
559 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
560 + agent_pool: mi250_1
561 + torch_nightly: true
562 + source_file_dependencies:
563 + - vllm/
564 + - tests/compile
565 + commands:
566 + - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -exec pytest -s -v {} \\\\;"
567 +
568 +- label: PyTorch Fullgraph Test # 27min
569 + timeout_in_minutes: 40
570 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
571 + agent_pool: mi250_1
572 + torch_nightly: true
573 + source_file_dependencies:
574 + - vllm/
575 + - tests/compile
576 + commands:
577 + - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'
578 +
579 +- label: Cudagraph test # 15min
580 + timeout_in_minutes: 20
581 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
582 + agent_pool: mi250_1
583 + source_file_dependencies:
584 + - tests/v1/cudagraph
585 + - vllm/v1/cudagraph_dispatcher.py
586 + - vllm/config/compilation.py
587 + - vllm/compilation
588 + commands:
589 + - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
590 + - pytest -v -s v1/cudagraph/test_cudagraph_mode.py
591 +
592 +- label: Kernels Core Operation Test # 48min
593 + timeout_in_minutes: 75
594 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
595 + agent_pool: mi250_1
596 + source_file_dependencies:
597 + - csrc/
598 + - tests/kernels/core
599 + - tests/kernels/test_top_k_per_row.py
600 + commands:
601 + - pytest -v -s kernels/core kernels/test_top_k_per_row.py
602 +
603 +- label: Kernels Attention Test %N # 23min
604 + timeout_in_minutes: 35
605 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
606 + agent_pool: mi250_1
607 + parallelism: 2
608 + source_file_dependencies:
609 + - csrc/attention/
610 + - vllm/v1/attention
611 + - vllm/model_executor/layers/attention
612 + - tests/kernels/attention
613 + commands:
614 + - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
615 +
616 +- label: Kernels Quantization Test %N # 64min
617 + timeout_in_minutes: 90
618 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
619 + agent_pool: mi250_1
620 + optional: true
621 + parallelism: 2
622 + source_file_dependencies:
623 + - csrc/quantization/
624 + - vllm/model_executor/layers/quantization
625 + - tests/kernels/quantization
626 + commands:
627 + - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
628 +
629 +- label: Kernels MoE Test %N # 40min
630 + timeout_in_minutes: 60
631 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
632 + agent_pool: mi250_1
633 + parallelism: 2
634 + source_file_dependencies:
635 + - csrc/quantization/cutlass_w8a8/moe/
636 + - csrc/moe/
637 + - tests/kernels/moe
638 + - vllm/model_executor/layers/fused_moe/
639 + - vllm/distributed/device_communicators/
640 + - vllm/envs.py
641 + - vllm/config
642 + commands:
643 + - pytest -v -s kernels/moe --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
644 +
645 +- label: Kernels Mamba Test # 31min
646 + timeout_in_minutes: 45
647 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
648 + agent_pool: mi250_1
649 + source_file_dependencies:
650 + - csrc/mamba/
651 + - tests/kernels/mamba
652 + - vllm/model_executor/layers/mamba/ops
653 + commands:
654 + - pytest -v -s kernels/mamba
655 +
656 +- label: Kernels Helion Test # 20min
657 + timeout_in_minutes: 30
658 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
659 + agent_pool: mi250_1
660 + source_file_dependencies:
661 + - vllm/utils/import_utils.py
662 + - tests/kernels/helion/
663 + commands:
664 + - pip install helion
665 + - pytest -v -s kernels/helion/
666 +
667 +- label: Model Executor Test # 23min
668 + timeout_in_minutes: 35
669 + torch_nightly: true
670 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
671 + agent_pool: mi250_1
672 + source_file_dependencies:
673 + - vllm/engine/arg_utils.py
674 + - vllm/config/model.py
675 + - vllm/model_executor
676 + - tests/model_executor
677 + - tests/entrypoints/openai/test_tensorizer_entrypoint.py
678 + commands:
679 + - apt-get update && apt-get install -y curl libsodium23
680 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
681 + - pytest -v -s model_executor
682 + - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py
683 +
684 +- label: Benchmarks # 11min
685 + timeout_in_minutes: 20
686 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
687 + agent_pool: mi250_1
688 + working_dir: "/vllm-workspace/.buildkite"
689 + source_file_dependencies:
690 + - benchmarks/
691 + commands:
692 + - bash scripts/run-benchmarks.sh
693 +
694 +- label: Benchmarks CLI Test # 7min
695 + timeout_in_minutes: 20
696 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
697 + agent_pool: mi250_1
698 + source_file_dependencies:
699 + - vllm/
700 + - tests/benchmarks/
701 + commands:
702 + - pytest -v -s benchmarks/
703 +
704 +- label: Quantization Test # 70min
705 + timeout_in_minutes: 90
706 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
707 + agent_pool: mi250_1
708 + source_file_dependencies:
709 + - csrc/
710 + - vllm/model_executor/layers/quantization
711 + - tests/quantization
712 + commands:
713 + - uv pip install --system torchao==0.14.1
714 + - uv pip install --system conch-triton-kernels
715 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
716 +
717 +- label: LM Eval Small Models # 53min
718 + timeout_in_minutes: 75
719 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
720 + agent_pool: mi250_1
721 + optional: true
722 + source_file_dependencies:
723 + - csrc/
724 + - vllm/model_executor/layers/quantization
725 + autorun_on_main: true
726 + commands:
727 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
728 +
729 +- label: OpenAI API correctness # 10min
730 + timeout_in_minutes: 15
731 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
732 + agent_pool: mi250_1
733 + source_file_dependencies:
734 + - csrc/
735 + - vllm/entrypoints/openai/
736 + - vllm/model_executor/models/whisper.py
737 + - tools/
738 + commands:
739 + - bash ../tools/install_torchcodec_rocm.sh || exit 1
740 + - pytest -s entrypoints/openai/correctness/
741 +
742 +- label: Basic Models Tests (Initialization) # 15min
743 + timeout_in_minutes: 45
744 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
745 + agent_pool: mi250_1
746 + torch_nightly: true
747 + source_file_dependencies:
748 + - vllm/
749 + - tests/models/test_initialization.py
750 + commands:
751 + - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
752 +
753 +- label: Basic Models Tests (Extra Initialization) %N # 15min
754 + timeout_in_minutes: 45
755 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
756 + agent_pool: mi250_1
757 + torch_nightly: true
758 + parallelism: 2
759 + source_file_dependencies:
760 + - vllm/model_executor/models/
761 + - vllm/transformers_utils/
762 + - tests/models/test_initialization.py
763 + commands:
764 + - pytest -v -s models/test_initialization.py \
765 + -k 'not test_can_initialize_small_subset' \
766 + --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \
767 + --shard-id=$$BUILDKITE_PARALLEL_JOB
768 +
769 +- label: Basic Models Tests (Other) # 15min
770 + timeout_in_minutes: 45
771 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
772 + agent_pool: mi250_1
773 + optional: true
774 + torch_nightly: true
775 + source_file_dependencies:
776 + - vllm/
777 + - tests/models/test_terratorch.py
778 + - tests/models/test_transformers.py
779 + - tests/models/test_registry.py
780 + commands:
781 + - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
782 +
783 +- label: Basic Models Test (Other CPU) # 5min
784 + timeout_in_minutes: 10
785 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
786 + agent_pool: mi250_1
787 + torch_nightly: true
788 + no_gpu: true
789 + source_file_dependencies:
790 + - vllm/
791 + - tests/models/test_utils.py
792 + - tests/models/test_vision.py
793 + commands:
794 + - pytest -v -s models/test_utils.py models/test_vision.py
795 +
796 +- label: Language Models Tests (Standard) # 18min
797 + timeout_in_minutes: 25
798 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
799 + agent_pool: mi250_1
800 + torch_nightly: true
801 + source_file_dependencies:
802 + - vllm/
803 + - tests/models/language
804 + commands:
805 + - pip freeze | grep -E 'torch'
806 + - pytest -v -s models/language -m 'core_model and (not slow_test)'
807 +
808 +- label: Language Models Tests (Extra Standard) %N # 27min
809 + timeout_in_minutes: 45
810 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
811 + agent_pool: mi250_1
812 + optional: true
813 + torch_nightly: true
814 + parallelism: 2
815 + source_file_dependencies:
816 + - vllm/model_executor/models/
817 + - tests/models/language/pooling/test_embedding.py
818 + - tests/models/language/generation/test_common.py
819 + - tests/models/language/pooling/test_classification.py
820 + commands:
821 + - pip freeze | grep -E 'torch'
822 + - export TORCH_NCCL_BLOCKING_WAIT=1
823 + - pytest -v -s models/language -m 'core_model and slow_test' \
824 + --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \
825 + --shard-id=$$BUILDKITE_PARALLEL_JOB
826 +
827 +- label: Language Models Tests (Hybrid) %N # 50min
828 + timeout_in_minutes: 75
829 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
830 + agent_pool: mi250_1
831 + optional: true
832 + torch_nightly: true
833 + parallelism: 2
834 + source_file_dependencies:
835 + - vllm/
836 + - tests/models/language/generation
837 + commands:
838 + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
839 + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
840 + - pytest -v -s models/language/generation \
841 + -m hybrid_model \
842 + --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \
843 + --shard-id=$$BUILDKITE_PARALLEL_JOB
844 +
845 +- label: Language Models Test (Extended Generation) # 80min
846 + timeout_in_minutes: 110
847 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
848 + agent_pool: mi250_1
849 + optional: true
850 + source_file_dependencies:
851 + - vllm/
852 + - tests/models/language/generation
853 + commands:
854 + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
855 + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
856 + - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
857 +
858 +- label: Language Models Test (PPL) # 80min
859 + timeout_in_minutes: 110
860 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
861 + agent_pool: mi250_1
862 + optional: true
863 + source_file_dependencies:
864 + - vllm/
865 + - tests/models/language/generation_ppl_test
866 + commands:
867 + - pytest -v -s models/language/generation_ppl_test
868 +
869 +- label: Language Models Test (Extended Pooling) # 36min
870 + timeout_in_minutes: 50
871 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
872 + agent_pool: mi250_1
873 + optional: true
874 + source_file_dependencies:
875 + - vllm/
876 + - tests/models/language/pooling
877 + commands:
878 + - pytest -v -s models/language/pooling -m 'not core_model'
879 +
880 +- label: Language Models Test (MTEB) # 80min
881 + timeout_in_minutes: 110
882 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
883 + agent_pool: mi250_1
884 + optional: true
885 + source_file_dependencies:
886 + - vllm/
887 + - tests/models/language/pooling_mteb_test
888 + commands:
889 + - pytest -v -s models/language/pooling_mteb_test
890 +
891 +- label: Multi-Modal Processor Test (CPU) # 15min
892 + timeout_in_minutes: 60
893 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
894 + agent_pool: mi250_1
895 + no_gpu: true
896 + source_file_dependencies:
897 + - vllm/
898 + - tests/models/multimodal
899 + - tests/models/registry.py
900 + commands:
901 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
902 + - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
903 +
904 +- label: Multi-Modal Processor Test # 44min
905 + timeout_in_minutes: 60
906 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
907 + agent_pool: mi250_1
908 + source_file_dependencies:
909 + - vllm/
910 + - tests/models/multimodal
911 + - tests/models/registry.py
912 + commands:
913 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
914 + - pytest -v -s models/multimodal/processing
915 +
916 +- label: Multi-Modal Models Test (Standard) # 60min
917 + timeout_in_minutes: 100
918 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
919 + agent_pool: mi250_1
920 + torch_nightly: true
921 + source_file_dependencies:
922 + - vllm/
923 + - tests/models/multimodal
924 + commands:
925 + - export MIOPEN_DEBUG_CONV_DIRECT=0
926 + - export MIOPEN_DEBUG_CONV_GEMM=0
927 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
928 + - pip freeze | grep -E 'torch'
929 + - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing --ignore models/multimodal/pooling/test_prithvi_mae.py
930 + - pytest -v -s models/multimodal/pooling/test_prithvi_mae.py -m core_model
931 + - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
932 +
933 +- label: Multi-Modal Accuracy Eval (Small Models) # 5min
934 + timeout_in_minutes: 10
935 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
936 + agent_pool: mi250_1
937 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
938 + source_file_dependencies:
939 + - vllm/multimodal/
940 + - vllm/inputs/
941 + - vllm/v1/core/
942 + commands:
943 + - export MIOPEN_DEBUG_CONV_DIRECT=0
944 + - export MIOPEN_DEBUG_CONV_GEMM=0
945 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt
946 +
947 +- label: Multi-Modal Models Test (Extended) 1 # 60min
948 + timeout_in_minutes: 120
949 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
950 + agent_pool: mi250_1
951 + optional: true
952 + source_file_dependencies:
953 + - vllm/
954 + - tests/models/multimodal
955 + commands:
956 + - export MIOPEN_DEBUG_CONV_DIRECT=0
957 + - export MIOPEN_DEBUG_CONV_GEMM=0
958 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
959 + - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing
960 +
961 +- label: Multi-Modal Models Test (Extended) 2 #60min
962 + timeout_in_minutes: 120
963 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
964 + agent_pool: mi250_1
965 + optional: true
966 + source_file_dependencies:
967 + - vllm/
968 + - tests/models/multimodal
969 + commands:
970 + - export MIOPEN_DEBUG_CONV_DIRECT=0
971 + - export MIOPEN_DEBUG_CONV_GEMM=0
972 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
973 + - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
974 +
975 +- label: Multi-Modal Models Test (Extended) 3 # 75min
976 + timeout_in_minutes: 150
977 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
978 + agent_pool: mi250_1
979 + optional: true
980 + source_file_dependencies:
981 + - vllm/
982 + - tests/models/multimodal
983 + commands:
984 + - export MIOPEN_DEBUG_CONV_DIRECT=0
985 + - export MIOPEN_DEBUG_CONV_GEMM=0
986 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
987 + - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
988 +
989 +- label: Quantized Models Test # 45 min
990 + timeout_in_minutes: 60
991 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
992 + agent_pool: mi250_1
993 + source_file_dependencies:
994 + - vllm/model_executor/layers/quantization
995 + - tests/models/quantization
996 + commands:
997 + - pytest -v -s models/quantization
998 +
999 +- label: Transformers Nightly Models Test # 60 min
1000 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1001 + agent_pool: mi250_1
1002 + working_dir: "/vllm-workspace/"
1003 + optional: true
1004 + commands:
1005 + - pip install --upgrade git+https://github.com/huggingface/transformers
1006 + - pytest -v -s tests/models/test_initialization.py -k 'not (Gemma3 or ModernBert or Qwen2_5_VL or Qwen2_5vl or Qwen2VL or TransformersMultiModalEmbeddingModel or TransformersMultiModalForSequenceClassification or Ultravox or Phi4Multimodal or LlavaNextVideo or MiniCPMO or Lfm2Moe or PaliGemma or RobertaForSequenceClassification or Ovis2_5 or Fuyu or DeepseekOCR or KimiVL)'
1007 + - pytest -v -s tests/models/test_transformers.py
1008 + - pytest -v -s tests/models/multimodal/test_mapping.py -k 'not (Gemma3 or Qwen2VL or Qwen2_5_VL)'
1009 + - python3 examples/offline_inference/basic/chat.py
1010 + - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
1011 +
1012 +- label: Distributed Comm Ops Test # 7min
1013 + timeout_in_minutes: 20
1014 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1015 + agent_pool: mi250_2
1016 + num_gpus: 2
1017 + working_dir: "/vllm-workspace/tests"
1018 + source_file_dependencies:
1019 + - vllm/distributed
1020 + - tests/distributed
1021 + commands:
1022 + - pytest -v -s distributed/test_comm_ops.py
1023 + - pytest -v -s distributed/test_shm_broadcast.py
1024 + - pytest -v -s distributed/test_shm_buffer.py
1025 + - pytest -v -s distributed/test_shm_storage.py
1026 +
1027 +- label: 2 Node Tests (4 GPUs in total) # 16min
1028 + timeout_in_minutes: 30
1029 + mirror_hardwares: [amdexperimental, amdproduction, amdmultinode, amdgfx90a]
1030 + agent_pool: mi250_4
1031 + optional: true
1032 + num_gpus: 2
1033 + num_nodes: 2
1034 + working_dir: "/vllm-workspace/tests"
1035 + source_file_dependencies:
1036 + - vllm/distributed/
1037 + - vllm/engine/
1038 + - vllm/executor/
1039 + - vllm/model_executor/models/
1040 + - tests/distributed/
1041 + - tests/examples/offline_inference/data_parallel.py
1042 + commands:
1043 + - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up) | grep 'Same node test passed' | grep 'Node count test passed'
1044 + - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
1045 + - NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py
1046 + - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code
1047 + - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py
1048 + - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
1049 + - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
1050 + - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
1051 + - NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py
1052 + - python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code
1053 +
1054 +- label: Distributed Tests (2 GPUs) # 68min
1055 + timeout_in_minutes: 90
1056 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1057 + agent_pool: mi250_2
1058 + optional: true
1059 + num_gpus: 2
1060 + working_dir: "/vllm-workspace/tests"
1061 + source_file_dependencies:
1062 + - vllm/compilation/
1063 + - vllm/distributed/
1064 + - vllm/engine/
1065 + - vllm/executor/
1066 + - vllm/worker/worker_base.py
1067 + - vllm/v1/engine/
1068 + - vllm/v1/worker/
1069 + - tests/compile/fullgraph/test_basic_correctness.py
1070 + - tests/compile/test_wrapper.py
1071 + - tests/distributed/
1072 + - tests/entrypoints/llm/test_collective_rpc.py
1073 + - tests/v1/distributed
1074 + - tests/v1/entrypoints/openai/test_multi_api_servers.py
1075 + - tests/v1/shutdown
1076 + - tests/v1/worker/test_worker_memory_snapshot.py
1077 + - examples/offline_inference/new_weight_syncing/
1078 + commands:
1079 + - export TORCH_NCCL_BLOCKING_WAIT=1
1080 + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
1081 + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
1082 + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
1083 + - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
1084 + - pytest -v -s entrypoints/llm/test_collective_rpc.py
1085 + - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
1086 + - pytest -v -s ./compile/test_wrapper.py
1087 + - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
1088 + - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
1089 + - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
1090 + - pytest -v -s v1/worker/test_worker_memory_snapshot.py
1091 +
1092 +- label: Distributed Model Tests (2 GPUs) # 37min
1093 + timeout_in_minutes: 50
1094 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1095 + agent_pool: mi250_2
1096 + optional: true
1097 + num_gpus: 2
1098 + working_dir: "/vllm-workspace/tests"
1099 + source_file_dependencies:
1100 + - vllm/model_executor/model_loader/sharded_state_loader.py
1101 + - vllm/model_executor/models/
1102 + - tests/basic_correctness/
1103 + - tests/model_executor/model_loader/test_sharded_state_loader.py
1104 + - tests/models/
1105 + commands:
1106 + - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
1107 + - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py
1108 + - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
1109 + - pytest models/language -v -s -m 'distributed(num_gpus=2)'
1110 + - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py
1111 + - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
1112 +
1113 +- label: Plugin Tests (2 GPUs) # 40min
1114 + timeout_in_minutes: 60
1115 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1116 + agent_pool: mi250_2
1117 + num_gpus: 2
1118 + working_dir: "/vllm-workspace/tests"
1119 + source_file_dependencies:
1120 + - vllm/plugins/
1121 + - tests/plugins/
1122 + commands:
1123 + # begin platform plugin and general plugin tests, all the code in-between runs on dummy platform
1124 + - pip install -e ./plugins/vllm_add_dummy_platform
1125 + - pytest -v -s plugins_tests/test_platform_plugins.py
1126 + - pip uninstall vllm_add_dummy_platform -y
1127 + # end platform plugin tests
1128 + # begin io_processor plugins test, all the code in between uses the prithvi_io_processor plugin
1129 + - pip install -e ./plugins/prithvi_io_processor_plugin
1130 + - pytest -v -s plugins_tests/test_io_processor_plugins.py
1131 + - pip uninstall prithvi_io_processor_plugin -y
1132 + # test bge_m3_sparse io_processor plugin
1133 + - pip install -e ./plugins/bge_m3_sparse_plugin
1134 + - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
1135 + - pip uninstall bge_m3_sparse_plugin -y
1136 + # end io_processor plugins test
1137 + # begin stat_logger plugins test
1138 + - pip install -e ./plugins/vllm_add_dummy_stat_logger
1139 + - pytest -v -s plugins_tests/test_stats_logger_plugins.py
1140 + - pip uninstall dummy_stat_logger -y
1141 + # end stat_logger plugins test
1142 + # other tests continue here:
1143 + - pytest -v -s plugins_tests/test_scheduler_plugins.py
1144 + - pip install -e ./plugins/vllm_add_dummy_model
1145 + - pytest -v -s distributed/test_distributed_oot.py
1146 + - pytest -v -s entrypoints/openai/test_oot_registration.py
1147 + - pytest -v -s models/test_oot_registration.py
1148 + - pytest -v -s plugins/lora_resolvers
1149 +
1150 +- label: Pipeline + Context Parallelism Test # 45min
1151 + timeout_in_minutes: 60
1152 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1153 + agent_pool: mi250_4
1154 + num_gpus: 4
1155 + working_dir: "/vllm-workspace/tests"
1156 + source_file_dependencies:
1157 + - vllm/distributed/
1158 + - vllm/engine/
1159 + - vllm/executor/
1160 + - vllm/model_executor/models/
1161 + - tests/distributed/
1162 + commands:
1163 + - pytest -v -s distributed/test_pp_cudagraph.py
1164 + - pytest -v -s distributed/test_pipeline_parallel.py
1165 +
1166 +- label: LoRA TP Test (Distributed) # 17 min
1167 + timeout_in_minutes: 30
1168 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1169 + agent_pool: mi250_4
1170 + num_gpus: 4
1171 + source_file_dependencies:
1172 + - vllm/lora
1173 + - tests/lora
1174 + commands:
1175 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1176 + - pytest -v -s -x lora/test_chatglm3_tp.py
1177 + - pytest -v -s -x lora/test_llama_tp.py
1178 + - pytest -v -s -x lora/test_llm_with_multi_loras.py
1179 + - pytest -v -s -x lora/test_olmoe_tp.py
1180 +
1181 +- label: Weight Loading Multiple GPU Test # 33min
1182 + timeout_in_minutes: 45
1183 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1184 + agent_pool: mi250_2
1185 + num_gpus: 2
1186 + optional: true
1187 + working_dir: "/vllm-workspace/tests"
1188 + source_file_dependencies:
1189 + - vllm/
1190 + - tests/weight_loading
1191 + commands:
1192 + - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
1193 +
1194 +- label: Weight Loading Multiple GPU Test - Large Models # optional
1195 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1196 + agent_pool: mi250_2
1197 + num_gpus: 2
1198 + optional: true
1199 + working_dir: "/vllm-workspace/tests"
1200 + source_file_dependencies:
1201 + - vllm/
1202 + - tests/weight_loading
1203 + commands:
1204 + - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt
1205 +
1206 +- label: NixlConnector PD accuracy tests (Distributed) # 30min
1207 + timeout_in_minutes: 30
1208 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1209 + agent_pool: mi250_4
1210 + num_gpus: 4
1211 + working_dir: "/vllm-workspace/tests"
1212 + source_file_dependencies:
1213 + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
1214 + - tests/v1/kv_connector/nixl_integration/
1215 + commands:
1216 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
1217 + - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
1218 +
1219 +- label: DP EP NixlConnector PD accuracy tests (Distributed) # 15min
1220 + timeout_in_minutes: 15
1221 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1222 + agent_pool: mi250_4
1223 + num_gpus: 4
1224 + working_dir: "/vllm-workspace/tests"
1225 + source_file_dependencies:
1226 + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
1227 + - tests/v1/kv_connector/nixl_integration/
1228 + commands:
1229 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
1230 + - DP_EP=1 ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
1231 +
1232 +- label: Distributed Tests (A100) # 68min
1233 + timeout_in_minutes: 90
1234 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1235 + agent_pool: mi250_4
1236 + optional: true
1237 + num_gpus: 4
1238 + source_file_dependencies:
1239 + - vllm/
1240 + commands:
1241 + - export TORCH_NCCL_BLOCKING_WAIT=1
1242 + - pytest -v -s distributed/test_custom_all_reduce.py
1243 + - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
1244 + - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
1245 + - pytest -v -s -x lora/test_mixtral.py
1246 +
1247 +- label: LM Eval Large Models # 80min
1248 + timeout_in_minutes: 110
1249 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1250 + agent_pool: mi250_4
1251 + optional: true
1252 + num_gpus: 4
1253 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1254 + source_file_dependencies:
1255 + - csrc/
1256 + - vllm/model_executor/layers/quantization
1257 + commands:
1258 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1259 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
1260 +
1261 +- label: LM Eval Large Models (H100) # 80min
1262 + timeout_in_minutes: 110
1263 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1264 + agent_pool: mi250_4
1265 + optional: true
1266 + num_gpus: 4
1267 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1268 + source_file_dependencies:
1269 + - csrc/
1270 + - vllm/model_executor/layers/quantization
1271 + commands:
1272 + - export VLLM_USE_DEEP_GEMM=0
1273 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=4
1274 +
1275 +- label: Distributed Tests (H200) # 68min
1276 + timeout_in_minutes: 90
1277 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1278 + agent_pool: mi250_2
1279 + optional: true
1280 + num_gpus: 2
1281 + working_dir: "/vllm-workspace/"
1282 + commands:
1283 + - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
1284 + - pytest -v -s tests/compile/passes/distributed/test_sequence_parallelism.py
1285 + - pytest -v -s tests/distributed/test_context_parallel.py
1286 + - HIP_VISIBLE_DEVICES=0,1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
1287 +
1288 +- label: LM Eval Small Models (1 Card) # 15min
1289 + timeout_in_minutes: 20
1290 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1291 + agent_pool: mi250_1
1292 + source_file_dependencies:
1293 + - csrc/
1294 + - vllm/model_executor/layers/quantization
1295 + commands:
1296 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
1297 +
1298 +- label: LM Eval Large Models (4 Card) # 80min
1299 + timeout_in_minutes: 110
1300 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1301 + agent_pool: mi250_4
1302 + optional: true
1303 + num_gpus: 4
1304 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1305 + source_file_dependencies:
1306 + - csrc/
1307 + - vllm/model_executor/layers/quantization
1308 + commands:
1309 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1310 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
1311 +
1312 +- label: ROCm LM Eval Large Models (8 Card) # 80min
1313 + timeout_in_minutes: 110
1314 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1315 + agent_pool: mi250_8
1316 + num_gpus: 8
1317 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1318 + commands:
1319 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1320 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
1321 +
1322 +- label: ROCm GPT-OSS Eval # 80min
1323 + timeout_in_minutes: 60
1324 + working_dir: "/vllm-workspace/"
1325 + agent_pool: mi250_1
1326 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1327 + optional: true
1328 + source_file_dependencies:
1329 + - tests/evals/gpt_oss
1330 + - vllm/model_executor/models/gpt_oss.py
1331 + - vllm/model_executor/layers/quantization/mxfp4.py
1332 + - vllm/v1/attention/backends/flashinfer.py
1333 + commands:
1334 + - uv pip install --system 'gpt-oss[eval]==0.0.5'
1335 + - VLLM_ROCM_USE_AITER_MHA=0 VLLM_ROCM_USE_AITER=1 VLLM_USE_AITER_UNIFIED_ATTENTION=1 pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58
1336 +
1337 +- label: DeepSeek V2-Lite Accuracy # 70min
1338 + timeout_in_minutes: 60
1339 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1340 + agent_pool: mi250_4
1341 + optional: true
1342 + num_gpus: 4
1343 + working_dir: "/vllm-workspace"
1344 + commands:
1345 + - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
1346 +
1347 +- label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy # 70min
1348 + timeout_in_minutes: 60
1349 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
1350 + agent_pool: mi250_4
1351 + optional: true
1352 + num_gpus: 4
1353 + working_dir: "/vllm-workspace"
1354 + commands:
1355 + - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
1356 +
1357 +
1358 +###################################################
1359 +# #
1360 +# MI325 test definitions #
1361 +# #
1362 +###################################################
1363 +
1364 +
1365 ##### fast check tests #####
1366
1367 - label: Pytorch Nightly Dependency Override Check # 2min