@tduncan / Nvidia-Optimization-Stack-VLLM / commits / f1c2f6ada8

[ROCm][CI] Stage H gating and MI355 test reallocation (#57080)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>

Andreas Karatzas committed Sep 17, 2026 at 03:44 UTC f1c2f6ada80f7c0d896b7870f53db97103ff339d
23 files changed +1719 -570
.buildkite/test-amd.yaml
+1208 -506
@@ -124,8 +124,9 @@ steps:
124 - tests/models/test_vision.py
125 - tests/models/test_adapters.py
126 - tests/models/transformers/fusers/
127 + - tests/models/test_qwen3_5_mtp_config.py
128 commands:
128 - - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/transformers/fusers/
129 + - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/
130
131 - label: ":computer: (CPU) Multimodal Processor Shard %N" # TBD
132 timeout_in_minutes: 90
@@ -214,7 +215,7 @@ steps:
215 - .buildkite/test_areas/rust_frontend_cargo.yaml
216 - .buildkite/scripts/run-rust-frontend-cargo-ci.sh
217 commands:
217 - - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy
218 + - .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy
219
220 - label: ":computer: (CPU) Rust Frontend Cargo" # TBD
221 timeout_in_minutes: 35
@@ -229,7 +230,7 @@ steps:
230 - .buildkite/test_areas/rust_frontend_cargo.yaml
231 - .buildkite/scripts/run-rust-frontend-cargo-ci.sh
232 commands:
232 - - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh test
233 + - .buildkite/scripts/run-rust-frontend-cargo-ci.sh test
234
235 - label: ":computer: (CPU) Docker Build Metadata" # TBD
236 timeout_in_minutes: 35
@@ -281,7 +282,7 @@ steps:
282 #-------------------------------------------------------- mi250 · distributed --------------------------------------------------------#
283
284 - label: ":amd: (MI250) Pipeline + Context Parallelism" # TBD
284 - timeout_in_minutes: 80
285 + timeout_in_minutes: 90
286 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
287 agent_pool: mi250_4
288 num_gpus: 4
@@ -297,14 +298,18 @@ steps:
298 - tests/distributed/
299 - vllm/_aiter_ops.py
300 - vllm/platforms/rocm.py
301 + - tests/v1/distributed/test_pp_dp_v2.py
302 + optional: true
303 commands:
304 - pytest -v -s distributed/test_pp_cudagraph.py
305 - pytest -v -s distributed/test_pipeline_parallel.py
306 + - pytest -v -s v1/distributed/test_pp_dp_v2.py
307
308 #---------------------------------------------------------- mi250 · kernels ----------------------------------------------------------#
309
306 -- label: ":amd: (MI250) Helion Kernels" # TBD
307 - timeout_in_minutes: 40
310 +- label: ":amd: (MI250) Helion Kernels Shard %N"
311 + timeout_in_minutes: 45
312 + parallelism: 2
313 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
314 agent_pool: mi250_1
315 optional: true
@@ -316,25 +321,43 @@ steps:
321 - vllm/platforms/rocm.py
322 commands:
323 - pip install helion==1.4.0
319 - - pytest -v -s kernels/helion/
324 + - pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
325
326 #----------------------------------------------------- mi250 · models / language -----------------------------------------------------#
327
328 - label: ":amd: (MI250) Language Models (PPL)" # TBD
324 - timeout_in_minutes: 45
329 + timeout_in_minutes: 65
330 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
331 agent_pool: mi250_1
332 working_dir: "/vllm-workspace/tests"
333 source_file_dependencies:
334 - vllm/
335 - tests/models/language/generation_ppl_test
336 + - vllm/model_executor/models/qwen3_5.py
337 + - vllm/model_executor/models/qwen3_5_mtp.py
338 + - vllm/transformers_utils/configs/qwen3_5.py
339 + - vllm/transformers_utils/configs/qwen3_5_moe.py
340 + - vllm/model_executor/models/qwen2.py
341 + - vllm/model_executor/models/qwen3.py
342 + - vllm/model_executor/models/qwen3_next.py
343 + - vllm/model_executor/models/qwen3_next_mtp.py
344 + - vllm/model_executor/layers/fla/ops/
345 + - vllm/_aiter_ops.py
346 + - vllm/v1/attention/backends/triton_attn.py
347 + - vllm/v1/attention/backends/rocm_attn.py
348 + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
349 + - vllm/v1/attention/backends/rocm_aiter_fa.py
350 + - vllm/v1/attention/backends/flex_attention.py
351 + - vllm/v1/attention/ops/
352 + - vllm/platforms/rocm.py
353 + optional: true
354 commands:
355 - pytest -v -s models/language/generation_ppl_test
356
357 #---------------------------------------------------- mi250 · models / multimodal ----------------------------------------------------#
358
359 - label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD
337 - timeout_in_minutes: 45
360 + timeout_in_minutes: 55
361 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
362 agent_pool: mi250_1
363 optional: true
@@ -367,7 +390,7 @@ steps:
390 - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
391
392 - label: ":amd: (MI250) CUDAGraph" # TBD
370 - timeout_in_minutes: 45
393 + timeout_in_minutes: 55
394 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
395 agent_pool: mi250_1
396 optional: true
@@ -378,9 +401,13 @@ steps:
401 - vllm/config/compilation.py
402 - vllm/compilation
403 - vllm/platforms/rocm.py
404 + - vllm/v1/worker/encoder_cudagraph.py
405 + - vllm/v1/worker/encoder_cudagraph_defs.py
406 commands:
407 - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
408 - pytest -v -s v1/cudagraph/test_cudagraph_mode.py
409 + - pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
410 + - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
411
412 - label: ":amd: (MI250) E2E Core" # TBD
413 timeout_in_minutes: 50
@@ -423,6 +450,7 @@ steps:
450 - vllm/engine/arg_utils.py
451 - vllm/transformers_utils/config.py
452 - vllm/platforms/rocm.py
453 + - vllm/v1/engine/
454 commands:
455 - pytest -v -s v1/engine/test_preprocess_error_handling.py
456 - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
@@ -447,7 +475,7 @@ steps:
475 - pytest -v -s v1/e2e/spec_decode/draft_model/
476
477 - label: ":amd: (MI250) Spec Decode Speculators + MTP" # TBD
450 - timeout_in_minutes: 65
478 + timeout_in_minutes: 75
479 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
480 agent_pool: mi250_1
481 optional: true
@@ -461,6 +489,7 @@ steps:
489 - vllm/transformers_utils/configs/speculators/
490 - tests/v1/e2e/spec_decode/
491 - vllm/platforms/rocm.py
492 + - vllm/v1/attention/backends/
493 commands:
494 - pytest -v -s v1/e2e/spec_decode/speculators/
495 - pytest -v -s v1/e2e/spec_decode/mtp/
@@ -468,7 +497,7 @@ steps:
497 #------------------------------------------------------------- mi250 · misc ------------------------------------------------------------#
498
499 - label: ":amd: (MI300) Python-only Installation" # TBD
471 - timeout_in_minutes: 25
500 + timeout_in_minutes: 55
501 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
502 dind: false
503 agent_pool: mi300_1
@@ -501,14 +530,16 @@ steps:
530 - tests/basic_correctness/test_basic_correctness
531 - tests/basic_correctness/test_cpu_offload
532 - tests/basic_correctness/test_mem.py
533 + optional: true
534 commands:
535 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
536 - pytest -v -s basic_correctness/test_mem.py
537 - VLLM_TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py
538 - pytest -v -s basic_correctness/test_cpu_offload.py
539
510 -- label: ":amd: (MI300) Distributed Models" # TBD
540 +- label: ":amd: (MI300) Distributed Models Shard %N"
541 timeout_in_minutes: 80
542 + parallelism: 3
543 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
544 dind: false
545 agent_pool: mi300_2
@@ -531,21 +562,17 @@ steps:
562 - tests/basic_correctness/
563 - tests/model_executor/model_loader/test_sharded_state_loader.py
564 - tests/models/
565 + - vllm/v1/worker/
566 + - tests/conftest.py
567 + - tests/utils.py
568 + - tests/models/test_vision.py
569 commands:
535 - - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
536 - - HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
537 - - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
538 - - pytest models/language -v -s -m 'distributed(num_gpus=2)'
539 - - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
540 - - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
541 - - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
542 - - pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)'
543 - - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)'
570 + - case "$$BUILDKITE_PARALLEL_JOB" in 0) TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' && HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' ;; 1) pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' && pytest models/language -v -s -m 'distributed(num_gpus=2)' && VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py && pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' && pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; esac
571
572 #-------------------------------------------------------- mi300 · benchmarks ---------------------------------------------------------#
573
574 - label: ":amd: (MI300) Benchmarks CLI" # TBD
548 - timeout_in_minutes: 35
575 + timeout_in_minutes: 40
576 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
577 dind: false
578 agent_pool: mi300_1
@@ -554,13 +581,14 @@ steps:
581 source_file_dependencies:
582 - vllm/
583 - tests/benchmarks/
584 + - rust/src/bench/tests/python_serve_flags.txt
585 commands:
586 - pytest -v -s benchmarks/
587
588 #---------------------------------------------------------- mi300 · compile ----------------------------------------------------------#
589
590 - label: ":amd: (MI300) PyTorch Compilation" # TBD
563 - timeout_in_minutes: 45
591 + timeout_in_minutes: 90
592 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
593 dind: false
594 agent_pool: mi300_1
@@ -576,6 +604,29 @@ steps:
604 - csrc/
605 - tests/compile
606 - vllm/platforms/rocm.py
607 + - vllm/__init__.py
608 + - vllm/_aiter_ops.py
609 + - vllm/_custom_ops.py
610 + - vllm/config/
611 + - vllm/distributed/
612 + - vllm/engine/
613 + - vllm/env_override.py
614 + - vllm/envs.py
615 + - vllm/forward_context.py
616 + - vllm/inputs/
617 + - vllm/ir/
618 + - vllm/kernels/
619 + - vllm/logger.py
620 + - vllm/model_executor/
621 + - vllm/multimodal/
622 + - vllm/platforms/
623 + - vllm/plugins/
624 + - vllm/sampling_params.py
625 + - vllm/sequence.py
626 + - vllm/transformers_utils/
627 + - vllm/triton_utils/
628 + - vllm/utils/
629 + - vllm/v1/
630 commands:
631 - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
632
@@ -596,6 +647,7 @@ steps:
647 - tests/compile/fusions_e2e/
648 - vllm/_aiter_ops.py
649 - vllm/platforms/rocm.py
650 + optional: true
651 commands:
652 - rocm-smi
653 - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3"
@@ -615,6 +667,7 @@ steps:
667 - tests/compile/fusions_e2e/
668 - vllm/_aiter_ops.py
669 - vllm/platforms/rocm.py
670 + optional: true
671 commands:
672 - rocm-smi
673 # Run all models and attn backends but only Inductor partition and native custom ops
@@ -631,11 +684,37 @@ steps:
684 source_file_dependencies:
685 - vllm/
686 - tests/compile/passes
687 + - vllm/platforms/rocm.py
688 + - vllm/__init__.py
689 + - vllm/_aiter_ops.py
690 + - vllm/_custom_ops.py
691 + - vllm/compilation/
692 + - vllm/config/
693 + - vllm/distributed/
694 + - vllm/engine/
695 + - vllm/env_override.py
696 + - vllm/envs.py
697 + - vllm/forward_context.py
698 + - vllm/inputs/
699 + - vllm/ir/
700 + - vllm/kernels/
701 + - vllm/logger.py
702 + - vllm/model_executor/
703 + - vllm/multimodal/
704 + - vllm/platforms/
705 + - vllm/plugins/
706 + - vllm/sampling_params.py
707 + - vllm/sequence.py
708 + - vllm/transformers_utils/
709 + - vllm/triton_utils/
710 + - vllm/utils/
711 + - vllm/v1/
712 + optional: true
713 commands:
714 - pytest -s -v compile/passes --ignore compile/passes/distributed
715
716 - label: ":amd: (MI300) PyTorch Fullgraph" # TBD
638 - timeout_in_minutes: 35
717 + timeout_in_minutes: 90
718 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
719 dind: false
720 agent_pool: mi300_1
@@ -649,11 +728,33 @@ steps:
728 - csrc/
729 - tests/compile
730 - vllm/platforms/rocm.py
731 + - vllm/__init__.py
732 + - vllm/_aiter_ops.py
733 + - vllm/_custom_ops.py
734 + - vllm/config/
735 + - vllm/distributed/
736 + - vllm/engine/
737 + - vllm/env_override.py
738 + - vllm/envs.py
739 + - vllm/forward_context.py
740 + - vllm/inputs/
741 + - vllm/ir/
742 + - vllm/kernels/
743 + - vllm/logger.py
744 + - vllm/multimodal/
745 + - vllm/platforms/
746 + - vllm/plugins/
747 + - vllm/sampling_params.py
748 + - vllm/sequence.py
749 + - vllm/transformers_utils/
750 + - vllm/triton_utils/
751 + - vllm/utils/
752 + - vllm/v1/
753 commands:
653 - - pytest -v -s compile/fullgraph/test_full_graph.py
754 + - find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'
755
756 - label: ":amd: (MI300) PyTorch Nightly Dependency Override Check" # TBD
656 - timeout_in_minutes: 25
757 + timeout_in_minutes: 30
758 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
759 dind: false
760 agent_pool: mi300_1
@@ -704,7 +805,7 @@ steps:
805 - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "dynamo_partition"
806
807 - label: ":amd: (MI300) Distributed Compile + RPC" # TBD
707 - timeout_in_minutes: 45
808 + timeout_in_minutes: 65
809 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
810 dind: false
811 agent_pool: mi300_2
@@ -722,9 +823,9 @@ steps:
823 - tests/compile/test_wrapper.py
824 - tests/entrypoints/llm/test_collective_rpc.py
825 - vllm/platforms/rocm.py
826 + optional: true
827 commands:
828 - pytest -v -s entrypoints/llm/test_collective_rpc.py
727 - - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
829 - pytest -v -s ./compile/test_wrapper.py
830
831 #----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------#
@@ -738,9 +839,16 @@ steps:
839 source_file_dependencies:
840 - vllm/
841 - tests/cuda
842 + - vllm/envs.py
843 + - vllm/logger.py
844 + - vllm/platforms/
845 + - vllm/plugins/
846 + - vllm/utils/
847 + optional: true
848 commands:
849 - pytest -v -s cuda/test_cuda_context.py
850 - pytest -v -s cuda/test_platform_no_cuda_init.py
851 + - pytest -v -s cuda/test_cuda_compatibility_path.py
852
853 #-------------------------------------------------------- mi300 · detokenizer --------------------------------------------------------#
854
@@ -756,6 +864,19 @@ steps:
864 - tests/detokenizer
865 - tests/multimodal
866 - tests/utils_
867 + - vllm/assets/
868 + - vllm/config/
869 + - vllm/distributed/
870 + - vllm/engine/
871 + - vllm/inputs/
872 + - vllm/model_executor/
873 + - vllm/multimodal/
874 + - vllm/platforms/
875 + - vllm/sampling_params.py
876 + - vllm/tokenizers/
877 + - vllm/transformers_utils/
878 + - vllm/utils/
879 + - vllm/v1/
880 commands:
881 - pytest -v -s detokenizer
882 - pytest -v -s -m 'not cpu_test' multimodal
@@ -782,7 +903,7 @@ steps:
903 - pytest -v -s distributed/test_shm_storage.py
904
905 - label: ":amd: (MI300) EPLB Algorithm" # TBD
785 - timeout_in_minutes: 25
906 + timeout_in_minutes: 30
907 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
908 dind: false
909 agent_pool: mi300_1
@@ -814,7 +935,7 @@ steps:
935 - pytest -v -s distributed/test_eplb_spec_decode.py
936
937 - label: ":amd: (MI300) Distributed Features" # TBD
817 - timeout_in_minutes: 45
938 + timeout_in_minutes: 90
939 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
940 dind: false
941 agent_pool: mi300_2
@@ -831,6 +952,26 @@ steps:
952 - examples/features/data_parallel/data_parallel_offline.py
953 - vllm/_aiter_ops.py
954 - vllm/platforms/rocm.py
955 + - csrc/custom_quickreduce.cu
956 + - csrc/ops.h
957 + - csrc/torch_bindings.cpp
958 + - vllm/model_executor/layers/
959 + - vllm/entrypoints/llm.py
960 + - vllm/config/parallel.py
961 + - vllm/v1/engine/
962 + - vllm/v1/executor/
963 + - vllm/v1/worker/
964 + - vllm/_custom_ops.py
965 + - vllm/envs.py
966 + - tests/distributed/test_rocm_aiter_custom_ar.py
967 + - tests/distributed/test_rocm_quick_reduce.py
968 + - tests/distributed/test_quick_all_reduce.py
969 + - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
970 + - tests/utils.py
971 + - examples/rl/rlhf_async_new_apis.py
972 + - tests/distributed/test_weight_transfer.py
973 + - tests/distributed/test_packed_tensor.py
974 + optional: true
975 commands:
976 - pytest -v -s tests/distributed/test_context_parallel.py
977 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
@@ -839,6 +980,10 @@ steps:
980 - pytest -v -s tests/v1/distributed/test_dbo.py
981 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
982 - pytest -v -s tests/distributed/test_packed_tensor.py
983 + - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py
984 + - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
985 + - pytest -v -s tests/distributed/test_rocm_quick_reduce.py
986 + - pytest -v -s tests/distributed/test_quick_all_reduce.py
987
988 - label: ":amd: (MI300) Distributed" # TBD
989 timeout_in_minutes: 40
@@ -993,7 +1138,7 @@ steps:
1138 #---------------------------------------------------------- mi300 · engine -----------------------------------------------------------#
1139
1140 - label: ":amd: (MI300) Engine Core" # TBD
996 - timeout_in_minutes: 35
1141 + timeout_in_minutes: 40
1142 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1143 dind: false
1144 agent_pool: mi300_1
@@ -1008,6 +1153,17 @@ steps:
1153 - tests/test_vllm_port
1154 - tests/jit_monitor/test_hooks.py
1155 - tests/jit_monitor/test_hooks_gpu.py
1156 + - vllm/compilation/
1157 + - vllm/config/
1158 + - vllm/engine/
1159 + - vllm/entrypoints/logger.py
1160 + - vllm/envs.py
1161 + - vllm/logger.py
1162 + - vllm/logging_utils/
1163 + - vllm/platforms/
1164 + - vllm/sequence.py
1165 + - vllm/triton_utils/
1166 + - vllm/utils/
1167 commands:
1168 - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py jit_monitor/test_hooks_gpu.py
1169
@@ -1049,8 +1205,9 @@ steps:
1205 - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
1206 - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
1207
1052 -- label: ":amd: (MI300) Entrypoints Integration (API Server)" # TBD
1208 +- label: ":amd: (MI300) Entrypoints Integration (API Server) %N"
1209 timeout_in_minutes: 65
1210 + parallelism: 4
1211 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1212 dind: false
1213 agent_pool: mi300_1
@@ -1063,12 +1220,12 @@ steps:
1220 - tests/entrypoints/scale_out
1221 commands:
1222 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1066 - - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
1067 - - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
1068 - - pytest -v -s entrypoints/scale_out
1223 + - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
1224 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi
1225 + - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
1226
1227 - label: ":amd: (MI300) Entrypoints Integration (OpenAI API completion)" # TBD
1071 - timeout_in_minutes: 40
1228 + timeout_in_minutes: 65
1229 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1230 dind: false
1231 agent_pool: mi300_1
@@ -1081,10 +1238,10 @@ steps:
1238 commands:
1239 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1240 - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
1084 - - pytest -v -s entrypoints/openai/ --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
1241 + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
1242
1243 - label: ":amd: (MI300) Entrypoints Integration (OpenAI API chat_completion)" # TBD
1087 - timeout_in_minutes: 55
1244 + timeout_in_minutes: 70
1245 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1246 dind: false
1247 agent_pool: mi300_1
@@ -1114,7 +1271,6 @@ steps:
1271 - tests/entrypoints/cohere
1272 - tests/entrypoints/generate
1273 commands:
1117 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1274 - pytest -v -s tool_use
1275 - pytest -v -s entrypoints/tool_parsers
1276 - pytest -v -s entrypoints/generate
@@ -1122,7 +1278,7 @@ steps:
1278 - pytest -v -s entrypoints/cohere
1279
1280 - label: ":amd: (MI300) Entrypoints Integration (Responses API)" # TBD
1125 - timeout_in_minutes: 45
1281 + timeout_in_minutes: 50
1282 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1283 dind: false
1284 agent_pool: mi300_1
@@ -1133,7 +1289,6 @@ steps:
1289 - vllm/
1290 - tests/entrypoints/openai/responses
1291 commands:
1136 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1292 - pytest -v -s entrypoints/openai/responses
1293
1294 - label: ":amd: (MI300) Entrypoints Integration (Speech to Text)" # TBD
@@ -1146,6 +1301,7 @@ steps:
1301 source_file_dependencies:
1302 - vllm/
1303 - tests/entrypoints/speech_to_text
1304 + optional: true
1305 commands:
1306 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1307 - pytest -v -s entrypoints/speech_to_text
@@ -1181,7 +1337,7 @@ steps:
1337 - pytest -v -s entrypoints/pooling
1338
1339 - label: ":amd: (MI300) OpenAI API Correctness" # TBD
1184 - timeout_in_minutes: 25
1340 + timeout_in_minutes: 30
1341 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1342 dind: false
1343 agent_pool: mi300_1
@@ -1220,11 +1376,12 @@ steps:
1376 - .buildkite/scripts/scheduled_integration_test/
1377 - vllm/_aiter_ops.py
1378 - vllm/platforms/rocm.py
1379 + optional: true
1380 commands:
1381 - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030
1382
1383 - label: ":amd: (MI300) LM Eval Small Models" # TBD
1227 - timeout_in_minutes: 40
1384 + timeout_in_minutes: 45
1385 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1386 dind: false
1387 agent_pool: mi300_1
@@ -1238,11 +1395,12 @@ steps:
1395 - vllm/v1/attention/selector.py
1396 - vllm/_aiter_ops.py
1397 - vllm/platforms/rocm.py
1398 + optional: true
1399 commands:
1400 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
1401
1402 - label: ":amd: (MI300) MRCR Eval Small Models" # TBD
1245 - timeout_in_minutes: 35
1403 + timeout_in_minutes: 40
1404 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1405 dind: false
1406 agent_pool: mi300_1
@@ -1275,11 +1433,12 @@ steps:
1433 - vllm/v1/attention/selector.py
1434 - vllm/_aiter_ops.py
1435 - vllm/platforms/rocm.py
1436 + - .buildkite/lm-eval-harness/
1437 commands:
1438 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt
1439
1440 - label: ":amd: (MI300) Multimodal Accuracy Eval (Small Models)" # TBD
1282 - timeout_in_minutes: 30
1441 + timeout_in_minutes: 35
1442 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1443 dind: false
1444 agent_pool: mi300_1
@@ -1295,7 +1454,7 @@ steps:
1454 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
1455
1456 - label: ":amd: (MI300) GPQA Eval (GPT-OSS)" # TBD
1298 - timeout_in_minutes: 30
1457 + timeout_in_minutes: 40
1458 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1459 dind: false
1460 agent_pool: mi300_2
@@ -1317,7 +1476,7 @@ steps:
1476 - uv pip install --system 'gpt-oss[eval]==0.0.5'
1477 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
1478
1320 -- label: ":amd: (MI300) LM Eval Small Models FP8 + Mixed" # TBD
1479 +- label: ":amd: (MI300) LM Eval Small Models Distributed"
1480 timeout_in_minutes: 55
1481 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1482 dind: false
@@ -1333,6 +1492,8 @@ steps:
1492 - vllm/v1/attention/selector.py
1493 - vllm/_aiter_ops.py
1494 - vllm/platforms/rocm.py
1495 + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt
1496 + optional: true
1497 commands:
1498 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
1499
@@ -1517,7 +1678,7 @@ steps:
1678 #--------------------------------------------------------- mi300 · examples ----------------------------------------------------------#
1679
1680 - label: ":amd: (MI300) Examples" # TBD
1520 - timeout_in_minutes: 50
1681 + timeout_in_minutes: 75
1682 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1683 dind: false
1684 agent_pool: mi300_1
@@ -1529,27 +1690,27 @@ steps:
1690 - examples/
1691 - vllm/platforms/rocm.py
1692 commands:
1532 - - pip install tensorizer
1693 # Basic
1534 - - python3 basic/offline_inference/chat.py
1535 - - python3 basic/offline_inference/generate.py --model facebook/opt-125m
1536 - - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
1537 - - python3 basic/offline_inference/classify.py
1538 - - python3 basic/offline_inference/embed.py
1539 - - python3 basic/offline_inference/score.py
1694 # Multi-modal models
1541 - - python3 generate/multimodal/audio_language_offline.py --seed 0
1542 - - python3 generate/multimodal/vision_language_offline.py --seed 0
1543 - - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
1544 - - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
1695 # Pooling models
1546 - - python3 pooling/embed/vision_embedding_offline.py --seed 0
1696 # Features demo
1548 - - python3 features/automatic_prefix_caching/prefix_caching_offline.py
1549 - - python3 deployment/llm_engine_example.py
1550 - - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
1551 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
1552 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
1697 + - pip install --no-deps tensorizer
1698 + - python3 basic/offline_inference/chat.py
1699 + - python3 basic/offline_inference/generate.py --model facebook/opt-125m
1700 + - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
1701 + - python3 basic/offline_inference/classify.py
1702 + - python3 basic/offline_inference/embed.py
1703 + - python3 basic/offline_inference/score.py
1704 + - python3 generate/multimodal/audio_language_offline.py --seed 0
1705 + - python3 generate/multimodal/vision_language_offline.py --seed 0
1706 + - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
1707 + - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
1708 + - python3 pooling/embed/vision_embedding_offline.py --seed 0
1709 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py
1710 + - python3 deployment/llm_engine_example.py
1711 + - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
1712 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
1713 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
1714
1715 #---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------#
1716
@@ -1565,6 +1726,8 @@ steps:
1726 - vllm/kernels
1727 - vllm/_aiter_ops.py
1728 - vllm/platforms/rocm.py
1729 + - tests/ir/
1730 + - tests/kernels/ir/
1731 commands:
1732 - pytest -v -s tests/ir
1733 - pytest -v -s tests/kernels/ir
@@ -1584,15 +1747,16 @@ steps:
1747 - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
1748 - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
1749 - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
1750 + optional: true
1751 commands:
1588 - - rocm-smi
1752 + - amd-smi
1753 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
1754 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py
1755 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
1756 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
1757
1758 - label: ":amd: (MI300) Attention Kernels Shard %N" # TBD
1595 - timeout_in_minutes: 70
1759 + timeout_in_minutes: 90
1760 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1761 dind: false
1762 agent_pool: mi300_1
@@ -1607,27 +1771,10 @@ steps:
1771 - vllm/_aiter_ops.py
1772 - vllm/envs.py
1773 - vllm/platforms/rocm.py
1774 + - vllm/utils/flashinfer.py
1775 commands:
1776 - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
1777
1613 -- label: ":amd: (MI300) Core Operation Kernels Shard %N" # TBD
1614 - timeout_in_minutes: 40
1615 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1616 - dind: false
1617 - agent_pool: mi300_1
1618 - parallelism: 3
1619 - working_dir: "/vllm-workspace/tests"
1620 - source_file_dependencies:
1621 - - csrc/
1622 - - tests/kernels/core
1623 - - tests/kernels/test_top_k_per_row.py
1624 - - tests/kernels/test_concat_mla_q.py
1625 - - vllm/model_executor/layers/rotary_embedding/
1626 - - vllm/_aiter_ops.py
1627 - - vllm/platforms/rocm.py
1628 - commands:
1629 - - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
1630 -
1778 - label: ":amd: (MI300) MiniMax Reduce RMS Kernels" # TBD
1779 timeout_in_minutes: 45
1780 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -1643,6 +1790,12 @@ steps:
1790 - vllm/platforms/rocm.py
1791 - tests/kernels/core/test_minimax_reduce_rms.py
1792 - tests/utils.py
1793 + - csrc/libtorch_stable/minimax_reduce_rms_kernel.cu
1794 + - csrc/libtorch_stable/minimax_reduce_rms_kernel.h
1795 + - csrc/libtorch_stable/ops.h
1796 + - csrc/libtorch_stable/torch_bindings.cpp
1797 + - vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py
1798 + optional: true
1799 commands:
1800 - pytest -v -s kernels/core/test_minimax_reduce_rms.py
1801
@@ -1667,7 +1820,7 @@ steps:
1820 - pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py
1821
1822 - label: ":amd: (MI300) Mamba Kernels" # TBD
1670 - timeout_in_minutes: 50
1823 + timeout_in_minutes: 55
1824 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1825 dind: false
1826 agent_pool: mi300_1
@@ -1682,7 +1835,7 @@ steps:
1835 - pytest -v -s kernels/mamba
1836
1837 - label: ":amd: (MI300) MoE Kernels Shard %N" # TBD
1685 - timeout_in_minutes: 50
1838 + timeout_in_minutes: 55
1839 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1840 dind: false
1841 agent_pool: mi300_1
@@ -1708,8 +1861,9 @@ steps:
1861 - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
1862 - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 pytest -v -s kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
1863
1711 -- label: ":amd: (MI300) Quantization Kernels" # TBD
1712 - timeout_in_minutes: 55
1864 +- label: ":amd: (MI300) Quantization Kernels Shard %N"
1865 + timeout_in_minutes: 120
1866 + parallelism: 6
1867 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1868 dind: false
1869 agent_pool: mi300_1
@@ -1729,8 +1883,10 @@ steps:
1883 - vllm/platforms/rocm.py
1884 - vllm/model_executor/kernels/
1885 - vllm/v1/attention/backends/rocm_aiter_fa.py
1886 + - vllm/config/
1887 + - tests/kernels/quantization/test_rocm_skinny_gemms.py
1888 commands:
1733 - - pytest -v -s kernels/quantization
1889 + - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
1890
1891 - label: ":amd: (MI300) DeepEP FP8 MoE Kernels" # TBD
1892 timeout_in_minutes: 55
@@ -1754,7 +1910,7 @@ steps:
1910 #----------------------------------------------------------- mi300 · lora ------------------------------------------------------------#
1911
1912 - label: ":amd: (MI300) LoRA Shard %N" # TBD
1757 - timeout_in_minutes: 40
1913 + timeout_in_minutes: 85
1914 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1915 dind: false
1916 agent_pool: mi300_1
@@ -1768,8 +1924,9 @@ steps:
1924 commands:
1925 - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py
1926
1771 -- label: ":amd: (MI300) LoRA TP (Distributed)" # TBD
1927 +- label: ":amd: (MI300) LoRA TP (Distributed) %N"
1928 timeout_in_minutes: 65
1929 + parallelism: 4
1930 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1931 dind: false
1932 agent_pool: mi300_4
@@ -1780,14 +1937,17 @@ steps:
1937 - vllm/model_executor/layers/fused_moe/
1938 - tests/lora
1939 - vllm/platforms/rocm.py
1940 + optional: true
1941 commands:
1784 - - pytest -v -s -x lora/test_chatglm3_tp.py
1785 - - pytest -v -s -x lora/test_llama_tp.py
1786 - - pytest -v -s -x lora/test_qwen3_with_multi_loras.py
1787 - - pytest -v -s -x lora/test_olmoe_tp.py
1788 - - pytest -v -s -x lora/test_gptoss_tp.py
1789 - - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
1790 - - pytest -v -s -x lora/test_gemma4_tp.py
1942 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1943 + - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac
1944 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi
1945 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi
1946 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi
1947 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi
1948 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi
1949 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi
1950 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi
1951
1952 #------------------------------------------------------ mi300 · model_executor -------------------------------------------------------#
1953
@@ -1806,189 +1966,50 @@ steps:
1966 - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
1967 - vllm/_aiter_ops.py
1968 - vllm/platforms/rocm.py
1969 + - vllm/model_executor/warmup
1970 + - tests/model_executor/test_jit_warmup.py
1971 + - tests/model_executor/test_jit_warmup_cutedsl_launcher.py
1972 + - tests/model_executor/test_jit_warmup_triton_launcher.py
1973 commands:
1974 - apt-get update && apt-get install -y curl libsodium23
1975 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1812 - - pytest -v -s model_executor -m '(not slow_test)'
1813 - - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py
1976 + - export PYTHONFAULTHANDLER=1
1977 + - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
1978 + - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
1979
1815 -#---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------#
1980 +#------------------------------------------------------ mi300 · models / basic -------------------------------------------------------#
1981
1817 -- label: ":amd: (MI300) Model Runner V2 Core" # TBD
1818 - timeout_in_minutes: 55
1982 +- label: ":amd: (MI300) Basic Models (Initialization)" # TBD
1983 + timeout_in_minutes: 30
1984 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1985 dind: false
1986 agent_pool: mi300_1
1822 - optional: true
1987 working_dir: "/vllm-workspace/tests"
1988 source_file_dependencies:
1825 - - vllm/v1/worker/gpu/
1826 - - vllm/v1/worker/gpu_worker.py
1827 - - vllm/v1/core/sched/
1828 - - vllm/v1/attention/
1829 - - tests/v1/engine/test_llm_engine.py
1830 - - tests/v1/e2e/
1831 - - tests/entrypoints/llm/test_struct_output_generate.py
1832 - - vllm/platforms/rocm.py
1989 + - vllm/
1990 + - tests/models/test_initialization.py
1991 + - tests/models/registry.py
1992 + optional: true
1993 commands:
1834 - - set -x
1835 - - export VLLM_USE_V2_MODEL_RUNNER=1
1836 - - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
1837 - - pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
1838 - - pytest -v -s v1/e2e/general/test_context_length.py
1839 - - pytest -v -s v1/e2e/general/test_min_tokens.py
1840 - - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
1994 + - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
1995
1842 -- label: ":amd: (MI300) Model Runner V2 Examples" # TBD
1843 - timeout_in_minutes: 40
1996 +- label: ":amd: (MI300) Basic Models (Other)" # TBD
1997 + timeout_in_minutes: 65
1998 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1999 dind: false
2000 agent_pool: mi300_1
2001 optional: true
1848 - working_dir: "/vllm-workspace/examples"
2002 + working_dir: "/vllm-workspace/tests"
2003 source_file_dependencies:
1850 - - vllm/v1/worker/gpu/
1851 - - vllm/v1/core/sched/
1852 - - vllm/v1/worker/gpu_worker.py
1853 - - examples/basic/offline_inference/
1854 - - examples/generate/multimodal/
1855 - - examples/features/
1856 - - examples/pooling/embed/vision_embedding_offline.py
1857 - - examples/features/tensorize_vllm_model.py
1858 - - examples/deployment/llm_engine_example.py
1859 - - vllm/platforms/rocm.py
2004 + - vllm/
2005 + - tests/models/test_terratorch.py
2006 + - tests/models/transformers/test_backend.py
2007 + - tests/models/test_registry.py
2008 + - tests/models/test_hyv4_rocm.py
2009 + - tests/models/test_vision.py
2010 + - tests/models/test_deepseek_v4_vl_rocm.py
2011 commands:
1861 - - set -x
1862 - - export VLLM_USE_V2_MODEL_RUNNER=1
1863 - - pip install tensorizer
1864 - - python3 basic/offline_inference/chat.py
1865 - - python3 basic/offline_inference/generate.py --model facebook/opt-125m
1866 - - python3 generate/multimodal/audio_language_offline.py --seed 0
1867 - - python3 generate/multimodal/vision_language_offline.py --seed 0
1868 - - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
1869 - - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
1870 - - python3 pooling/embed/vision_embedding_offline.py --seed 0
1871 - - python3 features/automatic_prefix_caching/prefix_caching_offline.py
1872 - - python3 deployment/llm_engine_example.py
1873 - - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
1874 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
1875 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
1876 -
1877 -- label: ":amd: (MI300) Model Runner V2 Distributed" # TBD
1878 - timeout_in_minutes: 45
1879 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1880 - dind: false
1881 - agent_pool: mi300_2
1882 - num_gpus: 2
1883 - optional: true
1884 - working_dir: "/vllm-workspace/tests"
1885 - source_file_dependencies:
1886 - - vllm/v1/worker/gpu/
1887 - - vllm/v1/worker/gpu_worker.py
1888 - - tests/basic_correctness/test_basic_correctness.py
1889 - - tests/v1/distributed/test_async_llm_dp.py
1890 - - tests/v1/distributed/test_eagle_dp.py
1891 - - vllm/platforms/rocm.py
1892 - commands:
1893 - - set -x
1894 - - export VLLM_USE_V2_MODEL_RUNNER=1
1895 - - TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True"
1896 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray"
1897 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
1898 -
1899 -- label: ":amd: (MI300) Model Runner V2 Pipeline Parallelism" # TBD
1900 - timeout_in_minutes: 65
1901 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1902 - dind: false
1903 - agent_pool: mi300_4
1904 - num_gpus: 4
1905 - optional: true
1906 - working_dir: "/vllm-workspace/tests"
1907 - source_file_dependencies:
1908 - - vllm/v1/worker/gpu/
1909 - - vllm/v1/worker/gpu_worker.py
1910 - - tests/distributed/test_pipeline_parallel.py
1911 - - tests/distributed/test_pp_cudagraph.py
1912 - - vllm/platforms/rocm.py
1913 - commands:
1914 - - set -x
1915 - - export VLLM_USE_V2_MODEL_RUNNER=1
1916 - - pytest -v -s distributed/test_pipeline_parallel.py -k "not ray"
1917 - - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
1918 -
1919 -- label: ":amd: (MI300) Model Runner V2 Spec Decode" # TBD
1920 - timeout_in_minutes: 90
1921 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1922 - dind: false
1923 - agent_pool: mi300_1
1924 - optional: true
1925 - working_dir: "/vllm-workspace/tests"
1926 - source_file_dependencies:
1927 - - vllm/v1/worker/gpu/
1928 - - vllm/v1/worker/gpu_worker.py
1929 - - tests/v1/spec_decode/test_max_len.py
1930 - - tests/v1/spec_decode/test_rejection_sampler_utils.py
1931 - - tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py
1932 - - tests/v1/e2e/spec_decode/
1933 - - vllm/platforms/rocm.py
1934 - commands:
1935 - - set -x
1936 - - export VLLM_USE_V2_MODEL_RUNNER=1
1937 - - pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp"
1938 - - pytest -v -s v1/spec_decode/test_rejection_sampler_utils.py
1939 - - pytest -v -s v1/spec_decode/test_synthetic_rejection_sampler_utils.py
1940 - - pytest -v -s v1/e2e/spec_decode/eagle/
1941 - - pytest -v -s v1/e2e/spec_decode/speculators/
1942 - - pytest -v -s v1/e2e/spec_decode/mtp/
1943 -
1944 -#------------------------------------------------------ mi300 · models / basic -------------------------------------------------------#
1945 -
1946 -- label: ":amd: (MI300) Basic Models (Extra Initialization) Shard %N" # TBD
1947 - timeout_in_minutes: 60
1948 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1949 - dind: false
1950 - agent_pool: mi300_1
1951 - parallelism: 6
1952 - working_dir: "/vllm-workspace/tests"
1953 - source_file_dependencies:
1954 - - vllm/model_executor/models/
1955 - - vllm/model_executor/layers/
1956 - - tests/models/test_initialization.py
1957 - - tests/models/registry.py
1958 - - vllm/_aiter_ops.py
1959 - - vllm/platforms/rocm.py
1960 - commands:
1961 - - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
1962 -
1963 -- label: ":amd: (MI300) Basic Models (Initialization)" # TBD
1964 - timeout_in_minutes: 30
1965 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1966 - dind: false
1967 - agent_pool: mi300_1
1968 - working_dir: "/vllm-workspace/tests"
1969 - source_file_dependencies:
1970 - - vllm/
1971 - - tests/models/test_initialization.py
1972 - - tests/models/registry.py
1973 - commands:
1974 - - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
1975 -
1976 -- label: ":amd: (MI300) Basic Models (Other)" # TBD
1977 - timeout_in_minutes: 65
1978 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1979 - dind: false
1980 - agent_pool: mi300_1
1981 - optional: true
1982 - working_dir: "/vllm-workspace/tests"
1983 - source_file_dependencies:
1984 - - vllm/
1985 - - tests/models/test_terratorch.py
1986 - - tests/models/transformers/test_backend.py
1987 - - tests/models/test_registry.py
1988 - - tests/models/test_hyv4_rocm.py
1989 - - tests/models/test_vision.py
1990 - commands:
1991 - - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py
2012 + - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py
2013 - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'not distributed'
2014 - pytest -v -s models/test_vision.py::test_simple_mrope_vision_model_spatial_merge
2015
@@ -2007,8 +2028,9 @@ steps:
2028 commands:
2029 - pytest -v -s models/language/pooling_mteb_test
2030
2010 -- label: ":amd: (MI300) Language Models (Extended Pooling)" # TBD
2011 - timeout_in_minutes: 80
2031 +- label: ":amd: (MI300) Language Models (Extended Pooling) Shard %N"
2032 + timeout_in_minutes: 95
2033 + parallelism: 4
2034 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2035 dind: false
2036 agent_pool: mi300_1
@@ -2018,10 +2040,10 @@ steps:
2040 - vllm/
2041 - tests/models/language/pooling
2042 commands:
2021 - - pytest -v -s models/language/pooling -m 'not core_model'
2043 + - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2044
2045 - label: ":amd: (MI300) Language Models (Standard)" # TBD
2024 - timeout_in_minutes: 45
2046 + timeout_in_minutes: 60
2047 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2048 dind: false
2049 agent_pool: mi300_1
@@ -2072,22 +2094,6 @@ steps:
2094 - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
2095 - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
2096
2075 -- label: ":amd: (MI300) Language Models (Hybrid) Shard %N" # TBD
2076 - timeout_in_minutes: 65
2077 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2078 - dind: false
2079 - agent_pool: mi300_1
2080 - parallelism: 2
2081 - optional: true
2082 - working_dir: "/vllm-workspace/tests"
2083 - source_file_dependencies:
2084 - - vllm/
2085 - - tests/models/language/generation
2086 - commands:
2087 - - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
2088 - - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
2089 - - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2090 -
2097 #---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------#
2098
2099 - label: ":amd: (MI300) Multimodal Models (Extended Generation 1)" # TBD
@@ -2105,8 +2111,9 @@ steps:
2111 - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
2112 - pytest -v -s models/multimodal/test_mapping.py
2113
2108 -- label: ":amd: (MI300) Multimodal Models (Extended Generation 2)" # TBD
2109 - timeout_in_minutes: 65
2114 +- label: ":amd: (MI300) Multimodal Models (Extended Generation 2) Shard %N"
2115 + timeout_in_minutes: 100
2116 + parallelism: 4
2117 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2118 dind: false
2119 agent_pool: mi300_1
@@ -2116,11 +2123,11 @@ steps:
2123 - vllm/
2124 - tests/models/multimodal/generation
2125 commands:
2119 - - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
2126 + - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2127
2128
2129 - label: ":amd: (MI300) Multimodal Models (Extended Generation 3)" # TBD
2123 - timeout_in_minutes: 55
2130 + timeout_in_minutes: 90
2131 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2132 dind: false
2133 agent_pool: mi300_1
@@ -2133,7 +2140,7 @@ steps:
2140 - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
2141
2142 - label: ":amd: (MI300) Multimodal Models (Standard) 1: qwen2" # TBD
2136 - timeout_in_minutes: 45
2143 + timeout_in_minutes: 65
2144 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2145 dind: false
2146 agent_pool: mi300_1
@@ -2147,7 +2154,7 @@ steps:
2154 - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
2155
2156 - label: ":amd: (MI300) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD
2150 - timeout_in_minutes: 40
2157 + timeout_in_minutes: 55
2158 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2159 dind: false
2160 agent_pool: mi300_1
@@ -2157,27 +2164,13 @@ steps:
2164 - vllm/
2165 - tests/models/multimodal/generation
2166 - tests/models/multimodal/test_mapping.py
2167 + - tests/models/multimodal
2168 commands:
2169 - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
2170 - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
2171
2164 -- label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper" # TBD
2165 - timeout_in_minutes: 70
2166 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2167 - dind: false
2168 - agent_pool: mi300_1
2169 - working_dir: "/vllm-workspace/tests"
2170 - source_file_dependencies:
2171 - - vllm/
2172 - - tests/models/multimodal
2173 - commands:
2174 - - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
2175 - - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
2176 - - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
2177 - - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
2178 -
2172 - label: ":amd: (MI300) Multimodal Processor Shard %N" # TBD
2180 - timeout_in_minutes: 110
2173 + timeout_in_minutes: 115
2174 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2175 dind: false
2176 agent_pool: mi300_1
@@ -2205,7 +2198,7 @@ steps:
2198 - pytest -v -s models/multimodal/pooling -m 'not core_model'
2199
2200 - label: ":amd: (MI300) Multimodal Models (Standard) 2: qwen3 + gemma" # TBD
2208 - timeout_in_minutes: 50
2201 + timeout_in_minutes: 55
2202 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2203 dind: false
2204 agent_pool: mi300_1
@@ -2233,7 +2226,9 @@ steps:
2226 - vllm/platforms/rocm.py
2227 - tests/models/quantization
2228 - vllm/model_executor/model_loader/
2229 + optional: true
2230 commands:
2231 + - unset VLLM_USE_V2_MODEL_RUNNER
2232 - pytest -v -s models/quantization
2233
2234 #-------------------------------------------------- mi300 · models / transformers ---------------------------------------------------#
@@ -2311,7 +2306,7 @@ steps:
2306 #---------------------------------------------------------- mi300 · plugins ----------------------------------------------------------#
2307
2308 - label: ":amd: (MI300) Plugin Integration" # TBD
2314 - timeout_in_minutes: 35
2309 + timeout_in_minutes: 50
2310 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2311 dind: false
2312 agent_pool: mi300_2
@@ -2324,43 +2319,43 @@ steps:
2319 - vllm/platforms/rocm.py
2320 commands:
2321 # BEGIN: platform plugin and general plugin tests, all the code in-between runs on dummy platform
2322 + # END: platform plugin tests
2323 + # BEGIN: `io_processor` plugins test, all the code in between uses the `prithvi_io_processor` plugin
2324 + # END: `io_processor` plugins test
2325 + # BEGIN: `bge_m3_sparse io_processor` test
2326 + # END: `bge_m3_sparse io_processor` test
2327 + # BEGIN: `colbert_query io_processor` test
2328 + # END: `colbert_query io_processor` test
2329 + # BEGIN: `stat_logger` plugins test
2330 + # END: `stat_logger` plugins test
2331 + # BEGIN: `endpoint` plugins test
2332 + # END: `endpoint` plugins test
2333 + # BEGIN: other tests
2334 - pip install -e ./plugins/vllm_add_dummy_platform
2335 - pytest -v -s plugins_tests/test_platform_plugins.py
2336 - pip uninstall vllm_add_dummy_platform -y
2330 - # END: platform plugin tests
2331 - # BEGIN: `io_processor` plugins test, all the code in between uses the `prithvi_io_processor` plugin
2337 + - pytest -v -s ./plugins_tests/test_io_processor_plugins.py
2338 - pip install -e ./plugins/prithvi_io_processor_plugin
2333 - - pytest -v -s plugins_tests/test_io_processor_plugins.py
2339 - pytest -v -s plugins_tests/test_terratorch_io_processor_plugins.py
2340 - pip uninstall prithvi_io_processor_plugin -y
2336 - # END: `io_processor` plugins test
2337 - # BEGIN: `bge_m3_sparse io_processor` test
2341 - pip install -e ./plugins/bge_m3_sparse_plugin
2342 - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
2343 - pip uninstall bge_m3_sparse_plugin -y
2341 - # END: `bge_m3_sparse io_processor` test
2342 - # BEGIN: `colbert_query io_processor` test
2344 - pip install -e ./plugins/colbert_query_plugin
2345 - pytest -v -s plugins_tests/test_colbert_query_io_processor_plugins.py
2346 - pip uninstall colbert_query_plugin -y
2346 - # END: `colbert_query io_processor` test
2347 - # BEGIN: `stat_logger` plugins test
2347 - pip install -e ./plugins/vllm_add_dummy_stat_logger
2348 - pytest -v -s plugins_tests/test_stats_logger_plugins.py
2349 - pip uninstall dummy_stat_logger -y
2351 - # END: `stat_logger` plugins test
2352 - # BEGIN: `endpoint` plugins test
2350 - pip install -e ./plugins/vllm_add_dummy_endpoint_plugin
2351 - pytest -v -s plugins_tests/test_endpoint_plugins.py
2352 - pip uninstall vllm_add_dummy_endpoint_plugin -y
2356 - # END: `endpoint` plugins test
2357 - # BEGIN: other tests
2353 - pytest -v -s plugins_tests/test_scheduler_plugins.py
2354 - pip install -e ./plugins/vllm_add_dummy_model
2355 - pytest -v -s distributed/test_distributed_oot.py
2361 - - pytest -v -s plugins_tests/test_oot_registration_online.py # it needs a clean process
2362 - - pytest -v -s plugins_tests/test_oot_registration_offline.py # it needs a clean process
2363 - - pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins
2356 + - pytest -v -s plugins_tests/test_oot_registration_online.py
2357 + - pytest -v -s plugins_tests/test_oot_registration_offline.py
2358 + - pytest -v -s plugins_tests/lora_resolvers
2359
2360 - label: ":amd: (MI300) GGUF Plugin" # TBD
2361 timeout_in_minutes: 45
@@ -2402,18 +2397,24 @@ steps:
2397 - tests/entrypoints/serve/instrumentator/test_uds.py
2398 - tests/v1/sample/test_logprobs_e2e.py
2399 - vllm/platforms/rocm.py
2400 + - tests/entrypoints/openai/chat_completion/test_include_reasoning.py
2401 + - tests/entrypoints/openai/chat_completion/test_serving_chat.py
2402 + - tests/entrypoints/openai/completion/test_shutdown.py
2403 + - tests/entrypoints/openai/test_uds.py
2404 commands:
2405 - export VLLM_USE_RUST_FRONTEND=1
2406 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2407 - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
2408 - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming"
2409 + - pytest -v -s entrypoints/openai/chat_completion/test_include_reasoning.py
2410 - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
2411 + - pytest -v -s entrypoints/openai/chat_completion/test_serving_chat.py -k "test_chat_per_request_metrics_follow_server_flag or test_streaming_reasoning_usage_counts_across_deltas or test_completion_tokens_details"
2412 - pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
2413 - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
2414 - pytest -v -s entrypoints/serve/instrumentator/test_uds.py
2415 - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
2416
2416 -- label: ":amd: (MI300) Rust Frontend Serve + Admin Coverage" # TBD
2417 +- label: ":amd: (MI300) Rust Frontend Serve/Admin Coverage"
2418 timeout_in_minutes: 30
2419 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2420 dind: false
@@ -2509,8 +2510,9 @@ steps:
2510
2511 #------------------------------------------------------- mi300 · quantization --------------------------------------------------------#
2512
2512 -- label: ":amd: (MI300) Quantization" # TBD
2513 +- label: ":amd: (MI300) Quantization Shard %N"
2514 timeout_in_minutes: 115
2515 + parallelism: 4
2516 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2517 dind: false
2518 agent_pool: mi300_1
@@ -2524,19 +2526,19 @@ steps:
2526 - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py
2527 - tests/quantization
2528 - tests/rocm/test_moe_weight_replay.py
2529 + optional: true
2530 commands:
2528 -
2531 # temporary install here since we need nightly, will move to requirements/test.in
2532 # after torchao 0.12 release, and pin a working version of torchao nightly here
2531 -
2533 # since torchao nightly is only compatible with torch nightly currently
2534 # https://github.com/pytorch/ao/issues/2919, we'll have to skip new torchao tests for now
2535 # we can only upgrade after this is resolved
2536 # TODO(jerryzh168): resolve the above comment
2537 + - unset VLLM_USE_V2_MODEL_RUNNER
2538 - uv pip install --system torchao==0.17.0
2539 - uv pip install --system conch-triton-kernels
2538 - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
2539 - - pytest -v -s rocm/test_moe_weight_replay.py
2540 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
2541 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
2542
2543 #--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------#
2544
@@ -2556,6 +2558,7 @@ steps:
2558 - tests/conftest.py
2559 - vllm/_aiter_ops.py
2560 - vllm/platforms/rocm.py
2561 + - vllm/entrypoints/generate/beam_search
2562 commands:
2563 - pytest -v -s samplers
2564
@@ -2570,6 +2573,18 @@ steps:
2573 source_file_dependencies:
2574 - vllm/
2575 - tests/test_regression
2576 + - vllm/config/
2577 + - vllm/distributed/
2578 + - vllm/engine/
2579 + - vllm/inputs/
2580 + - vllm/model_executor/
2581 + - vllm/multimodal/
2582 + - vllm/platforms/
2583 + - vllm/sampling_params.py
2584 + - vllm/transformers_utils/
2585 + - vllm/utils/
2586 + - vllm/v1/
2587 + optional: true
2588 commands:
2589 - pip install 'modelscope<1.38'
2590 - pytest -v -s test_regression.py
@@ -2586,6 +2601,8 @@ steps:
2601 - requirements/
2602 - setup.py
2603 - vllm/platforms/rocm.py
2604 + - .buildkite/scripts/check-ray-compatibility.sh
2605 + optional: true
2606 commands:
2607 - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
2608
@@ -2608,7 +2625,7 @@ steps:
2625 - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
2626
2627 - label: ":amd: (MI300) E2E Scheduling" # TBD
2611 - timeout_in_minutes: 45
2628 + timeout_in_minutes: 55
2629 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2630 dind: false
2631 agent_pool: mi300_1
@@ -2622,7 +2639,7 @@ steps:
2639 - pytest -v -s v1/e2e/general/test_async_scheduling.py
2640
2641 - label: ":amd: (MI300) V1 Engine" # TBD
2625 - timeout_in_minutes: 40
2642 + timeout_in_minutes: 45
2643 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2644 dind: false
2645 agent_pool: mi300_1
@@ -2632,12 +2649,14 @@ steps:
2649 - vllm/v1/engine/
2650 - tests/v1/engine/
2651 - vllm/platforms/rocm.py
2652 + - tests/v1/test_tensor_ipc_queue.py
2653 commands:
2654 - pytest -v -s v1/engine/test_preprocess_error_handling.py
2655 - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
2656 + - pytest -v -s v1/test_tensor_ipc_queue.py
2657
2658 - label: ":amd: (MI300) Spec Decode Draft Model" # TBD
2640 - timeout_in_minutes: 45
2659 + timeout_in_minutes: 60
2660 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2661 dind: false
2662 agent_pool: mi300_1
@@ -2654,7 +2673,7 @@ steps:
2673 commands:
2674 - pytest -v -s v1/e2e/spec_decode/draft_model/
2675
2657 -- label: ":amd: (MI300) Spec Decode Eagle" # TBD
2676 +- label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen"
2677 timeout_in_minutes: 60
2678 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2679 dind: false
@@ -2670,7 +2689,26 @@ steps:
2689 - tests/v1/e2e/spec_decode/
2690 - vllm/platforms/rocm.py
2691 commands:
2673 - - pytest -v -s v1/e2e/spec_decode/eagle/
2692 + - pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3"
2693 +
2694 +- label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other"
2695 + timeout_in_minutes: 60
2696 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2697 + dind: false
2698 + agent_pool: mi300_1
2699 + optional: true
2700 + working_dir: /vllm-workspace/tests
2701 + source_file_dependencies:
2702 + - vllm/v1/spec_decode/
2703 + - vllm/v1/worker/gpu/spec_decode/
2704 + - vllm/model_executor/model_loader/
2705 + - vllm/v1/sample/
2706 + - vllm/model_executor/layers/
2707 + - tests/v1/e2e/spec_decode/
2708 + - vllm/platforms/rocm.py
2709 + commands:
2710 + - pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3"
2711 +
2712
2713 - label: ":amd: (MI300) Spec Decode N-Gram + Suffix" # TBD
2714 timeout_in_minutes: 35
@@ -2693,7 +2731,7 @@ steps:
2731 - python3 spec_decode/test_custom_proposer.py
2732
2733 - label: ":amd: (MI300) Spec Decode Speculators + MTP" # TBD
2696 - timeout_in_minutes: 50
2734 + timeout_in_minutes: 75
2735 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2736 dind: false
2737 agent_pool: mi300_1
@@ -2708,12 +2746,13 @@ steps:
2746 - vllm/transformers_utils/configs/speculators/
2747 - tests/v1/e2e/spec_decode/
2748 - vllm/platforms/rocm.py
2749 + - vllm/v1/attention/backends/
2750 commands:
2751 - pytest -v -s v1/e2e/spec_decode/speculators/
2752 - pytest -v -s v1/e2e/spec_decode/mtp/
2753
2715 -- label: ":amd: (MI300) Speculators Correctness" # TBD
2716 - timeout_in_minutes: 35
2754 +- label: ":amd: (MI300) Speculators Correctness Nightly"
2755 + timeout_in_minutes: 40
2756 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2757 dind: false
2758 agent_pool: mi300_1
@@ -2740,6 +2779,7 @@ steps:
2779 - tests/evals/gsm8k/
2780 - tests/v1/spec_decode/test_speculators_correctness.py
2781 - vllm/platforms/rocm.py
2782 + - vllm/v1/worker/gpu/spec_decode/dflash/
2783 commands:
2784 - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
2785 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
@@ -2753,6 +2793,17 @@ steps:
2793 source_file_dependencies:
2794 - vllm/
2795 - tests/v1/spec_decode
2796 + - vllm/config/
2797 + - vllm/distributed/
2798 + - vllm/inputs/
2799 + - vllm/model_executor/
2800 + - vllm/models/qwen4_exp/
2801 + - vllm/platforms/
2802 + - vllm/sampling_params.py
2803 + - vllm/transformers_utils/
2804 + - vllm/utils/
2805 + - vllm/v1/
2806 + optional: true
2807 commands:
2808 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2809 - pytest -v -s -m 'not slow_test' v1/spec_decode
@@ -2763,7 +2814,7 @@ steps:
2814 dind: false
2815 agent_pool: mi300_2
2816 num_gpus: 2
2766 - working_dir: "/vllm-workspace/"
2817 + working_dir: /vllm-workspace
2818 source_file_dependencies:
2819 - vllm/config/speculative.py
2820 - vllm/distributed/kv_transfer/kv_connector/
@@ -2785,12 +2836,15 @@ steps:
2836 - vllm/_aiter_ops.py
2837 - tests/v1/kv_connector/extract_hidden_states_integration/
2838 - vllm/platforms/rocm.py
2839 + - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
2840 + - tests/v1/kv_connector/extract_hidden_states_integration
2841 + optional: true
2842 commands:
2843 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2790 - - pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration
2844 + - pytest -v -s -m 'distributed' tests/v1/kv_connector/extract_hidden_states_integration
2845
2846 - label: ":amd: (MI300) V1 Attention Shard %N" # TBD
2793 - timeout_in_minutes: 35
2847 + timeout_in_minutes: 125
2848 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2849 dind: false
2850 agent_pool: mi300_1
@@ -2808,7 +2862,7 @@ steps:
2862 commands:
2863 - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
2864
2811 -- label: ":amd: (MI300) V1 Core + KV + Metrics" # TBD
2865 +- label: ":amd: (MI300) V1 Core"
2866 timeout_in_minutes: 60
2867 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2868 dind: false
@@ -2826,71 +2880,267 @@ steps:
2880 - tests/v1/ec_connector/unit
2881 - tests/v1/metrics
2882 - tests/entrypoints/openai/correctness/test_lmeval.py
2883 + - vllm/config/
2884 + - vllm/distributed/
2885 + - vllm/engine/
2886 + - vllm/entrypoints/pooling/
2887 + - vllm/inputs/
2888 + - vllm/lora/
2889 + - vllm/model_executor/
2890 + - vllm/multimodal/
2891 + - vllm/outputs.py
2892 + - vllm/platforms/
2893 + - vllm/pooling_params.py
2894 + - vllm/profiler/
2895 + - vllm/sampling_params.py
2896 + - vllm/tokenizers/
2897 + - vllm/transformers_utils/
2898 + - vllm/utils/
2899 + - vllm/v1/
2900 + optional: true
2901 commands:
2830 - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
2902 + # - export HSA_NO_SCRATCH_RECLAIM=1
2903 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2904 - pytest -v -s -m 'not cpu_test' v1/core
2833 - - pytest -v -s v1/executor
2834 - - pytest -v -s v1/kv_offload
2835 - - pytest -v -s v1/simple_kv_offload
2836 - - pytest -v -s v1/worker
2837 - - pytest -v -s v1/streaming_input
2838 - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
2839 - - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit
2840 - - pytest -v -s -m 'not cpu_test' v1/metrics
2841 - - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api
2842 - # - export HSA_NO_SCRATCH_RECLAIM=1
2843 - - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
2905
2845 -- label: ":amd: (MI300) V1 Sample + Logits" # TBD
2846 - timeout_in_minutes: 70
2906 +- label: ":amd: (MI300) V1 Executor + Worker"
2907 + timeout_in_minutes: 60
2908 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2909 dind: false
2910 agent_pool: mi300_1
2850 - optional: true
2851 - working_dir: "/vllm-workspace/tests"
2911 + working_dir: /vllm-workspace/tests
2912 source_file_dependencies:
2913 - vllm/
2854 - - tests/v1/sample
2855 - - tests/v1/logits_processors
2856 - - tests/v1/test_oracle.py
2857 - - tests/v1/test_request.py
2858 - - tests/v1/test_outputs.py
2859 - commands:
2860 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2861 - - pytest -v -s v1/sample
2862 - - pytest -v -s v1/logits_processors
2863 - - pytest -v -s v1/test_oracle.py
2864 - - pytest -v -s v1/test_request.py
2865 - - pytest -v -s v1/test_outputs.py
2866 -
2867 -- label: ":amd: (MI300) Distributed DP Basic" # TBD
2868 - timeout_in_minutes: 45
2914 + - tests/v1/core
2915 + - tests/v1/executor
2916 + - tests/v1/kv_offload
2917 + - tests/v1/simple_kv_offload
2918 + - tests/v1/worker
2919 + - tests/v1/streaming_input
2920 + - tests/v1/kv_connector/unit
2921 + - tests/v1/ec_connector/unit
2922 + - tests/v1/metrics
2923 + - tests/entrypoints/openai/correctness/test_lmeval.py
2924 + - vllm/config/
2925 + - vllm/distributed/
2926 + - vllm/engine/
2927 + - vllm/entrypoints/pooling/
2928 + - vllm/inputs/
2929 + - vllm/lora/
2930 + - vllm/model_executor/
2931 + - vllm/multimodal/
2932 + - vllm/outputs.py
2933 + - vllm/platforms/
2934 + - vllm/pooling_params.py
2935 + - vllm/profiler/
2936 + - vllm/sampling_params.py
2937 + - vllm/tokenizers/
2938 + - vllm/transformers_utils/
2939 + - vllm/utils/
2940 + - vllm/v1/
2941 + commands:
2942 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2943 + - pytest -v -s v1/executor
2944 + - pytest -v -s v1/worker
2945 + - pytest -v -s v1/streaming_input
2946 + optional: true
2947 +
2948 +- label: ":amd: (MI300) V1 KV Offload"
2949 + timeout_in_minutes: 60
2950 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2951 dind: false
2871 - agent_pool: mi300_2
2952 + agent_pool: mi300_1
2953 + working_dir: /vllm-workspace/tests
2954 + source_file_dependencies:
2955 + - vllm/
2956 + - tests/v1/core
2957 + - tests/v1/executor
2958 + - tests/v1/kv_offload
2959 + - tests/v1/simple_kv_offload
2960 + - tests/v1/worker
2961 + - tests/v1/streaming_input
2962 + - tests/v1/kv_connector/unit
2963 + - tests/v1/ec_connector/unit
2964 + - tests/v1/metrics
2965 + - tests/entrypoints/openai/correctness/test_lmeval.py
2966 + - vllm/config/
2967 + - vllm/distributed/
2968 + - vllm/engine/
2969 + - vllm/entrypoints/pooling/
2970 + - vllm/inputs/
2971 + - vllm/lora/
2972 + - vllm/model_executor/
2973 + - vllm/multimodal/
2974 + - vllm/outputs.py
2975 + - vllm/platforms/
2976 + - vllm/pooling_params.py
2977 + - vllm/profiler/
2978 + - vllm/sampling_params.py
2979 + - vllm/tokenizers/
2980 + - vllm/transformers_utils/
2981 + - vllm/utils/
2982 + - vllm/v1/
2983 + commands:
2984 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2985 + - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh
2986 + - pytest -v -s v1/kv_offload
2987 + - pytest -v -s v1/simple_kv_offload
2988 + optional: true
2989 +
2990 +- label: ":amd: (MI300) V1 KV Connectors Shard %N"
2991 + timeout_in_minutes: 60
2992 + parallelism: 4
2993 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2994 + dind: false
2995 + agent_pool: mi300_1
2996 + working_dir: /vllm-workspace/tests
2997 + source_file_dependencies:
2998 + - vllm/
2999 + - tests/v1/core
3000 + - tests/v1/executor
3001 + - tests/v1/kv_offload
3002 + - tests/v1/simple_kv_offload
3003 + - tests/v1/worker
3004 + - tests/v1/streaming_input
3005 + - tests/v1/kv_connector/unit
3006 + - tests/v1/ec_connector/unit
3007 + - tests/v1/metrics
3008 + - tests/entrypoints/openai/correctness/test_lmeval.py
3009 + - vllm/config/
3010 + - vllm/distributed/
3011 + - vllm/engine/
3012 + - vllm/entrypoints/pooling/
3013 + - vllm/inputs/
3014 + - vllm/lora/
3015 + - vllm/model_executor/
3016 + - vllm/multimodal/
3017 + - vllm/outputs.py
3018 + - vllm/platforms/
3019 + - vllm/pooling_params.py
3020 + - vllm/profiler/
3021 + - vllm/sampling_params.py
3022 + - vllm/tokenizers/
3023 + - vllm/transformers_utils/
3024 + - vllm/utils/
3025 + - vllm/v1/
3026 + commands:
3027 + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
3028 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3029 + - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
3030 + - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
3031 + optional: true
3032 +
3033 +- label: ":amd: (MI300) V1 Metrics + LM Eval"
3034 + timeout_in_minutes: 65
3035 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3036 + dind: false
3037 + agent_pool: mi300_1
3038 + working_dir: /vllm-workspace/tests
3039 + source_file_dependencies:
3040 + - vllm/
3041 + - tests/v1/core
3042 + - tests/v1/executor
3043 + - tests/v1/kv_offload
3044 + - tests/v1/simple_kv_offload
3045 + - tests/v1/worker
3046 + - tests/v1/streaming_input
3047 + - tests/v1/kv_connector/unit
3048 + - tests/v1/ec_connector/unit
3049 + - tests/v1/metrics
3050 + - tests/entrypoints/openai/correctness/test_lmeval.py
3051 + - vllm/config/
3052 + - vllm/distributed/
3053 + - vllm/engine/
3054 + - vllm/entrypoints/pooling/
3055 + - vllm/inputs/
3056 + - vllm/lora/
3057 + - vllm/model_executor/
3058 + - vllm/multimodal/
3059 + - vllm/outputs.py
3060 + - vllm/platforms/
3061 + - vllm/pooling_params.py
3062 + - vllm/profiler/
3063 + - vllm/sampling_params.py
3064 + - vllm/tokenizers/
3065 + - vllm/transformers_utils/
3066 + - vllm/utils/
3067 + - vllm/v1/
3068 + commands:
3069 + - export GIT_CONFIG_COUNT=1
3070 + - export GIT_CONFIG_KEY_0=http.version
3071 + - export GIT_CONFIG_VALUE_0=HTTP/1.1
3072 + - export GIT_TERMINAL_PROMPT=0
3073 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3074 + - pytest -v -s -m 'not cpu_test' v1/metrics
3075 + - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api
3076 + - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
3077 + optional: true
3078 +
3079 +
3080 +- label: ":amd: (MI300) V1 Sample"
3081 + timeout_in_minutes: 70
3082 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3083 + dind: false
3084 + agent_pool: mi300_1
3085 optional: true
2873 - num_gpus: 2
3086 working_dir: "/vllm-workspace/tests"
3087 source_file_dependencies:
3088 + - vllm/
3089 + - tests/v1/sample
3090 + - tests/v1/logits_processors
3091 + - tests/v1/test_oracle.py
3092 + - tests/v1/test_request.py
3093 + - tests/v1/test_outputs.py
3094 + - vllm/config/
3095 - vllm/distributed/
3096 - vllm/engine/
2878 - - vllm/executor/
2879 - - vllm/worker/worker_base.py
2880 - - vllm/v1/engine/
2881 - - vllm/v1/worker/
2882 - - tests/v1/distributed
2883 - - tests/entrypoints/launchers/api_server/test_multi_api_servers.py
2884 - - vllm/platforms/rocm.py
3097 + - vllm/inputs/
3098 + - vllm/logger.py
3099 + - vllm/model_executor/
3100 + - vllm/platforms/
3101 + - vllm/sampling_params.py
3102 + - vllm/transformers_utils/
3103 + - vllm/utils/
3104 + - vllm/v1/
3105 commands:
2886 - - export NCCL_CUMEM_HOST_ENABLE=0
2887 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
2888 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
2889 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
2890 - - DP_SIZE=2 pytest -v -s entrypoints/launchers/api_server/test_multi_api_servers.py
3106 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3107 + - pytest -v -s v1/sample
3108 +
3109 +- label: ":amd: (MI300) V1 Logits + Oracle"
3110 + timeout_in_minutes: 70
3111 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3112 + dind: false
3113 + agent_pool: mi300_1
3114 + optional: true
3115 + working_dir: /vllm-workspace/tests
3116 + source_file_dependencies:
3117 + - vllm/
3118 + - tests/v1/sample
3119 + - tests/v1/logits_processors
3120 + - tests/v1/test_oracle.py
3121 + - tests/v1/test_request.py
3122 + - tests/v1/test_outputs.py
3123 + - vllm/config/
3124 + - vllm/distributed/
3125 + - vllm/engine/
3126 + - vllm/inputs/
3127 + - vllm/logger.py
3128 + - vllm/model_executor/
3129 + - vllm/platforms/
3130 + - vllm/sampling_params.py
3131 + - vllm/transformers_utils/
3132 + - vllm/utils/
3133 + - vllm/v1/
3134 + commands:
3135 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3136 + - pytest -v -s v1/logits_processors
3137 + - pytest -v -s v1/test_oracle.py
3138 + - pytest -v -s v1/test_request.py
3139 + - pytest -v -s v1/test_outputs.py
3140 +
3141
3142 - label: ":amd: (MI300) Metrics, Tracing" # TBD
2893 - timeout_in_minutes: 25
3143 + timeout_in_minutes: 30
3144 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3145 dind: false
3146 agent_pool: mi300_2
@@ -2901,6 +3151,18 @@ steps:
3151 - vllm/
3152 - tests/v1/tracing
3153 - tests/tracing/
3154 + - vllm/config/
3155 + - vllm/distributed/
3156 + - vllm/engine/
3157 + - vllm/inputs/
3158 + - vllm/model_executor/
3159 + - vllm/multimodal/
3160 + - vllm/platforms/
3161 + - vllm/sampling_params.py
3162 + - vllm/tracing/
3163 + - vllm/transformers_utils/
3164 + - vllm/utils/
3165 + - vllm/v1/
3166 commands:
3167 - "pip install \
3168 'opentelemetry-sdk>=1.26.0' \
@@ -2911,7 +3173,7 @@ steps:
3173 - pytest -v -s tracing
3174
3175 - label: ":amd: (MI300) V1 E2E" # TBD
2914 - timeout_in_minutes: 25
3176 + timeout_in_minutes: 30
3177 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3178 dind: false
3179 agent_pool: mi300_2
@@ -2921,31 +3183,32 @@ steps:
3183 source_file_dependencies:
3184 - vllm/
3185 - tests/v1/e2e
3186 + - vllm/compilation/
3187 + - vllm/config/
3188 + - vllm/distributed/
3189 + - vllm/engine/
3190 + - vllm/envs.py
3191 + - vllm/forward_context.py
3192 + - vllm/inputs/
3193 + - vllm/logger.py
3194 + - vllm/logging_utils/
3195 + - vllm/model_executor/
3196 + - vllm/multimodal/
3197 + - vllm/platforms/
3198 + - vllm/sampling_params.py
3199 + - vllm/transformers_utils/
3200 + - vllm/triton_utils/
3201 + - vllm/utils/
3202 + - vllm/v1/
3203 + - tests/v1/e2e/spec_decode/
3204 commands:
3205 - >-
3206 pytest -v -s
3207 v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_tensor_parallelism
3208 v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_engine_args_tensor_parallelism
3209
2930 -- label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance" # TBD
2931 - timeout_in_minutes: 45
2932 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2933 - dind: false
2934 - agent_pool: mi300_2
2935 - num_gpus: 2
2936 - optional: true
2937 - working_dir: "/vllm-workspace/tests"
2938 - source_file_dependencies:
2939 - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
2940 - - vllm/v1/worker/kv_connector_model_runner_mixin.py
2941 - - tests/v1/kv_connector/nixl_integration/
2942 - - vllm/platforms/rocm.py
2943 - commands:
2944 - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
2945 - - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
2946 -
3210 - label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" # TBD
2948 - timeout_in_minutes: 45
3211 + timeout_in_minutes: 60
3212 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3213 dind: false
3214 agent_pool: mi300_4
@@ -2984,38 +3247,6 @@ steps:
3247 - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
3248 - pytest -v -s distributed/test_utils.py
3249
2987 -- label: ":amd: (MI300) Distributed NixlConnector PD accuracy" # TBD
2988 - timeout_in_minutes: 45
2989 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2990 - dind: false
2991 - agent_pool: mi300_4
2992 - num_gpus: 4
2993 - optional: true
2994 - working_dir: "/vllm-workspace/tests"
2995 - source_file_dependencies:
2996 - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
2997 - - tests/v1/kv_connector/nixl_integration/
2998 - - vllm/platforms/rocm.py
2999 - commands:
3000 - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
3001 - - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
3002 -
3003 -- label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" # TBD
3004 - timeout_in_minutes: 30
3005 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3006 - dind: false
3007 - agent_pool: mi300_4
3008 - num_gpus: 4
3009 - optional: true
3010 - working_dir: "/vllm-workspace/tests"
3011 - source_file_dependencies:
3012 - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
3013 - - tests/v1/kv_connector/nixl_integration/
3014 - - vllm/platforms/rocm.py
3015 - commands:
3016 - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
3017 - - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
3018 -
3250 - label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" # TBD
3251 timeout_in_minutes: 60
3252 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -3089,16 +3320,18 @@ steps:
3320 - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
3321
3322 - label: ":amd: (MI300) V1 E2E Hybrid Chunked Prefill" # TBD
3092 - timeout_in_minutes: 30
3323 + timeout_in_minutes: 35
3324 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
3325 dind: false
3326 agent_pool: mi300_4
3327 num_gpus: 4
3328 optional: true
3329 source_file_dependencies:
3099 - - vllm/v1/attention/backends/utils.py
3100 - - vllm/v1/worker/gpu_model_runner.py
3101 - - tests/v1/e2e/test_hybrid_chunked_prefill.py
3330 + - vllm/v1/attention/backends/utils.py
3331 + - vllm/v1/worker/gpu_model_runner.py
3332 + - tests/v1/e2e/test_hybrid_chunked_prefill.py
3333 + - vllm/v1/core/
3334 + working_dir: /vllm-workspace/tests
3335 commands:
3336 - pytest -v -s v1/e2e/test_hybrid_chunked_prefill.py
3337
@@ -3114,6 +3347,7 @@ steps:
3347 source_file_dependencies:
3348 - vllm/
3349 - tests/weight_loading
3350 + optional: true
3351 commands:
3352 - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
3353
@@ -3156,11 +3390,90 @@ steps:
3390
3391 #-------------------------------------------------------- mi355 · distributed --------------------------------------------------------#
3392
3159 -- label: ":amd: (MI355) Distributed Features" # TBD
3393 +- label: ":amd: (MI355) Distributed DP Basic" # TBD
3394 timeout_in_minutes: 45
3395 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3396 dind: false
3397 agent_pool: mi355_2
3398 + optional: true
3399 + num_gpus: 2
3400 + working_dir: "/vllm-workspace/tests"
3401 + source_file_dependencies:
3402 + - vllm/distributed/
3403 + - vllm/engine/
3404 + - vllm/executor/
3405 + - vllm/worker/worker_base.py
3406 + - vllm/v1/engine/
3407 + - vllm/v1/worker/
3408 + - tests/v1/distributed
3409 + - tests/entrypoints/launchers/api_server/test_multi_api_servers.py
3410 + - vllm/platforms/rocm.py
3411 + - tests/v1/e2e/general/test_sharded_sampling.py
3412 + - tests/v1/e2e/spec_decode/test_sharded_sampling.py
3413 + - tests/v1/e2e/spec_decode/utils.py
3414 + commands:
3415 + - export NCCL_CUMEM_HOST_ENABLE=0
3416 + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
3417 + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
3418 + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
3419 + - DP_SIZE=2 pytest -v -s entrypoints/launchers/api_server/test_multi_api_servers.py
3420 + - pytest -v -s v1/e2e/general/test_sharded_sampling.py
3421 + - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py
3422 +
3423 +- label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance" # TBD
3424 + timeout_in_minutes: 45
3425 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3426 + dind: false
3427 + agent_pool: mi355_2
3428 + num_gpus: 2
3429 + optional: true
3430 + working_dir: "/vllm-workspace/tests"
3431 + source_file_dependencies:
3432 + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
3433 + - vllm/v1/worker/kv_connector_model_runner_mixin.py
3434 + - tests/v1/kv_connector/nixl_integration/
3435 + - vllm/platforms/rocm.py
3436 + commands:
3437 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
3438 + - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
3439 +
3440 +- label: ":amd: (MI355) Distributed NixlConnector PD accuracy" # TBD
3441 + timeout_in_minutes: 60
3442 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3443 + dind: false
3444 + agent_pool: mi355_4
3445 + num_gpus: 4
3446 + optional: true
3447 + working_dir: "/vllm-workspace/tests"
3448 + source_file_dependencies:
3449 + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
3450 + - tests/v1/kv_connector/nixl_integration/
3451 + - vllm/platforms/rocm.py
3452 + commands:
3453 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
3454 + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
3455 +
3456 +- label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" # TBD
3457 + timeout_in_minutes: 40
3458 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3459 + dind: false
3460 + agent_pool: mi355_4
3461 + num_gpus: 4
3462 + optional: true
3463 + working_dir: "/vllm-workspace/tests"
3464 + source_file_dependencies:
3465 + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
3466 + - tests/v1/kv_connector/nixl_integration/
3467 + - vllm/platforms/rocm.py
3468 + commands:
3469 + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
3470 + - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
3471 +
3472 +- label: ":amd: (MI355) Distributed Features" # TBD
3473 + timeout_in_minutes: 90
3474 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3475 + dind: false
3476 + agent_pool: mi355_2
3477 num_gpus: 2
3478 optional: true
3479 working_dir: "/vllm-workspace/"
@@ -3191,18 +3504,28 @@ steps:
3504 - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
3505 - tests/v1/distributed/test_dbo.py
3506 - tests/utils.py
3507 + - examples/features/data_parallel/data_parallel_offline.py
3508 + - examples/rl/rlhf_async_new_apis.py
3509 + - tests/distributed/test_weight_transfer.py
3510 + - tests/distributed/test_packed_tensor.py
3511 commands:
3512 - pytest -v -s tests/distributed/test_context_parallel.py
3513 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
3514 + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
3515 + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
3516 + - pytest -v -s tests/v1/distributed/test_dbo.py
3517 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
3518 + - pytest -v -s tests/distributed/test_packed_tensor.py
3519 - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py
3520 - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
3521 - pytest -v -s tests/distributed/test_rocm_quick_reduce.py
3522 - pytest -v -s tests/distributed/test_quick_all_reduce.py
3200 - - pytest -v -s tests/v1/distributed/test_dbo.py
3523
3524 #-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------#
3525
3204 -- label: ":amd: (MI355) Entrypoints Integration (API Server)" # TBD
3205 - timeout_in_minutes: 55
3526 +- label: ":amd: (MI355) Entrypoints Integration (API Server) %N"
3527 + timeout_in_minutes: 65
3528 + parallelism: 4
3529 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3530 dind: false
3531 agent_pool: mi355_1
@@ -3215,12 +3538,12 @@ steps:
3538 - tests/entrypoints/scale_out
3539 commands:
3540 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3218 - - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
3219 - - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
3220 - - pytest -v -s entrypoints/scale_out
3541 + - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
3542 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi
3543 + - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
3544
3545 - label: ":amd: (MI355) Entrypoints Integration (OpenAI API completion)" # TBD
3223 - timeout_in_minutes: 35
3546 + timeout_in_minutes: 65
3547 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3548 dind: false
3549 agent_pool: mi355_1
@@ -3236,7 +3559,7 @@ steps:
3559 - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
3560
3561 - label: ":amd: (MI355) Entrypoints Integration (OpenAI API chat_completion)" # TBD
3239 - timeout_in_minutes: 50
3562 + timeout_in_minutes: 70
3563 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3564 dind: false
3565 agent_pool: mi355_1
@@ -3251,7 +3574,7 @@ steps:
3574 - pytest -v -s entrypoints/openai/chat_completion
3575
3576 - label: ":amd: (MI355) Entrypoints Integration (API Server Generate)" # TBD
3254 - timeout_in_minutes: 40
3577 + timeout_in_minutes: 65
3578 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3579 dind: false
3580 agent_pool: mi355_1
@@ -3264,15 +3587,16 @@ steps:
3587 - tests/entrypoints/tool_parsers
3588 - tests/entrypoints/anthropic
3589 - tests/entrypoints/generate
3590 + - tests/entrypoints/cohere
3591 commands:
3268 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3592 - pytest -v -s tool_use
3593 - pytest -v -s entrypoints/tool_parsers
3594 - pytest -v -s entrypoints/generate
3595 - pytest -v -s entrypoints/anthropic
3596 + - pytest -v -s entrypoints/cohere
3597
3598 - label: ":amd: (MI355) Entrypoints Integration (Speech to Text)" # TBD
3275 - timeout_in_minutes: 50
3599 + timeout_in_minutes: 60
3600 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355]
3601 dind: false
3602 agent_pool: mi355_1
@@ -3281,12 +3605,13 @@ steps:
3605 source_file_dependencies:
3606 - vllm/
3607 - tests/entrypoints/speech_to_text
3608 + optional: true
3609 commands:
3610 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3611 - pytest -v -s entrypoints/speech_to_text
3612
3613 - label: ":amd: (MI355) Entrypoints Integration (Multimodal)"
3289 - timeout_in_minutes: 40
3614 + timeout_in_minutes: 55
3615 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355]
3616 dind: false
3617 agent_pool: mi355_1
@@ -3295,12 +3620,13 @@ steps:
3620 source_file_dependencies:
3621 - vllm/
3622 - tests/entrypoints/multimodal
3623 + optional: true
3624 commands:
3625 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3626 - pytest -v -s entrypoints/multimodal
3627
3628 - label: ":amd: (MI355) Entrypoints Integration (Pooling)" # TBD
3303 - timeout_in_minutes: 55
3629 + timeout_in_minutes: 65
3630 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3631 dind: false
3632 agent_pool: mi355_1
@@ -3309,6 +3635,7 @@ steps:
3635 source_file_dependencies:
3636 - vllm/
3637 - tests/entrypoints/pooling
3638 + optional: true
3639 commands:
3640 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
3641 - pytest -v -s entrypoints/pooling
@@ -3361,8 +3688,8 @@ steps:
3688 commands:
3689 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
3690
3364 -- label: ":amd: (MI355) LM Eval Small Models" # TBD
3365 - timeout_in_minutes: 45
3691 +- label: ":amd: (MI355) LM Eval Small Models Distributed"
3692 + timeout_in_minutes: 55
3693 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3694 dind: false
3695 agent_pool: mi355_2
@@ -3378,6 +3705,7 @@ steps:
3705 - vllm/v1/attention/selector.py
3706 - vllm/_aiter_ops.py
3707 - vllm/platforms/rocm.py
3708 + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt
3709 commands:
3710 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
3711
@@ -3424,7 +3752,7 @@ steps:
3752 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
3753
3754 - label: ":amd: (MI355) LM Eval Large Models" # TBD
3427 - timeout_in_minutes: 35
3755 + timeout_in_minutes: 65
3756 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3757 dind: false
3758 agent_pool: mi355_8
@@ -3449,7 +3777,7 @@ steps:
3777 #--------------------------------------------------------- mi355 · examples ----------------------------------------------------------#
3778
3779 - label: ":amd: (MI355) Examples" # TBD
3452 - timeout_in_minutes: 40
3780 + timeout_in_minutes: 75
3781 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3782 dind: false
3783 agent_pool: mi355_1
@@ -3459,23 +3787,24 @@ steps:
3787 - vllm/multimodal
3788 - examples/
3789 - vllm/platforms/rocm.py
3790 + optional: true
3791 commands:
3463 - - pip install tensorizer
3792 # Basic
3793 + # Multi-modal models
3794 + # Pooling models
3795 + # Features demo
3796 + - pip install --no-deps tensorizer
3797 - python3 basic/offline_inference/chat.py
3798 - python3 basic/offline_inference/generate.py --model facebook/opt-125m
3799 - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
3800 - python3 basic/offline_inference/classify.py
3801 - python3 basic/offline_inference/embed.py
3802 - python3 basic/offline_inference/score.py
3471 - # Multi-modal models
3803 - python3 generate/multimodal/audio_language_offline.py --seed 0
3804 - python3 generate/multimodal/vision_language_offline.py --seed 0
3805 - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
3806 - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
3476 - # Pooling models
3807 - python3 pooling/embed/vision_embedding_offline.py --seed 0
3478 - # Features demo
3808 - python3 features/automatic_prefix_caching/prefix_caching_offline.py
3809 - python3 deployment/llm_engine_example.py
3810 - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
@@ -3502,8 +3831,10 @@ steps:
3831 - vllm/model_executor/layers/rotary_embedding/
3832 - vllm/_aiter_ops.py
3833 - vllm/platforms/rocm.py
3834 + - tests/kernels/test_fused_qk_norm_rope_gate.py
3835 + optional: true
3836 commands:
3506 - - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
3837 + - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
3838
3839 - label: ":amd: (MI355) MiniMax Reduce RMS Kernels" # TBD
3840 timeout_in_minutes: 45
@@ -3520,6 +3851,12 @@ steps:
3851 - vllm/platforms/rocm.py
3852 - tests/kernels/core/test_minimax_reduce_rms.py
3853 - tests/utils.py
3854 + - csrc/libtorch_stable/minimax_reduce_rms_kernel.cu
3855 + - csrc/libtorch_stable/minimax_reduce_rms_kernel.h
3856 + - csrc/libtorch_stable/ops.h
3857 + - csrc/libtorch_stable/torch_bindings.cpp
3858 + - vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py
3859 + optional: true
3860 commands:
3861 - pytest -v -s kernels/core/test_minimax_reduce_rms.py
3862
@@ -3543,8 +3880,12 @@ steps:
3880 - vllm/v1/attention/selector.py
3881 - vllm/platforms/rocm.py
3882 - vllm/_aiter_ops.py
3883 + - examples/basic/offline_inference/chat.py
3884 + - tests/kernels/attention/test_attention_selector.py
3885 + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
3886 + optional: true
3887 commands:
3547 - - rocm-smi
3888 + - amd-smi
3889 - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN
3890 - pytest -v -s tests/kernels/attention/test_attention_selector.py
3891 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
@@ -3564,15 +3905,16 @@ steps:
3905 - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
3906 - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
3907 - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
3908 + optional: true
3909 commands:
3568 - - rocm-smi
3910 + - amd-smi
3911 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
3912 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py
3913 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
3914 - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
3915
3916 - label: ":amd: (MI355) Attention Kernels Shard %N" # TBD
3575 - timeout_in_minutes: 40
3917 + timeout_in_minutes: 90
3918 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3919 dind: false
3920 agent_pool: mi355_1
@@ -3586,11 +3928,13 @@ steps:
3928 - vllm/_aiter_ops.py
3929 - vllm/envs.py
3930 - vllm/platforms/rocm.py
3931 + - vllm/utils/flashinfer.py
3932 + optional: true
3933 commands:
3934 - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
3935
3936 - label: ":amd: (MI355) MoE Kernels Shard %N" # TBD
3593 - timeout_in_minutes: 45
3937 + timeout_in_minutes: 55
3938 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3939 dind: false
3940 agent_pool: mi355_1
@@ -3610,6 +3954,7 @@ steps:
3954 - vllm/config
3955 - vllm/_aiter_ops.py
3956 - vllm/platforms/rocm.py
3957 + optional: true
3958 commands:
3959 - pytest -v -s kernels/moe
3960 --ignore=kernels/moe/test_modular_oai_triton_moe.py
@@ -3646,11 +3991,15 @@ steps:
3991 - vllm/utils/torch_utils.py
3992 - vllm/platforms/
3993 - vllm/_aiter_ops.py
3994 + - csrc/quantization/cutlass_w8a8/moe/
3995 + - vllm/distributed/device_communicators/
3996 + - vllm/config
3997 commands:
3998 - pytest -v -s kernels/moe/test_moe_layer.py
3999
3652 -- label: ":amd: (MI355) Quantization Kernels" # TBD
3653 - timeout_in_minutes: 50
4000 +- label: ":amd: (MI355) Quantization Kernels Shard %N"
4001 + timeout_in_minutes: 120
4002 + parallelism: 6
4003 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4004 dind: false
4005 agent_pool: mi355_1
@@ -3669,8 +4018,11 @@ steps:
4018 - vllm/platforms/rocm.py
4019 - vllm/model_executor/kernels/
4020 - vllm/v1/attention/backends/rocm_aiter_fa.py
4021 + - vllm/config/
4022 + - tests/kernels/quantization/test_rocm_skinny_gemms.py
4023 + optional: true
4024 commands:
3673 - - pytest -v -s kernels/quantization
4025 + - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
4026
4027 - label: ":amd: (MI355) FP8 MoE Kernels" # TBD
4028 timeout_in_minutes: 40
@@ -3685,6 +4037,13 @@ steps:
4037 - vllm/_aiter_ops.py
4038 - vllm/platforms/rocm.py
4039 - vllm/envs.py
4040 + - tests/kernels/moe/test_gpt_oss_triton_kernels.py
4041 + - tests/kernels/moe/test_modular_oai_triton_moe.py
4042 + - tests/kernels/moe/test_moe.py
4043 + - tests/kernels/moe/test_block_int8.py
4044 + - tests/kernels/moe/test_triton_moe_no_act_mul.py
4045 + - tests/kernels/moe/test_triton_moe_ptpc_fp8.py
4046 + optional: true
4047 commands:
4048 - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py
4049 - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py
@@ -3694,7 +4053,7 @@ steps:
4053 - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
4054
4055 - label: ":amd: (MI355) DeepEP FP8 MoE Kernels" # TBD
3697 - timeout_in_minutes: 45
4056 + timeout_in_minutes: 55
4057 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4058 dind: false
4059 agent_pool: mi355_2
@@ -3710,12 +4069,51 @@ steps:
4069 - vllm/platforms/rocm.py
4070 - vllm/envs.py
4071 commands:
3713 - - pytest -v -s kernels/moe/test_deepep_moe.py
3714 -
3715 -#----------------------------------------------------- mi355 · models / language -----------------------------------------------------#
4072 + - pytest -v -s kernels/moe/test_deepep_moe.py
4073 +
4074 +#------------------------------------------------------ mi355 · models / basic -------------------------------------------------------#
4075 +
4076 +- label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N" # TBD
4077 + timeout_in_minutes: 60
4078 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4079 + dind: false
4080 + agent_pool: mi355_1
4081 + parallelism: 6
4082 + working_dir: "/vllm-workspace/tests"
4083 + source_file_dependencies:
4084 + - vllm/model_executor/models/
4085 + - vllm/model_executor/layers/
4086 + - tests/models/test_initialization.py
4087 + - tests/models/registry.py
4088 + - vllm/_aiter_ops.py
4089 + - vllm/platforms/rocm.py
4090 + optional: true
4091 + commands:
4092 + - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
4093 +
4094 +#----------------------------------------------------- mi355 · models / language -----------------------------------------------------#
4095 +
4096 +- label: ":amd: (MI355) Language Models (Hybrid) Shard %N" # TBD
4097 + timeout_in_minutes: 65
4098 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4099 + dind: false
4100 + agent_pool: mi355_1
4101 + parallelism: 2
4102 + optional: true
4103 + working_dir: "/vllm-workspace/tests"
4104 + source_file_dependencies:
4105 + - vllm/
4106 + - tests/models/language/generation
4107 + commands:
4108 + - export GIT_CONFIG_COUNT=1
4109 + - export GIT_CONFIG_KEY_0=http.version
4110 + - export GIT_CONFIG_VALUE_0=HTTP/1.1
4111 + - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
4112 + - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
4113 + - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
4114
4115 - label: ":amd: (MI355) Language Models (Extended Generation)" # TBD
3718 - timeout_in_minutes: 65
4116 + timeout_in_minutes: 95
4117 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4118 dind: false
4119 agent_pool: mi355_1
@@ -3723,13 +4121,15 @@ steps:
4121 source_file_dependencies:
4122 - vllm/
4123 - tests/models/language/generation
4124 + optional: true
4125 commands:
4126 - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@rocm-7.0-v2.3.0'
4127 - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
4128 - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
4129
3731 -- label: ":amd: (MI355) Language Models (Extended Pooling)" # TBD
3732 - timeout_in_minutes: 65
4130 +- label: ":amd: (MI355) Language Models (Extended Pooling) Shard %N"
4131 + timeout_in_minutes: 95
4132 + parallelism: 4
4133 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4134 dind: false
4135 agent_pool: mi355_1
@@ -3739,10 +4139,10 @@ steps:
4139 - vllm/
4140 - tests/models/language/pooling
4141 commands:
3742 - - pytest -v -s models/language/pooling -m 'not core_model'
4142 + - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
4143
4144 - label: ":amd: (MI355) Language Models (PPL)" # TBD
3745 - timeout_in_minutes: 35
4145 + timeout_in_minutes: 65
4146 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4147 dind: false
4148 agent_pool: mi355_1
@@ -3767,6 +4167,8 @@ steps:
4167 - vllm/v1/attention/ops/
4168 - vllm/platforms/rocm.py
4169 - tests/models/language/generation_ppl_test
4170 + - vllm/model_executor/layers/fla/ops/
4171 + - vllm/
4172 commands:
4173 - pytest -v -s models/language/generation_ppl_test
4174
@@ -3779,6 +4181,7 @@ steps:
4181 source_file_dependencies:
4182 - vllm/
4183 - tests/models/language
4184 + optional: true
4185 commands:
4186 - pip freeze | grep -E 'torch'
4187 - pytest -v -s models/language -m 'core_model and (not slow_test)'
@@ -3786,7 +4189,7 @@ steps:
4189 #---------------------------------------------------- mi355 · models / multimodal ----------------------------------------------------#
4190
4191 - label: ":amd: (MI355) Multimodal Models (Extended Generation 1)" # TBD
3789 - timeout_in_minutes: 60
4192 + timeout_in_minutes: 90
4193 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4194 dind: false
4195 agent_pool: mi355_1
@@ -3801,7 +4204,7 @@ steps:
4204 - pytest -v -s models/multimodal/test_mapping.py
4205
4206 - label: ":amd: (MI355) Multimodal Models (Extended Generation 3)" # TBD
3804 - timeout_in_minutes: 40
4207 + timeout_in_minutes: 90
4208 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4209 dind: false
4210 agent_pool: mi355_1
@@ -3814,7 +4217,7 @@ steps:
4217 - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
4218
4219 - label: ":amd: (MI355) Multimodal Models (Extended Pooling)" # TBD
3817 - timeout_in_minutes: 40
4220 + timeout_in_minutes: 75
4221 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4222 dind: false
4223 agent_pool: mi355_1
@@ -3827,7 +4230,7 @@ steps:
4230 - pytest -v -s models/multimodal/pooling -m 'not core_model'
4231
4232 - label: ":amd: (MI355) Multimodal Models (Standard) 1: qwen2" # TBD
3830 - timeout_in_minutes: 35
4233 + timeout_in_minutes: 65
4234 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4235 dind: false
4236 agent_pool: mi355_1
@@ -3841,7 +4244,7 @@ steps:
4244 - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
4245
4246 - label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper" # TBD
3844 - timeout_in_minutes: 55
4247 + timeout_in_minutes: 70
4248 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4249 dind: false
4250 agent_pool: mi355_1
@@ -3853,13 +4256,13 @@ steps:
4256 commands:
4257 - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
4258 - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
3856 - - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
4259 + - pytest models/multimodal/generation/test_memory_leak.py -m core_model
4260 - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
4261
4262 #----------------------------------------------------- mi355 · models / quantized -----------------------------------------------------#
4263
4264 - label: ":amd: (MI355) Quantized Models" # TBD
3862 - timeout_in_minutes: 50
4265 + timeout_in_minutes: 80
4266 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4267 dind: false
4268 agent_pool: mi355_1
@@ -3870,13 +4273,16 @@ steps:
4273 - vllm/_aiter_ops.py
4274 - vllm/platforms/rocm.py
4275 - vllm/model_executor/model_loader/
4276 + optional: true
4277 commands:
4278 + - unset VLLM_USE_V2_MODEL_RUNNER
4279 - pytest -v -s models/quantization
4280
4281 #------------------------------------------------------- mi355 · quantization --------------------------------------------------------#
4282
3878 -- label: ":amd: (MI355) Quantization" # TBD
3879 - timeout_in_minutes: 80
4283 +- label: ":amd: (MI355) Quantization Shard %N"
4284 + timeout_in_minutes: 115
4285 + parallelism: 4
4286 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4287 dind: false
4288 agent_pool: mi355_1
@@ -3887,10 +4293,16 @@ steps:
4293 - tests/quantization
4294 - vllm/_aiter_ops.py
4295 - vllm/platforms/rocm.py
4296 + - vllm/model_executor/layers/fused_moe/oracle/unquantized.py
4297 + - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py
4298 + - tests/rocm/test_moe_weight_replay.py
4299 + optional: true
4300 commands:
4301 + - unset VLLM_USE_V2_MODEL_RUNNER
4302 - uv pip install --system torchao==0.17.0
4303 - uv pip install --system conch-triton-kernels
3893 - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
4304 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
4305 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
4306
4307 # - label: Quantized MoE Test (B200-MI355) # TBD
4308 # timeout_in_minutes: 180
@@ -3922,7 +4334,7 @@ steps:
4334 #------------------------------------------------------------ mi355 · v1 -------------------------------------------------------------#
4335
4336 - label: ":amd: (MI355) V1 Attention Shard %N" # TBD
3925 - timeout_in_minutes: 35
4337 + timeout_in_minutes: 125
4338 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4339 dind: false
4340 agent_pool: mi355_1
@@ -3936,11 +4348,12 @@ steps:
4348 - vllm/_aiter_ops.py
4349 - vllm/envs.py
4350 - vllm/platforms/rocm.py
4351 + optional: true
4352 commands:
4353 - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
4354
3942 -- label: ":amd: (MI355) V1 Core + KV + Metrics" # TBD
3943 - timeout_in_minutes: 55
4355 +- label: ":amd: (MI355) V1 Core"
4356 + timeout_in_minutes: 60
4357 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4358 dind: false
4359 agent_pool: mi355_1
@@ -3956,24 +4369,243 @@ steps:
4369 - tests/v1/kv_connector/unit
4370 - tests/v1/metrics
4371 - tests/entrypoints/openai/correctness/test_lmeval.py
4372 + - vllm/config/
4373 + - vllm/distributed/
4374 + - vllm/engine/
4375 + - vllm/entrypoints/pooling/
4376 + - vllm/inputs/
4377 + - vllm/lora/
4378 + - vllm/model_executor/
4379 + - vllm/multimodal/
4380 + - vllm/outputs.py
4381 + - vllm/platforms/
4382 + - vllm/pooling_params.py
4383 + - vllm/profiler/
4384 + - vllm/sampling_params.py
4385 + - vllm/tokenizers/
4386 + - vllm/transformers_utils/
4387 + - vllm/utils/
4388 + - vllm/v1/
4389 + - tests/v1/simple_kv_offload
4390 + - tests/v1/streaming_input
4391 + - tests/v1/ec_connector/unit
4392 commands:
3960 - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
4393 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4394 - pytest -v -s -m 'not cpu_test' v1/core
4395 +
4396 +- label: ":amd: (MI355) V1 Executor + Worker"
4397 + timeout_in_minutes: 60
4398 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4399 + dind: false
4400 + agent_pool: mi355_1
4401 + optional: true
4402 + working_dir: /vllm-workspace/tests
4403 + source_file_dependencies:
4404 + - vllm/
4405 + - tests/v1/core
4406 + - tests/v1/executor
4407 + - tests/v1/kv_offload
4408 + - tests/v1/worker
4409 + - tests/v1/cudagraph
4410 + - tests/v1/kv_connector/unit
4411 + - tests/v1/metrics
4412 + - tests/entrypoints/openai/correctness/test_lmeval.py
4413 + - vllm/config/
4414 + - vllm/distributed/
4415 + - vllm/engine/
4416 + - vllm/entrypoints/pooling/
4417 + - vllm/inputs/
4418 + - vllm/lora/
4419 + - vllm/model_executor/
4420 + - vllm/multimodal/
4421 + - vllm/outputs.py
4422 + - vllm/platforms/
4423 + - vllm/pooling_params.py
4424 + - vllm/profiler/
4425 + - vllm/sampling_params.py
4426 + - vllm/tokenizers/
4427 + - vllm/transformers_utils/
4428 + - vllm/utils/
4429 + - vllm/v1/
4430 + - tests/v1/simple_kv_offload
4431 + - tests/v1/streaming_input
4432 + - tests/v1/ec_connector/unit
4433 + commands:
4434 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4435 - pytest -v -s v1/executor
3963 - - pytest -v -s v1/kv_offload
4436 - pytest -v -s v1/worker
3965 - - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
3966 - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
4437 + - pytest -v -s v1/streaming_input
4438 +
4439 +- label: ":amd: (MI355) V1 KV Offload"
4440 + timeout_in_minutes: 60
4441 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4442 + dind: false
4443 + agent_pool: mi355_1
4444 + optional: true
4445 + working_dir: /vllm-workspace/tests
4446 + source_file_dependencies:
4447 + - vllm/
4448 + - tests/v1/core
4449 + - tests/v1/executor
4450 + - tests/v1/kv_offload
4451 + - tests/v1/worker
4452 + - tests/v1/cudagraph
4453 + - tests/v1/kv_connector/unit
4454 + - tests/v1/metrics
4455 + - tests/entrypoints/openai/correctness/test_lmeval.py
4456 + - vllm/config/
4457 + - vllm/distributed/
4458 + - vllm/engine/
4459 + - vllm/entrypoints/pooling/
4460 + - vllm/inputs/
4461 + - vllm/lora/
4462 + - vllm/model_executor/
4463 + - vllm/multimodal/
4464 + - vllm/outputs.py
4465 + - vllm/platforms/
4466 + - vllm/pooling_params.py
4467 + - vllm/profiler/
4468 + - vllm/sampling_params.py
4469 + - vllm/tokenizers/
4470 + - vllm/transformers_utils/
4471 + - vllm/utils/
4472 + - vllm/v1/
4473 + - tests/v1/simple_kv_offload
4474 + - tests/v1/streaming_input
4475 + - tests/v1/ec_connector/unit
4476 + commands:
4477 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4478 + - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh
4479 + - pytest -v -s v1/kv_offload
4480 + - pytest -v -s v1/simple_kv_offload
4481 +
4482 +- label: ":amd: (MI355) V1 KV Connectors Shard %N"
4483 + timeout_in_minutes: 60
4484 + parallelism: 4
4485 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4486 + dind: false
4487 + agent_pool: mi355_1
4488 + optional: true
4489 + working_dir: /vllm-workspace/tests
4490 + source_file_dependencies:
4491 + - vllm/
4492 + - tests/v1/core
4493 + - tests/v1/executor
4494 + - tests/v1/kv_offload
4495 + - tests/v1/worker
4496 + - tests/v1/cudagraph
4497 + - tests/v1/kv_connector/unit
4498 + - tests/v1/metrics
4499 + - tests/entrypoints/openai/correctness/test_lmeval.py
4500 + - vllm/config/
4501 + - vllm/distributed/
4502 + - vllm/engine/
4503 + - vllm/entrypoints/pooling/
4504 + - vllm/inputs/
4505 + - vllm/lora/
4506 + - vllm/model_executor/
4507 + - vllm/multimodal/
4508 + - vllm/outputs.py
4509 + - vllm/platforms/
4510 + - vllm/pooling_params.py
4511 + - vllm/profiler/
4512 + - vllm/sampling_params.py
4513 + - vllm/tokenizers/
4514 + - vllm/transformers_utils/
4515 + - vllm/utils/
4516 + - vllm/v1/
4517 + - tests/v1/simple_kv_offload
4518 + - tests/v1/streaming_input
4519 + - tests/v1/ec_connector/unit
4520 + commands:
4521 + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
4522 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4523 + - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
4524 + - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
4525 +
4526 +- label: ":amd: (MI355) V1 Metrics + LM Eval"
4527 + timeout_in_minutes: 65
4528 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4529 + dind: false
4530 + agent_pool: mi355_1
4531 + optional: true
4532 + working_dir: /vllm-workspace/tests
4533 + source_file_dependencies:
4534 + - vllm/
4535 + - tests/v1/core
4536 + - tests/v1/executor
4537 + - tests/v1/kv_offload
4538 + - tests/v1/worker
4539 + - tests/v1/cudagraph
4540 + - tests/v1/kv_connector/unit
4541 + - tests/v1/metrics
4542 + - tests/entrypoints/openai/correctness/test_lmeval.py
4543 + - vllm/config/
4544 + - vllm/distributed/
4545 + - vllm/engine/
4546 + - vllm/entrypoints/pooling/
4547 + - vllm/inputs/
4548 + - vllm/lora/
4549 + - vllm/model_executor/
4550 + - vllm/multimodal/
4551 + - vllm/outputs.py
4552 + - vllm/platforms/
4553 + - vllm/pooling_params.py
4554 + - vllm/profiler/
4555 + - vllm/sampling_params.py
4556 + - vllm/tokenizers/
4557 + - vllm/transformers_utils/
4558 + - vllm/utils/
4559 + - vllm/v1/
4560 + - tests/v1/simple_kv_offload
4561 + - tests/v1/streaming_input
4562 + - tests/v1/ec_connector/unit
4563 + commands:
4564 + - export GIT_CONFIG_COUNT=1
4565 + - export GIT_CONFIG_KEY_0=http.version
4566 + - export GIT_CONFIG_VALUE_0=HTTP/1.1
4567 + - export GIT_TERMINAL_PROMPT=0
4568 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4569 - pytest -v -s -m 'not cpu_test' v1/metrics
4570 - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api
4571 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
4572
3971 -- label: ":amd: (MI355) V1 Sample + Logits" # TBD
4573 +- label: ":amd: (MI355) CUDAGraph"
4574 timeout_in_minutes: 55
4575 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4576 dind: false
4577 agent_pool: mi355_1
4578 optional: true
4579 + working_dir: /vllm-workspace/tests
4580 + source_file_dependencies:
4581 + - vllm/
4582 + - tests/v1/core
4583 + - tests/v1/executor
4584 + - tests/v1/kv_offload
4585 + - tests/v1/worker
4586 + - tests/v1/cudagraph
4587 + - tests/v1/kv_connector/unit
4588 + - tests/v1/metrics
4589 + - tests/entrypoints/openai/correctness/test_lmeval.py
4590 + - vllm/v1/cudagraph_dispatcher.py
4591 + - vllm/config/compilation.py
4592 + - vllm/compilation
4593 + - vllm/platforms/rocm.py
4594 + - vllm/v1/worker/encoder_cudagraph.py
4595 + - vllm/v1/worker/encoder_cudagraph_defs.py
4596 + commands:
4597 + - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
4598 + - pytest -v -s v1/cudagraph/test_cudagraph_mode.py
4599 + - pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
4600 + - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
4601 +
4602 +
4603 +- label: ":amd: (MI355) V1 Sample"
4604 + timeout_in_minutes: 70
4605 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4606 + dind: false
4607 + agent_pool: mi355_1
4608 + optional: true
4609 working_dir: "/vllm-workspace/tests"
4610 source_file_dependencies:
4611 - vllm/
@@ -3982,15 +4614,56 @@ steps:
4614 - tests/v1/test_oracle.py
4615 - tests/v1/test_request.py
4616 - tests/v1/test_outputs.py
4617 + - vllm/config/
4618 + - vllm/distributed/
4619 + - vllm/engine/
4620 + - vllm/inputs/
4621 + - vllm/logger.py
4622 + - vllm/model_executor/
4623 + - vllm/platforms/
4624 + - vllm/sampling_params.py
4625 + - vllm/transformers_utils/
4626 + - vllm/utils/
4627 + - vllm/v1/
4628 commands:
4629 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4630 - pytest -v -s v1/sample
4631 +
4632 +- label: ":amd: (MI355) V1 Logits + Oracle"
4633 + timeout_in_minutes: 70
4634 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4635 + dind: false
4636 + agent_pool: mi355_1
4637 + optional: true
4638 + working_dir: /vllm-workspace/tests
4639 + source_file_dependencies:
4640 + - vllm/
4641 + - tests/v1/sample
4642 + - tests/v1/logits_processors
4643 + - tests/v1/test_oracle.py
4644 + - tests/v1/test_request.py
4645 + - tests/v1/test_outputs.py
4646 + - vllm/config/
4647 + - vllm/distributed/
4648 + - vllm/engine/
4649 + - vllm/inputs/
4650 + - vllm/logger.py
4651 + - vllm/model_executor/
4652 + - vllm/platforms/
4653 + - vllm/sampling_params.py
4654 + - vllm/transformers_utils/
4655 + - vllm/utils/
4656 + - vllm/v1/
4657 + commands:
4658 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4659 - pytest -v -s v1/logits_processors
4660 - pytest -v -s v1/test_oracle.py
4661 - pytest -v -s v1/test_request.py
4662 - pytest -v -s v1/test_outputs.py
4663
4664 +
4665 - label: ":amd: (MI355) V1 Spec Decode" # TBD
3993 - timeout_in_minutes: 45
4666 + timeout_in_minutes: 50
4667 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4668 dind: false
4669 agent_pool: mi355_1
@@ -3998,11 +4671,24 @@ steps:
4671 source_file_dependencies:
4672 - vllm/
4673 - tests/v1/spec_decode
4674 + - vllm/config/
4675 + - vllm/distributed/
4676 + - vllm/inputs/
4677 + - vllm/model_executor/
4678 + - vllm/models/qwen4_exp/
4679 + - vllm/platforms/
4680 + - vllm/sampling_params.py
4681 + - vllm/transformers_utils/
4682 + - vllm/utils/
4683 + - vllm/v1/
4684 + optional: true
4685 commands:
4686 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
4687 - pytest -v -s -m 'not slow_test' v1/spec_decode
4688
4004 -- label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance" # TBD
4005 - timeout_in_minutes: 180
4689 +- label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly"
4690 + timeout_in_minutes: 55
4691 + parallelism: 3
4692 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4693 dind: false
4694 agent_pool: mi355_1
@@ -4026,13 +4712,17 @@ steps:
4712 - tests/utils.py
4713 - tests/v1/e2e/spec_decode/
4714 - vllm/platforms/rocm.py
4715 + - vllm/v1/worker/gpu/spec_decode/
4716 + - tests/v1/e2e/spec_decode/conftest.py
4717 commands:
4030 - - pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/
4718 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
4719 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi
4720 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
4721
4722 #------------------------------------------------------ mi355 · weight_loading -------------------------------------------------------#
4723
4724 - label: ":amd: (MI355) Weight Loading Multi-GPU" # TBD
4035 - timeout_in_minutes: 40
4725 + timeout_in_minutes: 70
4726 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
4727 dind: false
4728 agent_pool: mi355_2
@@ -4041,6 +4731,7 @@ steps:
4731 source_file_dependencies:
4732 - vllm/
4733 - tests/weight_loading
4734 + optional: true
4735 commands:
4736 - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
4737
@@ -4070,6 +4761,17 @@ steps:
4761 source_file_dependencies:
4762 - vllm/
4763 - tests/test_regression
4764 + - vllm/config/
4765 + - vllm/distributed/
4766 + - vllm/engine/
4767 + - vllm/inputs/
4768 + - vllm/model_executor/
4769 + - vllm/multimodal/
4770 + - vllm/platforms/
4771 + - vllm/sampling_params.py
4772 + - vllm/transformers_utils/
4773 + - vllm/utils/
4774 + - vllm/v1/
4775 commands:
4776 - pip install 'modelscope<1.38'
4777 - pytest -v -s test_regression.py
.buildkite/test_areas/compile.yaml
+2 -2
@@ -92,7 +92,7 @@ steps:
92 dind: false
93 device: mi355_1
94 working_dir: "/vllm-workspace/"
95 - timeout_in_minutes: 25
95 + timeout_in_minutes: 30
96 depends_on:
97 - image-build-amd
98 source_file_dependencies:
@@ -133,7 +133,7 @@ steps:
133 dind: false
134 device: mi355_1
135 working_dir: "/vllm-workspace/"
136 - timeout_in_minutes: 30
136 + timeout_in_minutes: 40
137 depends_on:
138 - image-build-amd
139 source_file_dependencies:
.buildkite/test_areas/cuda.yaml
+2 -2
@@ -23,7 +23,7 @@ steps:
23 dind: false
24 device: mi355_1
25 working_dir: "/vllm-workspace/tests"
26 - timeout_in_minutes: 20
26 + timeout_in_minutes: 25
27 depends_on:
28 - image-build-amd
29 source_file_dependencies:
@@ -50,7 +50,7 @@ steps:
50 amd:
51 label: ":amd: (MI250) CUDAGraph"
52 device: mi250_1
53 - timeout_in_minutes: 45
53 + timeout_in_minutes: 55
54 depends_on:
55 - image-build-amd
56 source_file_dependencies:
.buildkite/test_areas/disaggregated.yaml
+7 -7
@@ -16,9 +16,9 @@ steps:
16 - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
17 mirror:
18 amd:
19 - label: ":amd: (MI300) Distributed NixlConnector PD accuracy"
19 + label: ":amd: (MI355) Distributed NixlConnector PD accuracy"
20 dind: false
21 - device: mi300_4
21 + device: mi355_4
22 timeout_in_minutes: 60
23 depends_on:
24 - image-build-amd
@@ -71,9 +71,9 @@ steps:
71 - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
72 mirror:
73 amd:
74 - label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy"
74 + label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy"
75 dind: false
76 - device: mi300_4
76 + device: mi355_4
77 timeout_in_minutes: 40
78 depends_on:
79 - image-build-amd
@@ -130,7 +130,7 @@ steps:
130 label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy"
131 dind: false
132 device: mi300_4
133 - timeout_in_minutes: 55
133 + timeout_in_minutes: 60
134 depends_on:
135 - image-build-amd
136 source_file_dependencies:
@@ -221,9 +221,9 @@ steps:
221 - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
222 mirror:
223 amd:
224 - label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance"
224 + label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance"
225 dind: false
226 - device: mi300_2
226 + device: mi355_2
227 timeout_in_minutes: 45
228 depends_on:
229 - image-build-amd
.buildkite/test_areas/distributed.yaml
+56 -2
@@ -65,9 +65,9 @@ steps:
65 - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py
66 mirror:
67 amd:
68 - label: ":amd: (MI300) Distributed DP Basic"
68 + label: ":amd: (MI355) Distributed DP Basic"
69 dind: false
70 - device: mi300_2
70 + device: mi355_2
71 timeout_in_minutes: 45
72 depends_on:
73 - image-build-amd
@@ -125,6 +125,9 @@ steps:
125 - tests/compile/test_wrapper.py
126 - tests/entrypoints/llm/test_collective_rpc.py
127 - vllm/platforms/rocm.py
128 + commands:
129 + - pytest -v -s entrypoints/llm/test_collective_rpc.py
130 + - pytest -v -s ./compile/test_wrapper.py
131
132 - label: ":nvidia: (L4) Distributed Torchrun + Shutdown"
133 key: distributed-torchrun-shutdown-tests-2-gpus
@@ -296,6 +299,57 @@ steps:
299 - pytest -v -s tests/v1/distributed/test_dbo.py
300 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
301 - pytest -v -s tests/distributed/test_packed_tensor.py
302 + mirror:
303 + amd:
304 + label: ':amd: (MI355) Distributed Features'
305 + dind: false
306 + device: mi355_2
307 + timeout_in_minutes: 90
308 + working_dir: /vllm-workspace/
309 + depends_on:
310 + - image-build-amd
311 + source_file_dependencies:
312 + - vllm/distributed/
313 + - vllm/v1/distributed/
314 + - vllm/model_executor/layers/fused_moe/
315 + - vllm/v1/attention/backends/
316 + - vllm/v1/attention/selector.py
317 + - tests/v1/distributed/test_dbo.py
318 + - tests/distributed/test_context_parallel.py
319 + - examples/features/data_parallel/data_parallel_offline.py
320 + - vllm/_aiter_ops.py
321 + - vllm/platforms/rocm.py
322 + - csrc/custom_quickreduce.cu
323 + - csrc/ops.h
324 + - csrc/torch_bindings.cpp
325 + - vllm/model_executor/layers/
326 + - vllm/entrypoints/llm.py
327 + - vllm/config/parallel.py
328 + - vllm/v1/engine/
329 + - vllm/v1/executor/
330 + - vllm/v1/worker/
331 + - vllm/_custom_ops.py
332 + - vllm/envs.py
333 + - tests/distributed/test_rocm_aiter_custom_ar.py
334 + - tests/distributed/test_rocm_quick_reduce.py
335 + - tests/distributed/test_quick_all_reduce.py
336 + - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
337 + - tests/utils.py
338 + - examples/rl/rlhf_async_new_apis.py
339 + - tests/distributed/test_weight_transfer.py
340 + - tests/distributed/test_packed_tensor.py
341 + commands:
342 + - pytest -v -s tests/distributed/test_context_parallel.py
343 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
344 + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
345 + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
346 + - pytest -v -s tests/v1/distributed/test_dbo.py
347 + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
348 + - pytest -v -s tests/distributed/test_packed_tensor.py
349 + - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py
350 + - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
351 + - pytest -v -s tests/distributed/test_rocm_quick_reduce.py
352 + - pytest -v -s tests/distributed/test_quick_all_reduce.py
353
354 - label: ":nvidia: (B200) Distributed"
355 key: distributed-tests-2xb200
.buildkite/test_areas/e2e_integration.yaml
+1 -1
@@ -57,7 +57,7 @@ steps:
57 dind: false
58 device: mi355_1
59 working_dir: "/vllm-workspace"
60 - timeout_in_minutes: 20
60 + timeout_in_minutes: 30
61 depends_on:
62 - image-build-amd
63 source_file_dependencies:
.buildkite/test_areas/engine.yaml
+4
@@ -56,6 +56,8 @@ steps:
56 timeout_in_minutes: 45
57 depends_on:
58 - image-build-amd
59 + source_file_dependencies:
60 + - vllm/platforms/rocm.py
61
62 - label: ":nvidia: (H200 MIG 18GB) E2E Scheduling"
63 key: e2e-scheduling-1-gpu
@@ -73,6 +75,8 @@ steps:
75 timeout_in_minutes: 55
76 depends_on:
77 - image-build-amd
78 + source_file_dependencies:
79 + - vllm/platforms/rocm.py
80
81 - label: ":nvidia: (H200 MIG 18GB) E2E Core"
82 device: h200_18gb
.buildkite/test_areas/entrypoints.yaml
+22 -2
@@ -21,9 +21,11 @@ steps:
21 label: ":amd: (MI355) Entrypoints Unit"
22 dind: false
23 device: mi355_1
24 - timeout_in_minutes: 35
24 + timeout_in_minutes: 50
25 depends_on:
26 - image-build-amd
27 + source_file_dependencies:
28 + - vllm/platforms/rocm.py
29
30 - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (LLM)"
31 device: h200_35gb
@@ -44,7 +46,7 @@ steps:
46 label: ":amd: (MI300) Entrypoints Integration (LLM)"
47 dind: false
48 device: mi300_1
47 - timeout_in_minutes: 55
49 + timeout_in_minutes: 90
50 depends_on:
51 - image-build-amd
52
@@ -176,6 +178,15 @@ steps:
178 commands:
179 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
180 - pytest -v -s entrypoints/speech_to_text
181 + mirror:
182 + amd:
183 + label: ":amd: (MI355) Entrypoints Integration (Speech to Text)"
184 + dind: false
185 + device: mi355_1
186 + timeout_in_minutes: 60
187 + working_dir: "/vllm-workspace/tests"
188 + depends_on:
189 + - image-build-amd
190
191 - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (Multimodal)"
192 device: h200_35gb
@@ -210,6 +221,15 @@ steps:
221 commands:
222 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
223 - pytest -v -s entrypoints/pooling
224 + mirror:
225 + amd:
226 + label: ":amd: (MI355) Entrypoints Integration (Pooling)"
227 + dind: false
228 + device: mi355_1
229 + timeout_in_minutes: 65
230 + working_dir: "/vllm-workspace/tests"
231 + depends_on:
232 + - image-build-amd
233
234 - label: ":nvidia: (H200 MIG 18GB) OpenAI API Correctness"
235 key: openai-api-correctness
.buildkite/test_areas/kernels.yaml
+185 -4
@@ -13,6 +13,22 @@ steps:
13 commands:
14 - pytest -v -s tests/ir
15 - pytest -v -s tests/kernels/ir
16 + mirror:
17 + amd:
18 + label: ":amd: (MI355) vLLM IR"
19 + dind: false
20 + device: mi355_1
21 + timeout_in_minutes: 50
22 + working_dir: "/vllm-workspace/"
23 + depends_on:
24 + - image-build-amd
25 + source_file_dependencies:
26 + - tests/ir/
27 + - tests/kernels/ir/
28 + - vllm/ir
29 + - vllm/kernels
30 + - vllm/_aiter_ops.py
31 + - vllm/platforms/rocm.py
32
33 - label: ":nvidia: (H200 MIG 35GB) Core Operation Kernels Shard %N"
34 device: h200_35gb
@@ -26,6 +42,29 @@ steps:
42 commands:
43 - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
44 parallelism: 3
45 + mirror:
46 + amd:
47 + label: ":amd: (MI355) Core Operation Kernels Shard %N"
48 + dind: false
49 + device: mi355_1
50 + timeout_in_minutes: 40
51 + working_dir: "/vllm-workspace/tests"
52 + depends_on:
53 + - image-build-amd
54 + source_file_dependencies:
55 + - csrc/
56 + - tests/kernels/core
57 + - tests/kernels/test_top_k_per_row.py
58 + - tests/kernels/test_concat_mla_q.py
59 + - tests/kernels/test_rocm_fp32_router_gemm.py
60 + - vllm/model_executor/layers/fused_moe/router/gate_linear.py
61 + - vllm/model_executor/layers/fused_moe/router/rocm_fp32_router_gemm.py
62 + - vllm/model_executor/layers/rotary_embedding/
63 + - vllm/_aiter_ops.py
64 + - vllm/platforms/rocm.py
65 + - tests/kernels/test_fused_qk_norm_rope_gate.py
66 + commands:
67 + - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
68
69 - label: ":nvidia: (H100) MiniMax Reduce RMS Kernels"
70 key: kernels-minimax-reduce-rms-test-2-gpus
@@ -145,6 +184,7 @@ steps:
184 dind: false
185 device: mi300_1
186 timeout_in_minutes: 90
187 + parallelism: 6
188 depends_on:
189 - image-build-amd
190 source_file_dependencies:
@@ -208,9 +248,6 @@ steps:
248 label: ":amd: (MI300) Quantization Kernels Shard %N"
249 dind: false
250 device: mi300_1
211 - # Pin the AMD mirror at its current 2 shards so raising NVIDIA
212 - # parallelism does not multiply AMD copies (needs ci-infra #473).
213 - parallelism: 2
251 timeout_in_minutes: 120
252 source_file_dependencies:
253 - csrc/quantization/
@@ -222,6 +259,12 @@ steps:
259 - vllm/kernels/aiter_ops.py
260 - vllm/platforms/rocm.py
261 - vllm/model_executor/kernels/
262 + - csrc/rocm/
263 + - vllm/v1/attention/backends/rocm_aiter_fa.py
264 + - tests/kernels/quant_utils.py
265 + - tests/kernels/utils.py
266 + - vllm/_custom_ops.py
267 + - vllm/envs.py
268 depends_on:
269 - image-build-amd
270
@@ -258,6 +301,10 @@ steps:
301 - vllm/config
302 - vllm/_aiter_ops.py
303 - vllm/platforms/rocm.py
304 + - csrc/rocm/
305 + - tests/kernels/utils.py
306 + - vllm/model_executor/layers/quantization/mxfp4.py
307 + - vllm/model_executor/layers/quantization/utils/quant_utils.py
308 depends_on:
309 - image-build-amd
310
@@ -278,6 +325,8 @@ steps:
325 timeout_in_minutes: 55
326 depends_on:
327 - image-build-amd
328 + source_file_dependencies:
329 + - vllm/platforms/rocm.py
330
331 - label: ":nvidia: (H100) DeepGEMM Kernels"
332 key: kernels-deepgemm-test-h100
@@ -463,6 +512,9 @@ steps:
512 timeout_in_minutes: 45
513 depends_on:
514 - image-build-amd
515 + source_file_dependencies:
516 + - vllm/platforms/rocm.py
517 + - scripts/benchmark_helion_kernels.py
518
519
520 - label: ":nvidia: (H100) FP8 MoE Kernels"
@@ -491,6 +543,25 @@ steps:
543 commands:
544 - pytest -v -s kernels/moe/test_deepep_deepgemm_moe.py
545 - pytest -v -s kernels/moe/test_deepep_moe.py
546 + mirror:
547 + amd:
548 + label: ":amd: (MI355) DeepEP FP8 MoE Kernels"
549 + dind: false
550 + device: mi355_2
551 + timeout_in_minutes: 55
552 + working_dir: "/vllm-workspace/tests"
553 + depends_on:
554 + - image-build-amd
555 + source_file_dependencies:
556 + - csrc/moe/
557 + - csrc/quantization/w8a8/cutlass/moe/
558 + - vllm/model_executor/layers/fused_moe/
559 + - tests/kernels/moe/test_deepep_moe.py
560 + - vllm/_aiter_ops.py
561 + - vllm/platforms/rocm.py
562 + - vllm/envs.py
563 + commands:
564 + - pytest -v -s kernels/moe/test_deepep_moe.py
565
566 - label: ":nvidia: (B200) FP4 MoE Kernels"
567 key: kernels-fp4-moe-test-b200
@@ -520,7 +591,30 @@ steps:
591 - vllm/config
592 commands:
593 - pytest -v -s kernels/moe/test_moe_layer.py
523 -
594 + mirror:
595 + amd:
596 + label: ":amd: (MI355) FusedMoE Layer Kernels"
597 + dind: false
598 + device: mi355_2
599 + timeout_in_minutes: 35
600 + working_dir: "/vllm-workspace/tests"
601 + depends_on:
602 + - image-build-amd
603 + source_file_dependencies:
604 + - csrc/moe/
605 + - csrc/rocm/
606 + - tests/kernels/moe
607 + - vllm/model_executor/layers/fused_moe/
608 + - vllm/model_executor/layers/quantization/
609 + - vllm/distributed/
610 + - vllm/config/
611 + - vllm/forward_context.py
612 + - vllm/v1/worker/workspace.py
613 + - vllm/utils/import_utils.py
614 + - vllm/utils/math_utils.py
615 + - vllm/utils/torch_utils.py
616 + - vllm/platforms/
617 + - vllm/_aiter_ops.py
618
619 - label: ":nvidia: (B200) FusedMoE Layer Kernels"
620 key: kernels-fusedmoe-layer-test-2-b200s
@@ -538,3 +632,90 @@ steps:
632 commands:
633 - pytest -v -s kernels/moe/test_moe_layer.py
634 - pytest -v -s kernels/moe/test_deepep_v2_moe.py
635 +
636 +- label: ":amd: (MI355) MLA Kernels"
637 + key: amd-mla-kernels-mi355
638 + dind: false
639 + device: mi355_1
640 + num_devices: 1
641 + timeout_in_minutes: 25
642 + working_dir: /vllm-workspace/
643 + depends_on:
644 + - image-build-amd
645 + source_file_dependencies:
646 + - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
647 + - vllm/v1/attention/selector.py
648 + - vllm/platforms/rocm.py
649 + - vllm/_aiter_ops.py
650 + - tests/kernels/attention/test_rocm_aiter_mla_decode.py
651 + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
652 + - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
653 + - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
654 + commands:
655 + - amd-smi
656 + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
657 + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py
658 + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
659 + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
660 +
661 +- label: ":amd: (MI355) FP8 MoE Kernels"
662 + key: amd-fp8-moe-kernels-mi355
663 + dind: false
664 + device: mi355_1
665 + num_devices: 1
666 + timeout_in_minutes: 40
667 + working_dir: /vllm-workspace/tests
668 + depends_on:
669 + - image-build-amd
670 + source_file_dependencies:
671 + - csrc/moe/
672 + - vllm/model_executor/layers/fused_moe/
673 + - tests/kernels/moe/test_deepep_moe.py
674 + - vllm/_aiter_ops.py
675 + - vllm/platforms/rocm.py
676 + - vllm/envs.py
677 + - tests/kernels/moe/test_gpt_oss_triton_kernels.py
678 + - tests/kernels/moe/test_modular_oai_triton_moe.py
679 + - tests/kernels/moe/test_moe.py
680 + - tests/kernels/moe/test_block_int8.py
681 + - tests/kernels/moe/test_triton_moe_no_act_mul.py
682 + - tests/kernels/moe/test_triton_moe_ptpc_fp8.py
683 + commands:
684 + - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py
685 + - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py
686 + - pytest -v -s kernels/moe/test_moe.py
687 + - pytest -v -s kernels/moe/test_block_int8.py
688 + - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
689 + - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
690 +
691 +- label: ":amd: (MI355) Kernels"
692 + key: amd-kernels-mi355
693 + dind: false
694 + device: mi355_1
695 + num_devices: 1
696 + timeout_in_minutes: 25
697 + working_dir: /vllm-workspace/
698 + depends_on:
699 + - image-build-amd
700 + source_file_dependencies:
701 + - csrc/quantization/fp4/
702 + - csrc/attention/mla/
703 + - csrc/quantization/cutlass_w8a8/moe/
704 + - vllm/model_executor/layers/fused_moe/cutlass_moe.py
705 + - vllm/v1/attention/backends/triton_attn.py
706 + - vllm/v1/attention/backends/rocm_attn.py
707 + - vllm/v1/attention/backends/rocm_aiter_fa.py
708 + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
709 + - vllm/v1/attention/backends/mla/aiter_triton_mla.py
710 + - vllm/v1/attention/backends/mla/rocm_aiter_mla.py
711 + - vllm/v1/attention/selector.py
712 + - vllm/platforms/rocm.py
713 + - vllm/_aiter_ops.py
714 + - examples/basic/offline_inference/chat.py
715 + - tests/kernels/attention/test_attention_selector.py
716 + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
717 + commands:
718 + - amd-smi
719 + - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN
720 + - pytest -v -s tests/kernels/attention/test_attention_selector.py
721 + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
.buildkite/test_areas/lm_eval.yaml
+26 -2
@@ -30,6 +30,27 @@ steps:
30 - vllm/_aiter_ops.py
31 - vllm/platforms/rocm.py
32
33 +- label: ":amd: (MI300) LM Eval Small Models Harness"
34 + key: amd-lm-eval-small-models-harness
35 + dind: false
36 + device: mi300_1
37 + timeout_in_minutes: 35
38 + depends_on:
39 + - image-build-amd
40 + working_dir: /vllm-workspace/.buildkite/lm-eval-harness
41 + source_file_dependencies:
42 + - csrc/
43 + - vllm/model_executor/layers/quantization
44 + - vllm/model_executor/models/
45 + - vllm/model_executor/model_loader/
46 + - vllm/v1/attention/backends/
47 + - vllm/v1/attention/selector.py
48 + - vllm/_aiter_ops.py
49 + - vllm/platforms/rocm.py
50 + - .buildkite/lm-eval-harness/
51 + commands:
52 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt
53 +
54 - label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking"
55 device: h200_35gb
56 key: lm-eval-watermarking
@@ -100,7 +121,7 @@ steps:
121 label: ":amd: (MI355) LM Eval Small Models Distributed"
122 dind: false
123 device: mi355_2
103 - timeout_in_minutes: 45
124 + timeout_in_minutes: 55
125 depends_on:
126 - image-build-amd
127 source_file_dependencies:
@@ -286,13 +307,16 @@ steps:
307 label: ":amd: (MI300) LM Eval Large Models"
308 dind: false
309 device: mi300_8
289 - timeout_in_minutes: 40
310 + timeout_in_minutes: 65
311 depends_on:
312 - image-build-amd
313 commands:
314 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
315 - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
316 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
317 + source_file_dependencies:
318 + - vllm/_aiter_ops.py
319 + - vllm/platforms/rocm.py
320
321 - label: ":nvidia: (H100) MoE Refactor Integration TEMPORARY"
322 key: moe-refactor-integration-test-h100-temporary
.buildkite/test_areas/lora.yaml
+9 -9
@@ -61,7 +61,7 @@ steps:
61 dind: false
62 device: mi300_4
63 working_dir: "/vllm-workspace/tests"
64 - timeout_in_minutes: 25
64 + timeout_in_minutes: 65
65 depends_on:
66 - image-build-amd
67 source_file_dependencies:
@@ -73,11 +73,11 @@ steps:
73 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
74 # expandable_segments breaks custom all-reduce IPC on ROCm.
75 - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac
76 - - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_chatglm3_tp.py
77 - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_llama_tp.py
78 - - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_qwen3_with_multi_loras.py
79 - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_olmoe_tp.py
80 - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py
81 - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py
82 - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py
83 - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp
76 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi
77 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi
78 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi
79 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi
80 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi
81 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi
82 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi
83 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp; fi
.buildkite/test_areas/misc.yaml
+36 -7
@@ -99,7 +99,7 @@ steps:
99 label: ":amd: (MI300) V1 Core"
100 dind: false
101 device: mi300_1
102 - timeout_in_minutes: 45
102 + timeout_in_minutes: 60
103 depends_on:
104 - image-build-amd
105
@@ -119,7 +119,7 @@ steps:
119 label: ":amd: (MI300) V1 Executor + Worker"
120 dind: false
121 device: mi300_1
122 - timeout_in_minutes: 45
122 + timeout_in_minutes: 60
123 depends_on:
124 - image-build-amd
125
@@ -138,7 +138,7 @@ steps:
138 label: ":amd: (MI300) V1 KV Offload"
139 dind: false
140 device: mi300_1
141 - timeout_in_minutes: 45
141 + timeout_in_minutes: 60
142 depends_on:
143 - image-build-amd
144
@@ -158,7 +158,7 @@ steps:
158 label: ":amd: (MI300) V1 KV Connectors Shard %N"
159 dind: false
160 device: mi300_1
161 - timeout_in_minutes: 45
161 + timeout_in_minutes: 60
162 depends_on:
163 - image-build-amd
164
@@ -258,7 +258,7 @@ steps:
258 label: ":amd: (MI355) Extract Hidden States Integration"
259 dind: false
260 device: mi355_2
261 - timeout_in_minutes: 20
261 + timeout_in_minutes: 30
262 depends_on:
263 - image-build-amd
264 source_file_dependencies:
@@ -305,6 +305,15 @@ steps:
305 - pip install 'modelscope<1.38'
306 - pytest -v -s test_regression.py
307 working_dir: "/vllm-workspace/tests" # optional
308 + mirror:
309 + amd:
310 + label: ":amd: (MI355) Regression"
311 + dind: false
312 + device: mi355_1
313 + timeout_in_minutes: 25
314 + depends_on:
315 + - image-build-amd
316 + working_dir: /vllm-workspace/tests
317
318 - label: ":nvidia: (H200 MIG 35GB) Examples"
319 device: h200_35gb
@@ -446,9 +455,15 @@ steps:
455 label: ":amd: (MI355) Async Engine, Inputs, Utils, Worker"
456 dind: false
457 device: mi355_1
449 - timeout_in_minutes: 40
458 + timeout_in_minutes: 50
459 depends_on:
460 - image-build-amd
461 + commands:
462 + - pytest -v -s detokenizer
463 + - pytest -v -s -m 'not cpu_test' multimodal
464 + # test_cache.py marks every test cpu_test, including this GPU regression.
465 + - pytest -v -s multimodal/test_cache.py::test_sleep_wake_preserves_mm_cache_consistency
466 + - pytest -v -s utils_
467
468 - label: ":computer: (CPU) Params, Env, Tokenizers, Parser"
469 key: cpu-params-env-tokenizers-parser
@@ -572,7 +587,7 @@ steps:
587 amd:
588 label: ":amd: (MI250) Batch Invariance"
589 device: mi250_1
575 - timeout_in_minutes: 30
590 + timeout_in_minutes: 35
591 depends_on:
592 - image-build-amd
593 source_file_dependencies:
@@ -623,3 +638,17 @@ steps:
638 commands:
639 - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
640 - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
641 + mirror:
642 + amd:
643 + label: ":amd: (MI355) Acceptance Length (Large Models)"
644 + dind: false
645 + device: mi355_1
646 + timeout_in_minutes: 45
647 + depends_on:
648 + - image-build-amd
649 + working_dir: /vllm-workspace/tests
650 + source_file_dependencies:
651 + - vllm/v1/spec_decode/
652 + - vllm/model_executor/models/mlp_speculator.py
653 + - tests/v1/spec_decode/test_acceptance_length.py
654 + - vllm/platforms/rocm.py
.buildkite/test_areas/models_basic.yaml
+19 -2
@@ -19,7 +19,7 @@ steps:
19 label: ":amd: (MI355) Basic Models (Initialization)"
20 dind: false
21 device: mi355_1
22 - timeout_in_minutes: 25
22 + timeout_in_minutes: 30
23 depends_on:
24 - image-build-amd
25
@@ -37,6 +37,23 @@ steps:
37 # test.) Also run if model initialization test file is modified
38 - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
39 parallelism: 14
40 + mirror:
41 + amd:
42 + label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N"
43 + dind: false
44 + device: mi355_1
45 + timeout_in_minutes: 60
46 + parallelism: 10
47 + working_dir: "/vllm-workspace/tests"
48 + depends_on:
49 + - image-build-amd
50 + source_file_dependencies:
51 + - vllm/model_executor/models/
52 + - vllm/model_executor/layers/
53 + - tests/models/test_initialization.py
54 + - tests/models/registry.py
55 + - vllm/_aiter_ops.py
56 + - vllm/platforms/rocm.py
57
58 - label: ":nvidia: (H200 MIG 35GB) Basic Models (Other)"
59 device: h200_35gb
@@ -162,6 +179,6 @@ steps:
179 device: mi250_1
180 no_gpu: true
181 working_dir: "/vllm-workspace/tests"
165 - timeout_in_minutes: 25
182 + timeout_in_minutes: 35
183 depends_on:
184 - image-build-amd
.buildkite/test_areas/models_distributed.yaml
+4 -4
@@ -38,7 +38,7 @@ steps:
38 label: ":amd: (MI300) Distributed Models Shard %N"
39 dind: false
40 device: mi300_2
41 - timeout_in_minutes: 35
41 + timeout_in_minutes: 80
42 depends_on:
43 - image-build-amd
44 source_file_dependencies:
@@ -63,7 +63,7 @@ steps:
63 - |
64 case "$$BUILDKITE_PARALLEL_JOB" in
65 0)
66 - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
66 + TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' &&
67 HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
68 ;;
69 1)
@@ -72,8 +72,8 @@ steps:
72 VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)'
73 ;;
74 2)
75 - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
76 - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
75 + pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py &&
76 + pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' &&
77 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
78 ;;
79 *) exit 2 ;;
.buildkite/test_areas/models_language.yaml
+5 -8
@@ -19,7 +19,7 @@ steps:
19 label: ":amd: (MI300) Language Models (Standard)"
20 dind: false
21 device: mi300_1
22 - timeout_in_minutes: 45
22 + timeout_in_minutes: 60
23 depends_on:
24 - image-build-amd
25
@@ -82,10 +82,10 @@ steps:
82 parallelism: 2
83 mirror:
84 amd:
85 - label: ":amd: (MI300) Language Models (Hybrid) Shard %N"
85 + label: ":amd: (MI355) Language Models (Hybrid) Shard %N"
86 dind: false
87 - device: mi300_1
88 - timeout_in_minutes: 60
87 + device: mi355_1
88 + timeout_in_minutes: 65
89 depends_on:
90 - image-build-amd
91 commands:
@@ -133,7 +133,7 @@ steps:
133 label: ":amd: (MI355) Language Models (Extended Generation)"
134 dind: false
135 device: mi355_1
136 - timeout_in_minutes: 70
136 + timeout_in_minutes: 95
137 depends_on:
138 - image-build-amd
139 commands:
@@ -197,11 +197,8 @@ steps:
197 dind: false
198 device: mi300_1
199 timeout_in_minutes: 95
200 - parallelism: 2
200 depends_on:
201 - image-build-amd
203 - commands:
204 - - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
202
203 - label: ":nvidia: (H200 MIG 18GB) Language Models (MTEB)"
204 key: language-models-test-mteb
.buildkite/test_areas/models_multimodal.yaml
+4 -3
@@ -77,9 +77,9 @@ steps:
77 - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
78 mirror:
79 amd:
80 - label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper"
80 + label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper"
81 dind: false
82 - device: mi300_1
82 + device: mi355_1
83 timeout_in_minutes: 70
84 depends_on:
85 - image-build-amd
@@ -104,7 +104,8 @@ steps:
104 device: mi250_1
105 no_gpu: true
106 working_dir: "/vllm-workspace/tests"
107 - timeout_in_minutes: 80
107 + timeout_in_minutes: 90
108 + parallelism: 6
109 depends_on:
110 - image-build-amd
111
.buildkite/test_areas/plugins.yaml
+14
@@ -75,6 +75,20 @@ steps:
75 commands:
76 - pip install "vllm-gguf-plugin >= 0.0.2"
77 - pytest -v -s plugins_tests/gguf
78 + mirror:
79 + amd:
80 + label: ":amd: (MI355) GGUF Plugin"
81 + dind: false
82 + device: mi355_1
83 + timeout_in_minutes: 45
84 + working_dir: "/vllm-workspace/tests"
85 + depends_on:
86 + - image-build-amd
87 + source_file_dependencies:
88 + - vllm/model_executor/layers/quantization
89 + - tests/plugins_tests/test_gguf_plugin.py
90 + - tests/plugins_tests/gguf
91 + - vllm/platforms/rocm.py
92
93 - label: ":nvidia: (H200 MIG 18GB) BitsAndBytes Plugin"
94 key: bitsandbytes-plugin
.buildkite/test_areas/pytorch.yaml
+16
@@ -122,6 +122,21 @@ steps:
122 - tests/compile/passes
123 commands:
124 - pytest -s -v compile/passes --ignore compile/passes/distributed
125 + mirror:
126 + amd:
127 + label: ':amd: (MI300) PyTorch Compilation Passes'
128 + dind: false
129 + device: mi300_1
130 + timeout_in_minutes: 80
131 + working_dir: /vllm-workspace/tests
132 + depends_on:
133 + - image-build-amd
134 + source_file_dependencies:
135 + - vllm/
136 + - tests/compile/passes
137 + - vllm/platforms/rocm.py
138 + commands:
139 + - pytest -s -v compile/passes --ignore compile/passes/distributed
140
141 - label: ":nvidia: (H200 MIG 35GB) PyTorch Fullgraph"
142 device: h200_35gb
@@ -154,6 +169,7 @@ steps:
169 - vllm/utils/
170 - vllm/v1/
171 - tests/compile
172 + - csrc/
173 commands:
174 # Run smoke tests under fullgraph directory, except test_full_cudagraph.py
175 # as it is a heavy test that is covered in other steps.
.buildkite/test_areas/quantization.yaml
+44
@@ -20,6 +20,31 @@ steps:
20 # The SM90-only checkpoint currently contains a removed weight_chan_scale
21 # parameter. It was not exercised by the previous L4 job.
22 - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
23 + mirror:
24 + amd:
25 + label: ":amd: (MI355) Quantization Shard %N"
26 + dind: false
27 + device: mi355_1
28 + timeout_in_minutes: 115
29 + working_dir: "/vllm-workspace/tests"
30 + depends_on:
31 + - image-build-amd
32 + source_file_dependencies:
33 + - csrc/
34 + - vllm/model_executor/layers/quantization
35 + - tests/quantization
36 + - vllm/_aiter_ops.py
37 + - vllm/platforms/rocm.py
38 + - vllm/model_executor/layers/fused_moe/oracle/unquantized.py
39 + - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py
40 + - tests/rocm/test_moe_weight_replay.py
41 + commands:
42 + # Use the ROCm default instead of the NVIDIA runner override.
43 + - unset VLLM_USE_V2_MODEL_RUNNER
44 + - uv pip install --system torchao==0.17.0
45 + - uv pip install --system conch-triton-kernels
46 + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
47 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
48
49 - label: ":nvidia: (H200 MIG 35GB) Quantized Fusions"
50 device: h200_35gb
@@ -63,3 +88,22 @@ steps:
88 - tests/models/quantization
89 commands:
90 - pytest -v -s models/quantization
91 + mirror:
92 + amd:
93 + label: ":amd: (MI355) Quantized Models"
94 + dind: false
95 + device: mi355_1
96 + timeout_in_minutes: 80
97 + working_dir: "/vllm-workspace/tests"
98 + depends_on:
99 + - image-build-amd
100 + source_file_dependencies:
101 + - vllm/model_executor/layers/quantization
102 + - tests/models/quantization
103 + - vllm/_aiter_ops.py
104 + - vllm/platforms/rocm.py
105 + - vllm/model_executor/model_loader/
106 + commands:
107 + # Use the ROCm default instead of the NVIDIA runner override.
108 + - unset VLLM_USE_V2_MODEL_RUNNER
109 + - pytest -v -s models/quantization
.buildkite/test_areas/ray_compat.yaml
+16
@@ -16,3 +16,19 @@ steps:
16 - setup.py
17 commands:
18 - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
19 + mirror:
20 + amd:
21 + label: ":amd: (MI355) Ray Dependency Compatibility Check"
22 + dind: false
23 + device: mi355_1
24 + timeout_in_minutes: 25
25 + working_dir: "/"
26 + depends_on:
27 + - image-build-amd
28 + source_file_dependencies:
29 + - .buildkite/scripts/check-ray-compatibility.sh
30 + - requirements/
31 + - setup.py
32 + - vllm/platforms/rocm.py
33 + commands:
34 + - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
.buildkite/test_areas/rust_frontend.yaml
+2 -2
@@ -136,7 +136,7 @@ steps:
136 label: ":amd: (MI355) Rust Frontend Core Correctness"
137 dind: false
138 device: mi355_1
139 - timeout_in_minutes: 20
139 + timeout_in_minutes: 25
140 depends_on:
141 - image-build-amd
142 source_file_dependencies:
@@ -166,7 +166,7 @@ steps:
166 label: ":amd: (MI355) Rust Frontend Tool Use"
167 dind: false
168 device: mi355_1
169 - timeout_in_minutes: 25
169 + timeout_in_minutes: 30
170 depends_on:
171 - image-build-amd
172 source_file_dependencies:
.buildkite/test_areas/spec_decode.yaml
+36 -6
@@ -47,7 +47,7 @@ steps:
47 label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen"
48 dind: false
49 device: mi300_1
50 - timeout_in_minutes: 35
50 + timeout_in_minutes: 60
51 depends_on:
52 - image-build-amd
53 source_file_dependencies:
@@ -74,7 +74,7 @@ steps:
74 label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other"
75 dind: false
76 device: mi300_1
77 - timeout_in_minutes: 25
77 + timeout_in_minutes: 60
78 depends_on:
79 - image-build-amd
80 source_file_dependencies:
@@ -187,7 +187,7 @@ steps:
187 label: ":amd: (MI300) Spec Decode Draft Model"
188 dind: false
189 device: mi300_1
190 - timeout_in_minutes: 55
190 + timeout_in_minutes: 60
191 depends_on:
192 - image-build-amd
193 source_file_dependencies:
@@ -233,6 +233,9 @@ steps:
233 timeout_in_minutes: 40
234 depends_on:
235 - image-build-amd
236 + source_file_dependencies:
237 + - vllm/_aiter_ops.py
238 + - vllm/platforms/rocm.py
239
240 - label: ":nvidia: (B200) Spec Decode DeepSeek MTP Parallel Load"
241 key: spec-decode-deepseek-mtp-parallel-load-2xb200-2xmi300
@@ -362,6 +365,33 @@ steps:
365 # Each Gemma4 parametrization is an indivisible ~23m floor. Keep them in
366 # separate shards and put the remaining tests in shard 0, where new tests
367 # land by default.
365 - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"
366 - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"
367 - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"
368 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
369 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi
370 + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
371 + mirror:
372 + amd:
373 + label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly"
374 + dind: false
375 + device: mi355_1
376 + timeout_in_minutes: 55
377 + depends_on:
378 + - image-build-amd
379 + working_dir: /vllm-workspace/tests
380 + source_file_dependencies:
381 + - vllm/v1/spec_decode/
382 + - vllm/v1/sample/
383 + - vllm/v1/worker/gpu/
384 + - vllm/v1/worker/gpu_model_runner.py
385 + - vllm/v1/worker/gpu_worker.py
386 + - vllm/model_executor/model_loader/
387 + - vllm/model_executor/models/gemma4.py
388 + - vllm/model_executor/models/gemma4_mm.py
389 + - vllm/model_executor/models/gemma4_mtp.py
390 + - vllm/model_executor/models/llama.py
391 + - vllm/model_executor/models/llama_eagle3.py
392 + - vllm/model_executor/models/medusa.py
393 + - vllm/model_executor/models/registry.py
394 + - tests/evals/gsm8k/
395 + - tests/utils.py
396 + - tests/v1/e2e/spec_decode/
397 + - vllm/platforms/rocm.py
.buildkite/test_areas/weight_loading.yaml
+1 -1
@@ -20,7 +20,7 @@ steps:
20 label: ":amd: (MI355) Weight Loading Multi-GPU"
21 dind: false
22 device: mi355_2
23 - timeout_in_minutes: 40
23 + timeout_in_minutes: 70
24 depends_on:
25 - image-build-amd
26 commands: