@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 1479bd9e9d

[ROCm][CI] Prepare AMD mirrors for regating (#49270)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Andreas Karatzas committed Jul 23, 2026 at 20:42 UTC 1479bd9e9d3e7e06a4167980d4d4662eeda0638c
23 files changed +364 -292
.buildkite/scripts/hardware_ci/run-amd-test.sh
+3 -3
@@ -35,7 +35,7 @@ set -o pipefail
35 : "${PY_COLORS:=1}"
36 : "${ROCM_DOCKER_TTY:=1}"
37 : "${PYTHONFAULTHANDLER:=1}"
38 -: "${PYTEST_TIMEOUT:=2100}"
38 +: "${PYTEST_TIMEOUT:=2400}"
39 if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
40 PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
41 fi
@@ -45,9 +45,9 @@ fi
45 if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then
46 PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0"
47 fi
48 -# Dump stacks after 15 minutes, then stop an individual test after 35 minutes.
48 +# Dump stacks after 25 minutes, then stop an individual test after 40 minutes.
49 if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then
50 - PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=900"
50 + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=1500"
51 fi
52 if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* &&
53 " ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then
.buildkite/test-amd.yaml
+234 -215
@@ -40,7 +40,7 @@
40 #####################################################################################################################################
41 # #
42 # IMPORTANT: #
43 -# * Currently AMD CI has MI250 agents, MI300 agents, MI325 agents, and MI355 agents. All upcoming feature improvements are #
43 +# * Currently AMD CI has MI250 agents, MI300 agents, and MI355 agents. All upcoming feature improvements are #
44 # tracked in: https://github.com/vllm-project/vllm/issues/34994 #
45 # #
46 #-----------------------------------------------------------------------------------------------------------------------------------#
@@ -207,6 +207,7 @@ steps:
207 timeout_in_minutes: 180
208 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
209 agent_pool: mi250_1
210 + optional: true
211 working_dir: "/vllm-workspace/tests"
212 source_file_dependencies:
213 - vllm/
@@ -240,6 +241,20 @@ steps:
241 - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
242 - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
243
244 +- label: Multi-Modal Processor (CPU) %N # TBD
245 + timeout_in_minutes: 180
246 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
247 + agent_pool: mi250_1
248 + parallelism: 6
249 + optional: true
250 + working_dir: "/vllm-workspace/tests"
251 + source_file_dependencies:
252 + - vllm/
253 + - tests/models/multimodal
254 + - tests/models/registry.py
255 + commands:
256 + - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
257 +
258 #------------------------------------------------------------ mi250 · v1 -------------------------------------------------------------#
259
260 - label: Batch Invariance (H100-MI250) # TBD
@@ -367,6 +382,22 @@ steps:
382 commands:
383 - pytest -v -s v1/attention
384
385 +- label: V1 others (CPU) # TBD
386 + timeout_in_minutes: 180
387 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
388 + agent_pool: mi250_1
389 + optional: true
390 + working_dir: "/vllm-workspace/tests"
391 + source_file_dependencies:
392 + - vllm/
393 + - tests/v1
394 + commands:
395 + - pytest -v -s -m 'cpu_test' v1/core
396 + - pytest -v -s v1/structured_output
397 + - pytest -v -s v1/test_serial_utils.py
398 + - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
399 + - pytest -v -s -m 'cpu_test' v1/metrics
400 +
401 #------------------------------------------------------------- mi250 · misc ------------------------------------------------------------#
402
403 - label: Async Engine, Inputs, Utils, Worker, Config (CPU) # TBD
@@ -408,6 +439,19 @@ steps:
439 - pytest -v -s transformers_utils
440 - pytest -v -s config
441
442 +- label: Python-only Installation # TBD
443 + timeout_in_minutes: 180
444 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
445 + agent_pool: mi250_1
446 + optional: true
447 + working_dir: "/vllm-workspace/tests"
448 + source_file_dependencies:
449 + - tests/standalone_tests/python_only_compile.sh
450 + - setup.py
451 + - vllm/platforms/rocm.py
452 + commands:
453 + - bash standalone_tests/python_only_compile.sh
454 +
455 #------------------------------------------------------------ mi250 · rust -----------------------------------------------------------#
456
457 - label: Rust Frontend Cargo Style + Clippy # TBD
@@ -445,6 +489,7 @@ steps:
489 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
490 agent_pool: mi250_1
491 no_gpu: true
492 + optional: true
493 working_dir: "/vllm-workspace/tests"
494 source_file_dependencies:
495 - .buildkite/scripts/docker-build-metadata-args.sh
@@ -507,7 +552,7 @@ steps:
552 - tests/models/
553 commands:
554 - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
510 - - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
555 + - HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
556 - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
557 - pytest models/language -v -s -m 'distributed(num_gpus=2)'
558 - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
@@ -657,6 +702,30 @@ steps:
702 - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
703 - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions
704
705 +- label: Distributed Compile + RPC Tests (2 GPUs) # TBD
706 + timeout_in_minutes: 180
707 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
708 + dind: false
709 + agent_pool: mi300_2
710 + num_gpus: 2
711 + working_dir: "/vllm-workspace/tests"
712 + source_file_dependencies:
713 + - vllm/compilation/
714 + - vllm/distributed/
715 + - vllm/engine/
716 + - vllm/executor/
717 + - vllm/worker/worker_base.py
718 + - vllm/v1/engine/
719 + - vllm/v1/worker/
720 + - tests/compile/fullgraph/test_basic_correctness.py
721 + - tests/compile/test_wrapper.py
722 + - tests/entrypoints/llm/test_collective_rpc.py
723 + - vllm/platforms/rocm.py
724 + commands:
725 + - pytest -v -s entrypoints/llm/test_collective_rpc.py
726 + - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
727 + - pytest -v -s ./compile/test_wrapper.py
728 +
729 #----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------#
730
731 - label: Platform Tests # TBD
@@ -679,6 +748,7 @@ steps:
748 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
749 dind: false
750 agent_pool: mi300_1
751 + optional: true
752 working_dir: "/vllm-workspace/tests"
753 source_file_dependencies:
754 - vllm/
@@ -870,6 +940,71 @@ steps:
940 commands:
941 - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
942
943 +- label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD
944 + timeout_in_minutes: 180
945 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
946 + dind: false
947 + agent_pool: mi300_2
948 + num_gpus: 2
949 + working_dir: "/vllm-workspace/tests"
950 + source_file_dependencies:
951 + - vllm/distributed/
952 + - vllm/engine/
953 + - vllm/executor/
954 + - vllm/worker/worker_base.py
955 + - vllm/v1/engine/
956 + - vllm/v1/worker/
957 + - tests/distributed/
958 + - tests/v1/shutdown
959 + - tests/v1/worker/test_worker_memory_snapshot.py
960 + - vllm/platforms/rocm.py
961 + commands:
962 + - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
963 + - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
964 + - HIP_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
965 + - pytest -v -s v1/worker/test_worker_memory_snapshot.py
966 +
967 +- label: Distributed Compile + Comm (4 GPUs) # TBD
968 + timeout_in_minutes: 180
969 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
970 + dind: false
971 + agent_pool: mi300_4
972 + num_gpus: 4
973 + working_dir: "/vllm-workspace/tests"
974 + source_file_dependencies:
975 + - vllm/distributed/
976 + - tests/distributed/test_pynccl
977 + - tests/distributed/test_events
978 + - tests/compile/fullgraph/test_basic_correctness.py
979 + - tests/distributed/test_symm_mem_allreduce.py
980 + - tests/distributed/test_multiproc_executor.py
981 + - vllm/platforms/rocm.py
982 + commands:
983 + - pytest -v -s compile/fullgraph/test_basic_correctness.py
984 + - pytest -v -s distributed/test_pynccl.py
985 + - pytest -v -s distributed/test_events.py
986 + - pytest -v -s distributed/test_symm_mem_allreduce.py
987 + - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
988 +
989 +#---------------------------------------------------------- mi300 · engine -----------------------------------------------------------#
990 +
991 +- label: Engine # TBD
992 + timeout_in_minutes: 180
993 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
994 + dind: false
995 + agent_pool: mi300_1
996 + optional: true
997 + working_dir: "/vllm-workspace/tests"
998 + source_file_dependencies:
999 + - vllm/
1000 + - tests/engine
1001 + - tests/test_sequence
1002 + - tests/test_config
1003 + - tests/test_logger
1004 + - tests/test_vllm_port
1005 + commands:
1006 + - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
1007 +
1008 #-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------#
1009
1010 - label: Entrypoints Unit Tests # TBD
@@ -878,6 +1013,7 @@ steps:
1013 dind: false
1014 agent_pool: mi300_1
1015 fast_check: true
1016 + optional: true
1017 working_dir: "/vllm-workspace/tests"
1018 source_file_dependencies:
1019 - vllm/entrypoints
@@ -982,6 +1118,7 @@ steps:
1118 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1119 dind: false
1120 agent_pool: mi300_1
1121 + optional: true
1122 fast_check: true
1123 working_dir: "/vllm-workspace/tests"
1124 source_file_dependencies:
@@ -1010,6 +1147,7 @@ steps:
1147 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1148 dind: false
1149 agent_pool: mi300_1
1150 + optional: true
1151 fast_check: true
1152 working_dir: "/vllm-workspace/tests"
1153 source_file_dependencies:
@@ -1024,6 +1162,7 @@ steps:
1162 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1163 dind: false
1164 agent_pool: mi300_1
1165 + optional: true
1166 fast_check: true
1167 working_dir: "/vllm-workspace/tests"
1168 source_file_dependencies:
@@ -1345,6 +1484,27 @@ steps:
1484 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1485 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
1486
1487 +- label: LM Eval Large Models (4xH100-4xMI300) # TBD
1488 + timeout_in_minutes: 180
1489 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1490 + dind: false
1491 + agent_pool: mi300_4
1492 + num_gpus: 4
1493 + optional: true
1494 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1495 + source_file_dependencies:
1496 + - csrc/
1497 + - vllm/model_executor/layers/quantization
1498 + - vllm/model_executor/models/
1499 + - vllm/model_executor/model_loader/
1500 + - vllm/v1/attention/backends/
1501 + - vllm/v1/attention/selector.py
1502 + - vllm/_aiter_ops.py
1503 + - vllm/platforms/rocm.py
1504 + commands:
1505 + - export VLLM_USE_DEEP_GEMM=0
1506 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
1507 +
1508 #--------------------------------------------------------- mi300 · examples ----------------------------------------------------------#
1509
1510 - label: Examples # TBD
@@ -1593,7 +1753,7 @@ steps:
1753 - set -x
1754 - export VLLM_USE_V2_MODEL_RUNNER=1
1755 - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
1596 - - ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
1756 + - pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
1757 - pytest -v -s v1/e2e/general/test_context_length.py
1758 - pytest -v -s v1/e2e/general/test_min_tokens.py
1759 - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
@@ -1797,6 +1957,37 @@ steps:
1957 - pip freeze | grep -E 'torch'
1958 - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
1959
1960 +- label: Language Models Test (Extended Generation) # TBD
1961 + timeout_in_minutes: 180
1962 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1963 + dind: false
1964 + agent_pool: mi300_1
1965 + optional: true
1966 + working_dir: "/vllm-workspace/tests"
1967 + source_file_dependencies:
1968 + - vllm/
1969 + - tests/models/language/generation
1970 + commands:
1971 + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
1972 + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
1973 + - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
1974 +
1975 +- label: Language Models Tests (Hybrid) %N # TBD
1976 + timeout_in_minutes: 180
1977 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1978 + dind: false
1979 + agent_pool: mi300_1
1980 + parallelism: 2
1981 + optional: true
1982 + working_dir: "/vllm-workspace/tests"
1983 + source_file_dependencies:
1984 + - vllm/
1985 + - tests/models/language/generation
1986 + commands:
1987 + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
1988 + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
1989 + - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
1990 +
1991 #---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------#
1992
1993 - label: Multi-Modal Models (Extended Generation 1) # TBD
@@ -1899,20 +2090,32 @@ steps:
2090 commands:
2091 - pytest -v -s models/multimodal/processing/test_tensor_schema.py
2092
1902 -- label: Multi-Modal Processor (CPU) %N # TBD
2093 +- label: Multi-Modal Models (Extended Pooling) # TBD
2094 + timeout_in_minutes: 180
2095 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2096 + dind: false
2097 + agent_pool: mi300_1
2098 + optional: true
2099 + working_dir: "/vllm-workspace/tests"
2100 + source_file_dependencies:
2101 + - vllm/
2102 + - tests/models/multimodal/pooling
2103 + commands:
2104 + - pytest -v -s models/multimodal/pooling -m 'not core_model'
2105 +
2106 +- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD
2107 timeout_in_minutes: 180
2108 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2109 dind: false
2110 agent_pool: mi300_1
1907 - parallelism: 4
2111 optional: true
2112 working_dir: "/vllm-workspace/tests"
2113 source_file_dependencies:
2114 - vllm/
2115 - tests/models/multimodal
1913 - - tests/models/registry.py
2116 commands:
1915 - - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2117 + - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
2118 + - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
2119
2120 #----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------#
2121
@@ -1933,12 +2136,12 @@ steps:
2136
2137 #-------------------------------------------------- mi300 · models / transformers ---------------------------------------------------#
2138
1936 -- label: Transformers Nightly Models (Shardable) %N # TBD
2139 +- label: Transformers Nightly Models (Initialization) %N # TBD
2140 timeout_in_minutes: 180
2141 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2142 dind: false
2143 agent_pool: mi300_1
1941 - parallelism: 4
2144 + parallelism: 6
2145 optional: true
2146 working_dir: "/vllm-workspace/"
2147 source_file_dependencies:
@@ -1954,6 +2157,27 @@ steps:
2157 commands:
2158 - pip install --upgrade git+https://github.com/huggingface/transformers
2159 - pytest -v -s tests/models/test_initialization.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2160 +
2161 +- label: Transformers Nightly Models (Processing) %N # TBD
2162 + timeout_in_minutes: 180
2163 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2164 + dind: false
2165 + agent_pool: mi300_1
2166 + parallelism: 4
2167 + optional: true
2168 + working_dir: "/vllm-workspace/"
2169 + source_file_dependencies:
2170 + - vllm/model_executor/models/
2171 + - vllm/model_executor/model_loader/
2172 + - vllm/multimodal/
2173 + - vllm/model_executor/layers/
2174 + - vllm/v1/attention/backends/
2175 + - vllm/v1/attention/selector.py
2176 + - vllm/_aiter_ops.py
2177 + - vllm/platforms/rocm.py
2178 + - tests/models/
2179 + commands:
2180 + - pip install --upgrade git+https://github.com/huggingface/transformers
2181 - pytest -v -s tests/models/multimodal/processing/ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2182
2183 - label: Transformers Nightly Models (Single) # TBD
@@ -2517,23 +2741,6 @@ steps:
2741 # - export HSA_NO_SCRATCH_RECLAIM=1
2742 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
2743
2520 -- label: V1 others (CPU) # TBD
2521 - timeout_in_minutes: 180
2522 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2523 - dind: false
2524 - agent_pool: mi300_1
2525 - optional: true
2526 - working_dir: "/vllm-workspace/tests"
2527 - source_file_dependencies:
2528 - - vllm/
2529 - - tests/v1
2530 - commands:
2531 - - pytest -v -s -m 'cpu_test' v1/core
2532 - - pytest -v -s v1/structured_output
2533 - - pytest -v -s v1/test_serial_utils.py
2534 - - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
2535 - - pytest -v -s -m 'cpu_test' v1/metrics
2536 -
2744 - label: V1 Sample + Logits # TBD
2745 timeout_in_minutes: 180
2746 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -2830,195 +3037,6 @@ steps:
3037 commands:
3038 - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt
3039
2833 -#########################################################################################################################################
2834 -# #
2835 -# MI325 (gfx942) tests #
2836 -# #
2837 -#########################################################################################################################################
2838 -
2839 -#---------------------------------------------------------- mi325 · compile ----------------------------------------------------------#
2840 -
2841 -- label: Distributed Compile + RPC Tests (2 GPUs) # TBD
2842 - timeout_in_minutes: 180
2843 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2844 - agent_pool: mi325_2
2845 - num_gpus: 2
2846 - working_dir: "/vllm-workspace/tests"
2847 - source_file_dependencies:
2848 - - vllm/compilation/
2849 - - vllm/distributed/
2850 - - vllm/engine/
2851 - - vllm/executor/
2852 - - vllm/worker/worker_base.py
2853 - - vllm/v1/engine/
2854 - - vllm/v1/worker/
2855 - - tests/compile/fullgraph/test_basic_correctness.py
2856 - - tests/compile/test_wrapper.py
2857 - - tests/entrypoints/llm/test_collective_rpc.py
2858 - - vllm/platforms/rocm.py
2859 - commands:
2860 - - pytest -v -s entrypoints/llm/test_collective_rpc.py
2861 - - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
2862 - - pytest -v -s ./compile/test_wrapper.py
2863 -
2864 -#-------------------------------------------------------- mi325 · distributed --------------------------------------------------------#
2865 -
2866 -- label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD
2867 - timeout_in_minutes: 180
2868 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2869 - agent_pool: mi325_2
2870 - num_gpus: 2
2871 - working_dir: "/vllm-workspace/tests"
2872 - source_file_dependencies:
2873 - - vllm/distributed/
2874 - - vllm/engine/
2875 - - vllm/executor/
2876 - - vllm/worker/worker_base.py
2877 - - vllm/v1/engine/
2878 - - vllm/v1/worker/
2879 - - tests/distributed/
2880 - - tests/v1/shutdown
2881 - - tests/v1/worker/test_worker_memory_snapshot.py
2882 - - vllm/platforms/rocm.py
2883 - commands:
2884 - - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
2885 - - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
2886 - - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
2887 - - pytest -v -s v1/worker/test_worker_memory_snapshot.py
2888 -
2889 -- label: Distributed Compile + Comm (4 GPUs) # TBD
2890 - timeout_in_minutes: 180
2891 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2892 - agent_pool: mi325_4
2893 - num_gpus: 4
2894 - working_dir: "/vllm-workspace/tests"
2895 - source_file_dependencies:
2896 - - vllm/distributed/
2897 - - tests/distributed/test_pynccl
2898 - - tests/distributed/test_events
2899 - - tests/compile/fullgraph/test_basic_correctness.py
2900 - - tests/distributed/test_symm_mem_allreduce.py
2901 - - tests/distributed/test_multiproc_executor.py
2902 - - vllm/platforms/rocm.py
2903 - commands:
2904 - - pytest -v -s compile/fullgraph/test_basic_correctness.py
2905 - - pytest -v -s distributed/test_pynccl.py
2906 - - pytest -v -s distributed/test_events.py
2907 - - pytest -v -s distributed/test_symm_mem_allreduce.py
2908 - - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
2909 -
2910 -#---------------------------------------------------------- mi325 · engine -----------------------------------------------------------#
2911 -
2912 -- label: Engine # TBD
2913 - timeout_in_minutes: 180
2914 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2915 - agent_pool: mi325_1
2916 - optional: true
2917 - working_dir: "/vllm-workspace/tests"
2918 - source_file_dependencies:
2919 - - vllm/
2920 - - tests/engine
2921 - - tests/test_sequence
2922 - - tests/test_config
2923 - - tests/test_logger
2924 - - tests/test_vllm_port
2925 - commands:
2926 - - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
2927 -
2928 -#----------------------------------------------------------- mi325 · evals -----------------------------------------------------------#
2929 -
2930 -- label: LM Eval Large Models (4xH100-4xMI325) # TBD
2931 - timeout_in_minutes: 180
2932 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2933 - agent_pool: mi325_4
2934 - num_gpus: 4
2935 - optional: true
2936 - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
2937 - source_file_dependencies:
2938 - - csrc/
2939 - - vllm/model_executor/layers/quantization
2940 - - vllm/model_executor/models/
2941 - - vllm/model_executor/model_loader/
2942 - - vllm/v1/attention/backends/
2943 - - vllm/v1/attention/selector.py
2944 - - vllm/_aiter_ops.py
2945 - - vllm/platforms/rocm.py
2946 - commands:
2947 - - export VLLM_USE_DEEP_GEMM=0
2948 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
2949 -
2950 -#----------------------------------------------------- mi325 · models / language -----------------------------------------------------#
2951 -
2952 -- label: Language Models Test (Extended Generation) # TBD
2953 - timeout_in_minutes: 180
2954 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2955 - agent_pool: mi325_1
2956 - working_dir: "/vllm-workspace/tests"
2957 - source_file_dependencies:
2958 - - vllm/
2959 - - tests/models/language/generation
2960 - commands:
2961 - - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
2962 - - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
2963 - - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
2964 -
2965 -- label: Language Models Tests (Hybrid) %N # TBD
2966 - timeout_in_minutes: 180
2967 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2968 - agent_pool: mi325_1
2969 - parallelism: 2
2970 - optional: true
2971 - working_dir: "/vllm-workspace/tests"
2972 - source_file_dependencies:
2973 - - vllm/
2974 - - tests/models/language/generation
2975 - commands:
2976 - - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
2977 - - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
2978 - - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
2979 -
2980 -#---------------------------------------------------- mi325 · models / multimodal ----------------------------------------------------#
2981 -
2982 -- label: Multi-Modal Models (Extended Pooling) # TBD
2983 - timeout_in_minutes: 180
2984 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2985 - agent_pool: mi325_1
2986 - optional: true
2987 - working_dir: "/vllm-workspace/tests"
2988 - source_file_dependencies:
2989 - - vllm/
2990 - - tests/models/multimodal/pooling
2991 - commands:
2992 - - pytest -v -s models/multimodal/pooling -m 'not core_model'
2993 -
2994 -- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD
2995 - timeout_in_minutes: 180
2996 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2997 - agent_pool: mi325_1
2998 - optional: true
2999 - working_dir: "/vllm-workspace/tests"
3000 - source_file_dependencies:
3001 - - vllm/
3002 - - tests/models/multimodal
3003 - commands:
3004 - - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
3005 - - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
3006 -
3007 -#----------------------------------------------------------- mi325 · misc ------------------------------------------------------------#
3008 -
3009 -- label: Python-only Installation # TBD
3010 - timeout_in_minutes: 180
3011 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
3012 - agent_pool: mi325_1
3013 - optional: true
3014 - working_dir: "/vllm-workspace/tests"
3015 - source_file_dependencies:
3016 - - tests/standalone_tests/python_only_compile.sh
3017 - - setup.py
3018 - - vllm/platforms/rocm.py
3019 - commands:
3020 - - bash standalone_tests/python_only_compile.sh
3021 -
3040 #########################################################################################################################################
3041 # #
3042 # MI355 (gfx950) tests #
@@ -3463,6 +3481,7 @@ steps:
3481 timeout_in_minutes: 180
3482 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3483 agent_pool: mi355_1
3484 + optional: true
3485 working_dir: "/vllm-workspace/tests"
3486 source_file_dependencies:
3487 - vllm/model_executor/models/qwen3_5.py
.buildkite/test_areas/attention.yaml
+3 -2
@@ -16,8 +16,9 @@ steps:
16 parallelism: 2
17 mirror:
18 amd:
19 - device: mi325_1
20 - timeout_in_minutes: 95
19 + dind: false
20 + device: mi300_1
21 + timeout_in_minutes: 125
22 depends_on:
23 - image-build-amd
24 source_file_dependencies:
.buildkite/test_areas/basic_correctness.yaml
+3 -2
@@ -18,7 +18,8 @@ steps:
18 - pytest -v -s basic_correctness/test_cpu_offload.py
19 mirror:
20 amd:
21 - device: mi325_1
22 - timeout_in_minutes: 70
21 + dind: false
22 + device: mi300_1
23 + timeout_in_minutes: 60
24 depends_on:
25 - image-build-amd
.buildkite/test_areas/benchmarks.yaml
+1
@@ -15,6 +15,7 @@ steps:
15 amd:
16 dind: false
17 device: mi300_1
18 + timeout_in_minutes: 40
19 depends_on:
20 - image-build-amd
21
.buildkite/test_areas/disaggregated.yaml
+5 -5
@@ -17,7 +17,7 @@ steps:
17 amd:
18 dind: false
19 device: mi300_4
20 - timeout_in_minutes: 85
20 + timeout_in_minutes: 60
21 depends_on:
22 - image-build-amd
23 source_file_dependencies:
@@ -68,7 +68,7 @@ steps:
68 amd:
69 dind: false
70 device: mi300_4
71 - timeout_in_minutes: 60
71 + timeout_in_minutes: 40
72 depends_on:
73 - image-build-amd
74 source_file_dependencies:
@@ -94,7 +94,7 @@ steps:
94 amd:
95 dind: false
96 device: mi300_4
97 - timeout_in_minutes: 85
97 + timeout_in_minutes: 60
98 depends_on:
99 - image-build-amd
100 source_file_dependencies:
@@ -120,7 +120,7 @@ steps:
120 amd:
121 dind: false
122 device: mi300_4
123 - timeout_in_minutes: 80
123 + timeout_in_minutes: 55
124 depends_on:
125 - image-build-amd
126 source_file_dependencies:
@@ -177,7 +177,7 @@ steps:
177 amd:
178 dind: false
179 device: mi300_2
180 - timeout_in_minutes: 70
180 + timeout_in_minutes: 45
181 depends_on:
182 - image-build-amd
183 source_file_dependencies:
.buildkite/test_areas/distributed.yaml
+1
@@ -41,6 +41,7 @@ steps:
41 amd:
42 dind: false
43 device: mi300_2
44 + timeout_in_minutes: 45
45 depends_on:
46 - image-build-amd
47 source_file_dependencies:
.buildkite/test_areas/engine.yaml
+10 -8
@@ -28,8 +28,9 @@ steps:
28 - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
29 mirror:
30 amd:
31 - device: mi325_1
32 - timeout_in_minutes: 50
31 + dind: false
32 + device: mi300_1
33 + timeout_in_minutes: 40
34 depends_on:
35 - image-build-amd
36
@@ -44,8 +45,8 @@ steps:
45 - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
46 mirror:
47 amd:
47 - device: mi325_1
48 - timeout_in_minutes: 55
48 + device: mi250_1
49 + timeout_in_minutes: 45
50 depends_on:
51 - image-build-amd
52
@@ -60,8 +61,8 @@ steps:
61 - pytest -v -s v1/e2e/general/test_async_scheduling.py
62 mirror:
63 amd:
63 - device: mi325_1
64 - timeout_in_minutes: 70
64 + device: mi250_1
65 + timeout_in_minutes: 55
66 depends_on:
67 - image-build-amd
68
@@ -76,8 +77,8 @@ steps:
77 - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
78 mirror:
79 amd:
79 - device: mi325_1
80 - timeout_in_minutes: 60
80 + device: mi250_1
81 + timeout_in_minutes: 50
82 depends_on:
83 - image-build-amd
84 source_file_dependencies:
@@ -116,6 +117,7 @@ steps:
117 amd:
118 dind: false
119 device: mi300_2
120 + timeout_in_minutes: 30
121 depends_on:
122 - image-build-amd
123
.buildkite/test_areas/entrypoints.yaml
+16 -9
@@ -30,9 +30,9 @@ steps:
30 - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
31 mirror:
32 amd:
33 - device: mi325_1
34 - # TODO(akaratza): Test after Torch >= 2.12 bump
35 - soft_fail: true
33 + dind: false
34 + device: mi300_1
35 + timeout_in_minutes: 55
36 depends_on:
37 - image-build-amd
38
@@ -52,7 +52,9 @@ steps:
52 - pytest -v -s entrypoints/scale_out
53 mirror:
54 amd:
55 - device: mi325_1
55 + dind: false
56 + device: mi300_1
57 + timeout_in_minutes: 65
58 depends_on:
59 - image-build-amd
60
@@ -70,7 +72,8 @@ steps:
72 - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
73 mirror:
74 amd:
73 - device: mi325_1
75 + dind: false
76 + device: mi300_1
77 timeout_in_minutes: 65
78 depends_on:
79 - image-build-amd
@@ -90,8 +93,9 @@ steps:
93 - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
94 mirror:
95 amd:
93 - device: mi325_1
94 - timeout_in_minutes: 80
96 + dind: false
97 + device: mi300_1
98 + timeout_in_minutes: 70
99 depends_on:
100 - image-build-amd
101
@@ -113,7 +117,8 @@ steps:
117 - pytest -v -s entrypoints/anthropic
118 mirror:
119 amd:
116 - device: mi325_1
120 + dind: false
121 + device: mi300_1
122 timeout_in_minutes: 65
123 depends_on:
124 - image-build-amd
@@ -176,7 +181,9 @@ steps:
181 - pytest -s entrypoints/openai/correctness/
182 mirror:
183 amd:
179 - device: mi325_1
184 + dind: false
185 + device: mi300_1
186 + timeout_in_minutes: 30
187 depends_on:
188 - image-build-amd
189 source_file_dependencies:
.buildkite/test_areas/expert_parallelism.yaml
+1
@@ -18,6 +18,7 @@ steps:
18 amd:
19 dind: false
20 device: mi300_1
21 + timeout_in_minutes: 30
22 depends_on:
23 - image-build-amd
24 source_file_dependencies:
.buildkite/test_areas/kernels.yaml
+8 -4
@@ -80,7 +80,8 @@ steps:
80 parallelism: 2
81 mirror:
82 amd:
83 - device: mi325_1
83 + dind: false
84 + device: mi300_1
85 timeout_in_minutes: 90
86 depends_on:
87 - image-build-amd
@@ -118,7 +119,9 @@ steps:
119 parallelism: 2
120 mirror:
121 amd:
121 - device: mi325_1
122 + dind: false
123 + device: mi300_1
124 + timeout_in_minutes: 120
125 source_file_dependencies:
126 - csrc/quantization/
127 - vllm/model_executor/layers/quantization
@@ -148,8 +151,9 @@ steps:
151 parallelism: 5
152 mirror:
153 amd:
151 - device: mi325_1
152 - timeout_in_minutes: 65
154 + dind: false
155 + device: mi300_1
156 + timeout_in_minutes: 55
157 source_file_dependencies:
158 - csrc/quantization/cutlass_w8a8/moe/
159 - csrc/moe/
.buildkite/test_areas/lm_eval.yaml
+4 -3
@@ -14,8 +14,9 @@ steps:
14 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
15 mirror:
16 amd:
17 - device: mi325_1
18 - timeout_in_minutes: 55
17 + dind: false
18 + device: mi300_1
19 + timeout_in_minutes: 45
20 depends_on:
21 - image-build-amd
22 source_file_dependencies:
@@ -141,7 +142,7 @@ steps:
142 amd:
143 dind: false
144 device: mi300_8
144 - timeout_in_minutes: 60
145 + timeout_in_minutes: 40
146 depends_on:
147 - image-build-amd
148 commands:
.buildkite/test_areas/lora.yaml
+4 -3
@@ -14,9 +14,10 @@ steps:
14 parallelism: 4
15 mirror:
16 amd:
17 - device: mi325_1
17 + dind: false
18 + device: mi300_1
19 working_dir: "/vllm-workspace/tests"
19 - timeout_in_minutes: 65
20 + timeout_in_minutes: 85
21 source_file_dependencies:
22 - vllm/lora
23 - tests/lora
@@ -46,4 +47,4 @@ steps:
47 - pytest -v -s -x lora/test_qwen3_with_multi_loras.py
48 - pytest -v -s -x lora/test_olmoe_tp.py
49 - pytest -v -s -x lora/test_gptoss_tp.py
49 - - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
\ No newline at end of file
50 + - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
.buildkite/test_areas/misc.yaml
+15 -9
@@ -25,7 +25,7 @@ steps:
25 amd:
26 dind: false
27 device: mi300_1
28 - timeout_in_minutes: 75
28 + timeout_in_minutes: 50
29 depends_on:
30 - image-build-amd
31
@@ -59,7 +59,9 @@ steps:
59 - pytest -v -s v1/test_outputs.py
60 mirror:
61 amd:
62 - device: mi325_1
62 + dind: false
63 + device: mi300_1
64 + timeout_in_minutes: 70
65 depends_on:
66 - image-build-amd
67
@@ -113,8 +115,9 @@ steps:
115 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
116 mirror:
117 amd:
116 - device: mi325_1
117 - timeout_in_minutes: 75
118 + dind: false
119 + device: mi300_1
120 + timeout_in_minutes: 65
121 depends_on:
122 - image-build-amd
123
@@ -233,7 +236,9 @@ steps:
236 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
237 mirror:
238 amd:
236 - device: mi325_1
239 + dind: false
240 + device: mi300_1
241 + timeout_in_minutes: 75
242 source_file_dependencies:
243 - vllm/entrypoints
244 - vllm/multimodal
@@ -269,10 +274,11 @@ steps:
274 - pytest -v -s v1/tracing
275 mirror:
276 amd:
272 - device: mi325_2
277 + dind: false
278 + device: mi300_2
279 + timeout_in_minutes: 30
280 depends_on:
281 - image-build-amd
275 - optional: true
282
283 - label: Python-only Installation
284 key: python-only-installation
@@ -287,8 +293,8 @@ steps:
293 - bash standalone_tests/python_only_compile.sh
294 mirror:
295 amd:
290 - device: mi325_1
291 - timeout_in_minutes: 45
296 + device: mi250_1
297 + timeout_in_minutes: 55
298 depends_on:
299 - image-build-amd
300 source_file_dependencies:
.buildkite/test_areas/model_executor.yaml
+1
@@ -30,6 +30,7 @@ steps:
30 amd:
31 dind: false
32 device: mi300_1
33 + timeout_in_minutes: 60
34 depends_on:
35 - image-build-amd
36 source_file_dependencies:
.buildkite/test_areas/models_basic.yaml
+3 -1
@@ -42,7 +42,9 @@ steps:
42 - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py
43 mirror:
44 amd:
45 - device: mi325_1
45 + dind: false
46 + device: mi300_1
47 + timeout_in_minutes: 50
48 depends_on:
49 - image-build-amd
50
.buildkite/test_areas/models_language.yaml
+8 -4
@@ -17,6 +17,7 @@ steps:
17 amd:
18 dind: false
19 device: mi300_1
20 + timeout_in_minutes: 45
21 depends_on:
22 - image-build-amd
23
@@ -39,6 +40,7 @@ steps:
40 amd:
41 dind: false
42 device: mi300_1
43 + timeout_in_minutes: 40
44 depends_on:
45 - image-build-amd
46 source_file_dependencies:
@@ -69,8 +71,9 @@ steps:
71 parallelism: 2
72 mirror:
73 amd:
72 - device: mi325_1
73 - timeout_in_minutes: 70
74 + dind: false
75 + device: mi300_1
76 + timeout_in_minutes: 60
77 depends_on:
78 - image-build-amd
79 commands:
@@ -130,8 +133,9 @@ steps:
133 - pytest -v -s models/language/pooling -m 'not core_model'
134 mirror:
135 amd:
133 - device: mi325_1
134 - timeout_in_minutes: 120
136 + dind: false
137 + device: mi300_1
138 + timeout_in_minutes: 95
139 depends_on:
140 - image-build-amd
141
.buildkite/test_areas/models_multimodal.yaml
+18 -7
@@ -14,7 +14,9 @@ steps:
14 - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
15 mirror:
16 amd:
17 - device: mi325_1
17 + dind: false
18 + device: mi300_1
19 + timeout_in_minutes: 65
20 depends_on:
21 - image-build-amd
22
@@ -31,7 +33,9 @@ steps:
33 - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
34 mirror:
35 amd:
34 - device: mi325_1
36 + dind: false
37 + device: mi300_1
38 + timeout_in_minutes: 55
39 depends_on:
40 - image-build-amd
41
@@ -47,7 +51,8 @@ steps:
51 - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
52 mirror:
53 amd:
50 - device: mi325_1
54 + device: mi250_1
55 + timeout_in_minutes: 55
56 depends_on:
57 - image-build-amd
58
@@ -65,7 +70,9 @@ steps:
70 - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
71 mirror:
72 amd:
68 - device: mi325_1
73 + dind: false
74 + device: mi300_1
75 + timeout_in_minutes: 50
76 depends_on:
77 - image-build-amd
78
@@ -109,6 +116,7 @@ steps:
116 amd:
117 dind: false
118 device: mi300_1
119 + timeout_in_minutes: 35
120 depends_on:
121 - image-build-amd
122 source_file_dependencies:
@@ -131,7 +139,9 @@ steps:
139 - pytest -v -s models/multimodal/test_mapping.py
140 mirror:
141 amd:
134 - device: mi325_1
142 + dind: false
143 + device: mi300_1
144 + timeout_in_minutes: 90
145 depends_on:
146 - image-build-amd
147
@@ -166,8 +176,9 @@ steps:
176 - pytest -v -s models/multimodal/pooling -m 'not core_model'
177 mirror:
178 amd:
169 - device: mi325_1
170 - timeout_in_minutes: 75
179 + dind: false
180 + device: mi300_1
181 + timeout_in_minutes: 60
182 depends_on:
183 - image-build-amd
184 source_file_dependencies:
.buildkite/test_areas/pytorch.yaml
+1 -7
@@ -107,13 +107,6 @@ steps:
107 - tests/compile/passes
108 commands:
109 - pytest -s -v compile/passes --ignore compile/passes/distributed
110 - mirror:
111 - amd:
112 - dind: false
113 - device: mi300_1
114 - timeout_in_minutes: 65
115 - depends_on:
116 - - image-build-amd
110
111 - label: PyTorch Fullgraph Smoke Test
112 device: h200_35gb
@@ -236,6 +229,7 @@ steps:
229 amd:
230 dind: false
231 device: mi300_1
232 + timeout_in_minutes: 30
233 depends_on:
234 - image-build-amd
235 source_file_dependencies:
.buildkite/test_areas/samplers.yaml
+11 -1
@@ -19,8 +19,18 @@ steps:
19 - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers
20 mirror:
21 amd:
22 - device: mi325_1
22 + device: mi250_1
23 + timeout_in_minutes: 40
24 depends_on:
25 - image-build-amd
26 + source_file_dependencies:
27 + - vllm/model_executor/layers
28 + - vllm/sampling_metadata.py
29 + - vllm/v1/sample/
30 + - vllm/entrypoints/generate/beam_search/
31 + - tests/samplers
32 + - tests/conftest.py
33 + - vllm/_aiter_ops.py
34 + - vllm/platforms/rocm.py
35 commands:
36 - pytest -v -s samplers
.buildkite/test_areas/spec_decode.yaml
+11 -9
@@ -14,8 +14,9 @@ steps:
14 - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
15 mirror:
16 amd:
17 - device: mi325_1
18 - timeout_in_minutes: 60
17 + dind: false
18 + device: mi300_1
19 + timeout_in_minutes: 55
20 depends_on:
21 - image-build-amd
22 source_file_dependencies:
@@ -53,8 +54,9 @@ steps:
54 - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
55 mirror:
56 amd:
56 - device: mi325_1
57 - timeout_in_minutes: 65
57 + dind: false
58 + device: mi300_1
59 + timeout_in_minutes: 75
60 depends_on:
61 - image-build-amd
62 source_file_dependencies:
@@ -92,10 +94,9 @@ steps:
94 - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
95 mirror:
96 amd:
95 - device: mi325_1
96 - timeout_in_minutes: 55
97 - # TODO(akaratza): Test after Torch >= 2.12 bump
98 - soft_fail: true
97 + dind: false
98 + device: mi300_1
99 + timeout_in_minutes: 35
100 depends_on:
101 - image-build-amd
102 source_file_dependencies:
@@ -119,7 +120,8 @@ steps:
120 - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
121 mirror:
122 amd:
122 - device: mi325_1
123 + dind: false
124 + device: mi300_1
125 timeout_in_minutes: 55
126 depends_on:
127 - image-build-amd
.buildkite/test_areas/weight_loading.yaml
+1
@@ -17,6 +17,7 @@ steps:
17 amd:
18 dind: false
19 device: mi300_2
20 + timeout_in_minutes: 35
21 depends_on:
22 - image-build-amd
23 commands:
docker/Dockerfile.rocm
+2
@@ -745,6 +745,7 @@ ENV MIOPEN_DEBUG_CONV_GEMM=0
745 # See: https://github.com/ROCm/rocm-libraries/issues/6266
746 ENV HSA_ENABLE_IPC_MODE_LEGACY=1
747 ENV UCX_RMA_PPLN_ENABLE=y
748 +ENV UCX_ROCM_COPY_SIGPOOL_MAX_ELEMS=inf
749
750 # ROCm profiler limits workaround.
751 RUN echo "ROCTRACER_MAX_EVENTS=10000000" > ${COMMON_WORKDIR}/libkineto.conf
@@ -835,6 +836,7 @@ COPY --from=export_vllm /docker ${COMMON_WORKDIR}/vllm/docker
836 # See: https://github.com/ROCm/rocm-libraries/issues/6266
837 ENV HSA_ENABLE_IPC_MODE_LEGACY=1
838 ENV UCX_RMA_PPLN_ENABLE=y
839 +ENV UCX_ROCM_COPY_SIGPOOL_MAX_ELEMS=inf
840
841 ENV TOKENIZERS_PARALLELISM=false
842