[Hardware][AMD][CI] Tweak mirrored tests; improve CI base dependency change detection (#46930)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Matt committed
Jun 28, 2026 at 02:07 UTC
a2a92cbbaac1175de96fc6f4712b4bba789a0c02
6 files changed
+30
-35
.buildkite/scripts/ci-bake-rocm.sh
+1
-1
@@ -15,7 +15,7 @@ set -euo pipefail
15
16
DEFAULT_REPO_SLUG="vllm-project/vllm"
17
DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
18
-DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base tools/install_torchcodec_rocm.sh tests/vllm_test_utils"
18
+DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh"
19
DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
20
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base"
21
DEFAULT_CI_BASE_METADATA_VERSION="1"
.buildkite/test-amd.yaml
+12
-21
@@ -438,7 +438,7 @@ steps:
438
#----------------------------------------------------- mi300 · basic_correctness -----------------------------------------------------#
439
440
- label: Basic Correctness # TBD
441
- timeout_in_minutes: 40
441
+ timeout_in_minutes: 50
442
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
443
agent_pool: mi300_1
444
fast_check: true
@@ -456,7 +456,7 @@ steps:
456
- pytest -v -s basic_correctness/test_cpu_offload.py
457
458
- label: Distributed Model Tests (2 GPUs) # TBD
459
- timeout_in_minutes: 180
459
+ timeout_in_minutes: 65
460
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
461
agent_pool: mi300_2
462
num_gpus: 2
@@ -678,7 +678,7 @@ steps:
678
- pytest -v -s distributed/test_eplb_spec_decode.py
679
680
- label: Distributed Tests (2xH100-2xMI300) # TBD
681
- timeout_in_minutes: 180
681
+ timeout_in_minutes: 30
682
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
683
agent_pool: mi300_2
684
num_gpus: 2
@@ -1199,7 +1199,7 @@ steps:
1199
#--------------------------------------------------------- mi300 · examples ----------------------------------------------------------#
1200
1201
- label: Examples # TBD
1202
- timeout_in_minutes: 180
1202
+ timeout_in_minutes: 45
1203
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1204
agent_pool: mi300_1
1205
optional: true
@@ -1212,7 +1212,7 @@ steps:
1212
commands:
1213
- pip install tensorizer
1214
# Basic
1215
- - python3 basic/offline_inference/chat.py --attention-backend TRITON_ATTN
1215
+ - python3 basic/offline_inference/chat.py
1216
- python3 basic/offline_inference/generate.py --model facebook/opt-125m
1217
- python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
1218
- python3 basic/offline_inference/classify.py
@@ -1220,11 +1220,8 @@ steps:
1220
- python3 basic/offline_inference/score.py
1221
# Multi-modal models
1222
- python3 generate/multimodal/audio_language_offline.py --seed 0
1223
- # These two examples import transformers before vllm, so on ROCm the HIP context
1224
- # is initialized in the parent before vllm sets this guard, poisoning fork. Set it
1225
- # inline to keep torch.cuda.is_available() fork-safe (see vllm/env_override.py).
1226
- - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 generate/multimodal/vision_language_offline.py --seed 0
1227
- - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
1223
+ - python3 generate/multimodal/vision_language_offline.py --seed 0
1224
+ - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
1225
- python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
1226
# Pooling models
1227
- python3 pooling/embed/vision_embedding_offline.py --seed 0
@@ -1666,10 +1663,7 @@ steps:
1663
- pytest -v -s tests/models/test_transformers.py
1664
- pytest -v -s tests/models/multimodal/test_mapping.py
1665
- python3 examples/basic/offline_inference/chat.py
1669
- # This example imports transformers before vllm, so on ROCm the HIP context is
1670
- # initialized in the parent before vllm sets this guard, poisoning fork. Set it
1671
- # inline to keep torch.cuda.is_available() fork-safe (see vllm/env_override.py).
1672
- - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 examples/generate/multimodal/vision_language_offline.py --model-type qwen2_5_vl
1666
+ - python3 examples/generate/multimodal/vision_language_offline.py --model-type qwen2_5_vl
1667
- VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/generate/multimodal/audio_language_offline.py --model-type whisper
1668
1669
#---------------------------------------------------------- mi300 · plugins ----------------------------------------------------------#
@@ -2773,7 +2767,7 @@ steps:
2767
#--------------------------------------------------------- mi355 · examples ----------------------------------------------------------#
2768
2769
- label: Examples # TBD
2776
- timeout_in_minutes: 180
2770
+ timeout_in_minutes: 45
2771
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
2772
agent_pool: mi355_1
2773
working_dir: "/vllm-workspace/examples"
@@ -2785,7 +2779,7 @@ steps:
2779
commands:
2780
- pip install tensorizer
2781
# Basic
2788
- - python3 basic/offline_inference/chat.py --attention-backend TRITON_ATTN
2782
+ - python3 basic/offline_inference/chat.py
2783
- python3 basic/offline_inference/generate.py --model facebook/opt-125m
2784
- python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
2785
- python3 basic/offline_inference/classify.py
@@ -2793,11 +2787,8 @@ steps:
2787
- python3 basic/offline_inference/score.py
2788
# Multi-modal models
2789
- python3 generate/multimodal/audio_language_offline.py --seed 0
2796
- # These two examples import transformers before vllm, so on ROCm the HIP context
2797
- # is initialized in the parent before vllm sets this guard, poisoning fork. Set it
2798
- # inline to keep torch.cuda.is_available() fork-safe (see vllm/env_override.py).
2799
- - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 generate/multimodal/vision_language_offline.py --seed 0
2800
- - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
2790
+ - python3 generate/multimodal/vision_language_offline.py --seed 0
2791
+ - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
2792
- python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
2793
# Pooling models
2794
- python3 pooling/embed/vision_embedding_offline.py --seed 0
.buildkite/test_areas/basic_correctness.yaml
+1
-1
@@ -19,6 +19,6 @@ steps:
19
mirror:
20
amd:
21
device: mi325_1
22
- timeout_in_minutes: 40
22
+ timeout_in_minutes: 50
23
depends_on:
24
- image-build-amd
.buildkite/test_areas/distributed.yaml
-12
@@ -224,18 +224,6 @@ steps:
224
- pytest -v -s tests/v1/distributed/test_dbo.py
225
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
226
- pytest -v -s tests/distributed/test_packed_tensor.py
227
- mirror:
228
- amd:
229
- device: mi300_2
230
- timeout_in_minutes: 180
231
- depends_on:
232
- - image-build-amd
233
- commands:
234
- - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
235
- - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048
236
- - pytest -v -s tests/v1/distributed/test_dbo.py
237
- - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
238
- - pytest -v -s tests/distributed/test_packed_tensor.py
227
228
- label: Distributed Tests (2xB200)
229
key: distributed-tests-2xb200
.buildkite/test_areas/misc.yaml
+10
@@ -224,6 +224,16 @@ steps:
224
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
225
# https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
226
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
227
+ mirror:
228
+ amd:
229
+ device: mi325_1
230
+ source_file_dependencies:
231
+ - vllm/entrypoints
232
+ - vllm/multimodal
233
+ - examples/
234
+ - vllm/platforms/rocm.py
235
+ depends_on:
236
+ - image-build-amd
237
238
- label: Metrics, Tracing (2 GPUs)
239
key: metrics-tracing-2-gpus
docker/Dockerfile.rocm
+6
@@ -575,6 +575,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
575
ENV HF_XET_HIGH_PERFORMANCE=1
576
ENV HF_HUB_DOWNLOAD_TIMEOUT=60
577
578
+# Keep torch.cuda.is_available() fork-safe (see vllm/env_override.py).
579
+ENV PYTORCH_NVML_BASED_CUDA_CHECK=1
580
+
581
# Pre-install vLLM test dependencies.
582
COPY requirements/test/rocm.txt /tmp/rocm-test-reqs.txt
583
RUN --mount=type=cache,target=/root/.cache/uv \
@@ -695,6 +698,9 @@ ENV SAFETENSORS_FAST_GPU=1
698
# Performance environment variable.
699
ENV HIP_FORCE_DEV_KERNARG=1
700
701
+# Keep torch.cuda.is_available() fork-safe (see vllm/env_override.py).
702
+ENV PYTORCH_NVML_BASED_CUDA_CHECK=1
703
+
704
# Workaround for ROCm profiler limits
705
RUN echo "ROCTRACER_MAX_EVENTS=10000000" > ${COMMON_WORKDIR}/libkineto.conf
706
ENV KINETO_CONFIG="${COMMON_WORKDIR}/libkineto.conf"