A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 08:30 UTC
[ROCm][Bugfix] Patch for the `Multi-Modal Processor Test` group (#29702)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Nov 28, 2025 at 19:31 UTC
ea3370b428e1b192d29a4451d439f4ed0895f1f3
4 files changed
+104
-28
docker/Dockerfile.rocm
+18
-1
@@ -65,6 +65,8 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
65
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
66
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
67
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
68
+# Centralized v1 package - copied to both test and final stages
69
+COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1
70
71
# -----------------------
72
# Test vLLM image
@@ -88,10 +90,22 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace
90
91
# install development dependencies (for testing)
92
RUN cd /vllm-workspace \
91
- && rm -rf vllm \
93
&& python3 -m pip install -e tests/vllm_test_utils \
94
&& python3 -m pip install pytest-shard
95
96
+# enable fast downloads from hf (for testing)
97
+RUN --mount=type=cache,target=/root/.cache/uv \
98
+ uv pip install --system hf_transfer
99
+ENV HF_HUB_ENABLE_HF_TRANSFER=1
100
+
101
+# Copy in the v1 package
102
+COPY --from=export_vllm /vllm_v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
103
+
104
+# Source code is used in the `python_only_compile.sh` test
105
+# We hide it inside `src/` so that this source code
106
+# will not be imported by other tests
107
+RUN mkdir src && mv vllm src/vllm
108
+
109
# -----------------------
110
# Final vLLM image
111
FROM base AS final
@@ -116,6 +130,9 @@ RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
130
&& pip uninstall -y vllm \
131
&& uv pip install --system *.whl
132
133
+# Copy in the v1 package
134
+COPY --from=export_vllm /vllm_v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
135
+
136
ARG COMMON_WORKDIR
137
138
# Copy over the benchmark scripts as well
docker/Dockerfile.rocm_base
+22
-4
@@ -5,6 +5,8 @@ ARG PYTORCH_BRANCH="1c57644d"
5
ARG PYTORCH_VISION_BRANCH="v0.23.0"
6
ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git"
7
ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git"
8
+ARG PYTORCH_AUDIO_BRANCH="v2.9.0"
9
+ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git"
10
ARG FA_BRANCH="0e60e394"
11
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
12
ARG AITER_BRANCH="59bd8ff2"
@@ -23,6 +25,7 @@ ENV AITER_ROCM_ARCH=gfx942;gfx950
25
ENV HSA_NO_SCRATCH_RECLAIM=1
26
27
ARG PYTHON_VERSION=3.12
28
+ENV PYTHON_VERSION=${PYTHON_VERSION}
29
30
RUN mkdir -p /app
31
WORKDIR /app
@@ -45,6 +48,7 @@ RUN apt-get update -y \
48
&& python3 --version && python3 -m pip --version
49
50
RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' pybind11 Cython
51
+RUN apt-get update && apt-get install -y libjpeg-dev libsox-dev libsox-fmt-all sox && rm -rf /var/lib/apt/lists/*
52
53
FROM base AS build_triton
54
ARG TRITON_BRANCH
@@ -66,11 +70,14 @@ RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install
70
FROM base AS build_pytorch
71
ARG PYTORCH_BRANCH
72
ARG PYTORCH_VISION_BRANCH
73
+ARG PYTORCH_AUDIO_BRANCH
74
ARG PYTORCH_REPO
75
ARG PYTORCH_VISION_REPO
76
+ARG PYTORCH_AUDIO_REPO
77
+
78
RUN git clone ${PYTORCH_REPO} pytorch
72
-RUN cd pytorch && git checkout ${PYTORCH_BRANCH} && \
73
- pip install -r requirements.txt && git submodule update --init --recursive \
79
+RUN cd pytorch && git checkout ${PYTORCH_BRANCH} \
80
+ && pip install -r requirements.txt && git submodule update --init --recursive \
81
&& python3 tools/amd_build/build_amd.py \
82
&& CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \
83
&& pip install dist/*.whl
@@ -78,8 +85,15 @@ RUN git clone ${PYTORCH_VISION_REPO} vision
85
RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
86
&& python3 setup.py bdist_wheel --dist-dir=dist \
87
&& pip install dist/*.whl
88
+RUN git clone ${PYTORCH_AUDIO_REPO} audio
89
+RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \
90
+ && git submodule update --init --recursive \
91
+ && pip install -r requirements.txt \
92
+ && python3 setup.py bdist_wheel --dist-dir=dist \
93
+ && pip install dist/*.whl
94
RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install \
82
- && cp /app/vision/dist/*.whl /app/install
95
+ && cp /app/vision/dist/*.whl /app/install \
96
+ && cp /app/audio/dist/*.whl /app/install
97
98
FROM base AS build_fa
99
ARG FA_BRANCH
@@ -130,6 +144,8 @@ ARG PYTORCH_BRANCH
144
ARG PYTORCH_VISION_BRANCH
145
ARG PYTORCH_REPO
146
ARG PYTORCH_VISION_REPO
147
+ARG PYTORCH_AUDIO_BRANCH
148
+ARG PYTORCH_AUDIO_REPO
149
ARG FA_BRANCH
150
ARG FA_REPO
151
ARG AITER_BRANCH
@@ -141,7 +157,9 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
157
&& echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \
158
&& echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \
159
&& echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \
160
+ && echo "PYTORCH_AUDIO_BRANCH: ${PYTORCH_AUDIO_BRANCH}" >> /app/versions.txt \
161
+ && echo "PYTORCH_AUDIO_REPO: ${PYTORCH_AUDIO_REPO}" >> /app/versions.txt \
162
&& echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \
163
&& echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \
164
&& echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
147
- && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt
165
+ && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt
\ No newline at end of file
requirements/rocm-test.txt
+56
-22
@@ -1,51 +1,85 @@
1
# Common dependencies
2
-r common.txt
3
+
4
+# Test infrastructure
5
tblib==3.1.0
4
-bm25s==0.2.13
5
-pystemmer==3.0.0
6
+pytest==8.3.5
7
+pytest-asyncio==0.24.0
8
+pytest-timeout==2.3.1
9
+pytest-cov==6.3.0
10
+pytest-forked==1.6.0
11
+pytest-rerunfailures==14.0
12
+pytest-shard==0.1.2
13
+
14
+# Async/HTTP dependencies
15
+anyio==4.6.2.post1
16
+ # via httpx, starlette
17
+aiohttp==3.13.0
18
+ # via gpt-oss
19
+httpx==0.27.2
20
+ # HTTP testing
21
7
-# Entrypoints test
8
-# librosa==0.10.2.post1 # required by audio tests in entrypoints/openai
22
+# Audio processing dependencies
23
audioread==3.0.1
24
+ # via librosa
25
cffi==1.17.1
26
+ # via soundfile
27
decorator==5.2.1
28
+ # via librosa
29
lazy-loader==0.4
30
+ # via librosa
31
platformdirs==4.3.6
32
+ # via pooch
33
pooch==1.8.2
15
-#pycparse==2.22
34
+ # via librosa
35
soundfile==0.13.1
36
+ # via librosa
37
soxr==0.5.0.post1
38
+ # via librosa
39
librosa==0.10.2.post1
40
20
-# Entrypoints test
21
-#vllm[video] # required by entrypoints/openai/test_video.py
22
-decord==0.6.0
23
-
24
-# Entrypoints test
25
-#sentence-transformers # required by entrypoints/openai/test_score.py
26
-sentence-transformers==3.4.1
27
-
28
-# Basic Models Test
29
-matplotlib==3.10.3
41
+# Retrieval and search
42
+bm25s==0.2.13
43
+ # via mteb
44
+pystemmer==3.0.0
45
+ # via mteb
46
31
-# Multi-Modal Models Test (Extended) 3
47
+# Multi-modal processing
48
blobfile==3.0.0
49
+ # Multi-Modal Models Test
50
+decord==0.6.0
51
+ # video processing, required by entrypoints/openai/test_video.py
52
34
-# Required for openai schema test.
53
+# OpenAI compatibility and testing
54
+gpt-oss==0.0.8
55
+ # OpenAI compatibility tests
56
schemathesis==3.39.15
57
+ # OpenAI schema test
58
37
-# Required for mteb test
38
-mteb[bm25s]>=1.38.11, <2
39
-
40
-# Required for eval tests
59
+# Evaluation and benchmarking
60
lm-eval[api] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d
61
43
-# Required for multiprocessed tests that use spawn method
62
+# Required for multiprocessed tests that use spawn method, Datasets and Evaluate Test
63
multiprocess==0.70.16
64
65
# Plugins test
66
terratorch @ git+https://github.com/IBM/terratorch.git@07184fcf91a1324f831ff521dd238d97fe350e3e
67
torchgeo==0.7.0
68
+ # via terratorch
69
+# MTEB Benchmark Test
70
+mteb==2.1.2
71
+
72
+# Data processing
73
+xgrammar @ git+https://github.com/mlc-ai/xgrammar.git@eafd4db51b78acc64b3f0764ef27dfd206c28628
74
+ # Test async scheduling
75
+
76
+# Utilities
77
+num2words==0.5.14
78
+ # via lm-eval
79
+pqdm==0.2.0
80
+ # via lm-eval
81
82
# Required for suffix decoding test
83
arctic-inference == 0.1.1
84
+# Required for Nemotron test
85
+open-clip-torch==2.32.0
tests/models/multimodal/processing/test_tensor_schema.py
+8
-1
@@ -30,6 +30,7 @@ from vllm.model_executor.models.interfaces import (
30
from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
31
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
32
from vllm.multimodal.utils import group_mm_kwargs_by_modality
33
+from vllm.platforms import current_platform
34
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
35
from vllm.utils.collection_utils import is_list_of
36
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -176,6 +177,12 @@ def test_model_tensor_schema(model_id: str):
177
exist_overrides=model_info.hf_overrides,
178
)
179
180
+ # ROCm: Detect if model uses AWQ quantization and set appropriate dtype
181
+ if "awq" in model_id.lower() and current_platform.is_rocm():
182
+ dtype = "float16"
183
+ else:
184
+ dtype = model_info.dtype
185
+
186
model_config = ModelConfig(
187
model_id,
188
tokenizer=model_info.tokenizer or model_id,
@@ -187,7 +194,7 @@ def test_model_tensor_schema(model_id: str):
194
enable_prompt_embeds=model_info.require_embed_inputs,
195
enable_mm_embeds=model_info.require_embed_inputs,
196
enforce_eager=model_info.enforce_eager,
190
- dtype=model_info.dtype,
197
+ dtype=dtype,
198
)
199
200
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)