@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 03f8d3a548
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 02:15 UTC

Update to transformers v5 (#30566)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: khluu <khluu000@gmail.com> Signed-off-by: Kevin H. Luu <khluu000@gmail.com> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: khluu <khluu000@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: jiang1.li <jiang1.li@intel.com>

Harry Mellor committed Apr 16, 2026 at 00:29 UTC 03f8d3a548ce9769f9fd89cb4505e8b77649c943
41 files changed +445 -115
.buildkite/scripts/hardware_ci/run-cpu-test.sh
+1 -1
@@ -16,5 +16,5 @@ echo "--- :docker: Building Docker image"
16 docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
17
18 # Run the image, setting --shm-size=4g for tensor parallel.
19 -docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 --shm-size=4g "$IMAGE_NAME" \
19 +docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 --shm-size=4g "$IMAGE_NAME" \
20 timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
.buildkite/test_areas/models_basic.yaml
+15 -1
@@ -4,7 +4,6 @@ depends_on:
4 steps:
5 - label: Basic Models Tests (Initialization)
6 timeout_in_minutes: 45
7 - device: h200_18gb
7 torch_nightly: true
8 source_file_dependencies:
9 - vllm/
@@ -73,3 +72,18 @@ steps:
72 - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
73 # Whisper needs spawn method to avoid deadlock
74 - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
75 +
76 +- label: Transformers Backward Compatibility Models Test
77 + working_dir: "/vllm-workspace/"
78 + optional: true
79 + soft_fail: true
80 + commands:
81 + - pip install transformers==4.57.5
82 + - pytest -v -s tests/models/test_initialization.py
83 + - pytest -v -s tests/models/test_transformers.py
84 + - pytest -v -s tests/models/multimodal/processing/
85 + - pytest -v -s tests/models/multimodal/test_mapping.py
86 + - python3 examples/offline_inference/basic/chat.py
87 + - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
88 + # Whisper needs spawn method to avoid deadlock
89 + - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
docker/Dockerfile
+5 -4
@@ -642,7 +642,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
642 else \
643 BITSANDBYTES_VERSION="${BITSANDBYTES_VERSION_X86}"; \
644 fi; \
645 - uv pip install --system accelerate hf_transfer modelscope \
645 + uv pip install --system accelerate modelscope \
646 "bitsandbytes>=${BITSANDBYTES_VERSION}" "timm${TIMM_VERSION}" "runai-model-streamer[s3,gcs,azure]${RUNAI_MODEL_STREAMER_VERSION}"
647
648 # ============================================================
@@ -756,9 +756,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
756 uv pip install --system -e tests/vllm_test_utils
757
758 # enable fast downloads from hf (for testing)
759 -RUN --mount=type=cache,target=/root/.cache/uv \
760 - uv pip install --system hf_transfer
761 -ENV HF_HUB_ENABLE_HF_TRANSFER 1
759 +ENV HF_XET_HIGH_PERFORMANCE 1
760 +
761 +# increase timeout for hf downloads (for testing)
762 +ENV HF_HUB_DOWNLOAD_TIMEOUT 60
763
764 # Copy in the v1 package for testing (it isn't distributed yet)
765 COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
docker/Dockerfile.cpu
+6
@@ -197,6 +197,12 @@ ADD ./.buildkite/ ./.buildkite/
197 RUN --mount=type=cache,target=/root/.cache/uv \
198 uv pip install -e tests/vllm_test_utils
199
200 +# enable fast downloads from hf (for testing)
201 +ENV HF_XET_HIGH_PERFORMANCE 1
202 +
203 +# increase timeout for hf downloads (for testing)
204 +ENV HF_HUB_DOWNLOAD_TIMEOUT 60
205 +
206 ######################### RELEASE IMAGE #########################
207 FROM base AS vllm-openai
208
docker/Dockerfile.nightly_torch
+4 -3
@@ -272,9 +272,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
272 uv pip install --system -e tests/vllm_test_utils
273
274 # enable fast downloads from hf (for testing)
275 -RUN --mount=type=cache,target=/root/.cache/uv \
276 - uv pip install --system hf_transfer
277 -ENV HF_HUB_ENABLE_HF_TRANSFER 1
275 +ENV HF_XET_HIGH_PERFORMANCE 1
276 +
277 +# increase timeout for hf downloads (for testing)
278 +ENV HF_HUB_DOWNLOAD_TIMEOUT 60
279
280 RUN --mount=type=cache,target=/root/.cache/uv \
281 uv pip install --system -r requirements/test/nightly-torch.txt
docker/Dockerfile.rocm
+4 -3
@@ -365,9 +365,10 @@ RUN cd /vllm-workspace \
365 && python3 -m pip install pytest-shard
366
367 # enable fast downloads from hf (for testing)
368 -RUN --mount=type=cache,target=/root/.cache/uv \
369 - uv pip install --system hf_transfer
370 -ENV HF_HUB_ENABLE_HF_TRANSFER=1
368 +ENV HF_XET_HIGH_PERFORMANCE=1
369 +
370 +# increase timeout for hf downloads (for testing)
371 +ENV HF_HUB_DOWNLOAD_TIMEOUT 60
372
373 # install audio decode package `torchcodec` from source (required due to
374 # ROCm and torch version mismatch) for tests with datasets package
docs/getting_started/installation/gpu.rocm.inc.md
+1 -1
@@ -240,7 +240,7 @@ uv pip install vllm==${VLLM_VERSION} \
240 # Install dependencies
241 pip install --upgrade numba \
242 scipy \
243 - huggingface-hub[cli,hf_transfer] \
243 + huggingface-hub[cli] \
244 setuptools_scm
245 pip install -r requirements/rocm.txt
246
requirements/common.txt
+2 -2
@@ -7,7 +7,7 @@ requests >= 2.26.0
7 tqdm
8 blake3
9 py-cpuinfo
10 -transformers >= 4.56.0, < 5
10 +transformers >= 4.56.0, != 5.0.*, != 5.1.*, != 5.2.*, != 5.3.*, != 5.4.*, != 5.5.0
11 tokenizers >= 0.21.1 # Required for fast incremental detokenization.
12 protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.* # Required by LlamaTokenizer, gRPC. CVE-2026-0994
13 fastapi[standard] >= 0.115.0 # Required by FastAPI's form models in the OpenAI API server's audio transcriptions endpoint.
@@ -37,7 +37,7 @@ pyyaml
37 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
38 setuptools>=77.0.3,<81.0.0; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
39 einops # Required for Qwen2-VL.
40 -compressed-tensors == 0.14.0.1 # required for compressed-tensors
40 +compressed-tensors == 0.15.0.1 # required for compressed-tensors
41 depyf==0.20.0 # required for profiling and debugging with compilation config
42 cloudpickle # allows pickling lambda functions in model_executor/models/registry.py
43 watchfiles # required for http server to monitor the updates of TLS files
requirements/test/cuda.in
+3 -3
@@ -18,7 +18,7 @@ httpx
18 librosa # required for audio tests
19 vector_quantize_pytorch # required for minicpmo_26 test
20 vocos # required for minicpmo_26 test
21 -peft>=0.15.0 # required for phi-4-mm test
21 +peft>=0.18.1 # required for phi-4-mm test
22 pqdm
23 ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
24 resampy # required for audio tests
@@ -39,8 +39,8 @@ opencv-python-headless >= 4.13.0 # required for video test
39 datamodel_code_generator # required for minicpm3 test
40 lm-eval[api]>=0.4.11 # required for model evaluation test
41 mteb[bm25s]>=2, <3 # required for mteb test
42 -transformers==4.57.5
43 -tokenizers==0.22.0
42 +transformers==5.5.3
43 +tokenizers==0.22.2
44 schemathesis>=3.39.15 # Required for openai schema test.
45 # quantization
46 bitsandbytes==0.49.2
requirements/test/cuda.txt
+10 -10
@@ -4,7 +4,7 @@ absl-py==2.1.0
4 # via
5 # rouge-score
6 # tensorboard
7 -accelerate==1.0.1
7 +accelerate==1.13.0
8 # via peft
9 aenum==3.1.16
10 # via lightly
@@ -248,7 +248,6 @@ filelock==3.16.1
248 # huggingface-hub
249 # ray
250 # torch
251 - # transformers
251 # virtualenv
252 fiona==1.10.1
253 # via torchgeo
@@ -331,7 +330,7 @@ h5py==3.13.0
330 # via terratorch
331 harfile==0.3.0
332 # via schemathesis
334 -hf-xet==1.1.7
333 +hf-xet==1.4.3
334 # via huggingface-hub
335 hiredis==3.0.0
336 # via tensorizer
@@ -345,9 +344,10 @@ httpx==0.27.2
344 # via
345 # -r requirements/test/cuda.in
346 # diffusers
347 + # huggingface-hub
348 # perceptron
349 # schemathesis
350 -huggingface-hub==0.36.2
350 +huggingface-hub==1.10.2
351 # via
352 # accelerate
353 # datasets
@@ -756,7 +756,7 @@ pathvalidate==3.2.1
756 # via pytablewriter
757 patsy==1.0.1
758 # via statsmodels
759 -peft==0.16.0
759 +peft==0.18.1
760 # via -r requirements/test/cuda.in
761 perceptron==0.1.4
762 # via -r requirements/test/cuda.in
@@ -982,7 +982,7 @@ referencing==0.35.1
982 # via
983 # jsonschema
984 # jsonschema-specifications
985 -regex==2024.9.11
985 +regex==2026.2.28
986 # via
987 # diffusers
988 # nltk
@@ -1002,7 +1002,6 @@ requests==2.32.3
1002 # google-api-core
1003 # google-cloud-storage
1004 # gpt-oss
1005 - # huggingface-hub
1005 # lightly
1006 # lm-eval
1007 # mistral-common
@@ -1015,7 +1014,6 @@ requests==2.32.3
1014 # starlette-testclient
1015 # tacoreader
1016 # tiktoken
1018 - # transformers
1017 # wandb
1018 resampy==0.4.3
1019 # via -r requirements/test/cuda.in
@@ -1216,7 +1214,7 @@ timm==1.0.17
1214 # segmentation-models-pytorch
1215 # terratorch
1216 # torchgeo
1219 -tokenizers==0.22.0
1217 +tokenizers==0.22.2
1218 # via
1219 # -c requirements/common.txt
1220 # -r requirements/test/cuda.in
@@ -1295,7 +1293,7 @@ tqdm==4.67.3
1293 # tacoreader
1294 # terratorch
1295 # transformers
1298 -transformers==4.57.5
1296 +transformers==5.5.3
1297 # via
1298 # -c requirements/common.txt
1299 # -r requirements/test/cuda.in
@@ -1317,7 +1315,9 @@ typepy==1.3.2
1315 typer==0.15.2
1316 # via
1317 # fastsafetensors
1318 + # huggingface-hub
1319 # perceptron
1320 + # transformers
1321 types-python-dateutil==2.9.0.20241206
1322 # via arrow
1323 typeshed-client==2.8.2
requirements/test/nightly-torch.txt
+2 -2
@@ -29,8 +29,8 @@ opencv-python-headless >= 4.13.0 # required for video test
29 datamodel_code_generator # required for minicpm3 test
30 lm-eval[api]>=0.4.11 # required for model evaluation test
31 mteb[bm25s]>=2, <3 # required for mteb test
32 -transformers==4.57.5
33 -tokenizers==0.22.0
32 +transformers==5.5.3
33 +tokenizers==0.22.2
34 schemathesis>=3.39.15 # Required for openai schema test.
35 # quantization
36 bitsandbytes>=0.49.2
requirements/test/rocm.in
+2 -3
@@ -38,8 +38,8 @@ opencv-python-headless>=4.13.0 # required for video test
38 datamodel_code_generator # required for minicpm3 test
39 lm-eval[api]>=0.4.11 # required for model evaluation test
40 mteb[bm25s]>=2, <3 # required for mteb test
41 -transformers==4.57.5
42 -tokenizers==0.22.0
41 +transformers==5.5.3
42 +tokenizers==0.22.2
43 schemathesis>=3.39.15 # Required for openai schema test
44 # quantization
45 bitsandbytes==0.49.2
@@ -82,4 +82,3 @@ plotly # required for perf comparison html report
82 rapidfuzz
83 torchgeo==0.7.0
84 multiprocess==0.70.16
85 -huggingface-hub==0.36.2
requirements/test/rocm.txt
+15 -16
@@ -39,7 +39,7 @@ annotated-doc==0.0.4
39 # typer
40 annotated-types==0.7.0
41 # via pydantic
42 -anthropic==0.89.0
42 +anthropic==0.93.0
43 # via
44 # -c requirements/common.txt
45 # -r requirements/test/../common.txt
@@ -172,7 +172,7 @@ colorful==0.5.8
172 # via ray
173 colorlog==6.10.1
174 # via optuna
175 -compressed-tensors==0.14.0.1
175 +compressed-tensors==0.15.0.1
176 # via
177 # -c requirements/common.txt
178 # -r requirements/test/../common.txt
@@ -269,9 +269,9 @@ fastapi==0.135.2
269 # model-hosting-container-standards
270 fastapi-cli==0.0.24
271 # via fastapi
272 -fastapi-cloud-cli==0.15.1
272 +fastapi-cloud-cli==0.16.1
273 # via fastapi-cli
274 -fastar==0.9.0
274 +fastar==0.10.0
275 # via fastapi-cloud-cli
276 fastparquet==2026.3.0
277 # via genai-perf
@@ -290,7 +290,6 @@ filelock==3.25.2
290 # python-discovery
291 # ray
292 # torch
293 - # transformers
293 # virtualenv
294 fiona==1.10.1
295 # via torchgeo
@@ -384,7 +383,7 @@ h5py==3.16.0
383 # via terratorch
384 harfile==0.4.0
385 # via schemathesis
387 -hf-xet==1.4.2
386 +hf-xet==1.4.3
387 # via huggingface-hub
388 hiredis==3.3.1
389 # via tensorizer
@@ -403,6 +402,7 @@ httpx==0.27.2
402 # diffusers
403 # fastapi
404 # fastapi-cloud-cli
405 + # huggingface-hub
406 # mcp
407 # model-hosting-container-standards
408 # openai
@@ -410,9 +410,8 @@ httpx==0.27.2
410 # schemathesis
411 httpx-sse==0.4.3
412 # via mcp
413 -huggingface-hub==0.36.2
413 +huggingface-hub==1.10.2
414 # via
415 - # -r requirements/test/rocm.in
415 # accelerate
416 # datasets
417 # diffusers
@@ -484,7 +483,7 @@ jinja2==3.1.6
483 # genai-perf
484 # lm-eval
485 # torch
487 -jiter==0.13.0
486 +jiter==0.14.0
487 # via
488 # anthropic
489 # openai
@@ -631,7 +630,7 @@ msgpack==1.1.2
630 # via
631 # librosa
632 # ray
634 -msgspec==0.20.0
633 +msgspec==0.21.0
634 # via -r requirements/test/../common.txt
635 mteb==2.11.5
636 # via -r requirements/test/rocm.in
@@ -742,7 +741,7 @@ omegaconf==2.3.0
741 # lightning
742 open-clip-torch==2.32.0
743 # via -r requirements/test/rocm.in
745 -openai==2.30.0
744 +openai==2.31.0
745 # via
746 # -c requirements/common.txt
747 # -r requirements/test/../common.txt
@@ -1093,7 +1092,7 @@ python-dotenv==1.2.2
1092 # uvicorn
1093 python-json-logger==4.1.0
1094 # via -r requirements/test/../common.txt
1096 -python-multipart==0.0.22
1095 +python-multipart==0.0.26
1096 # via
1097 # fastapi
1098 # mcp
@@ -1180,7 +1179,6 @@ requests==2.32.5
1179 # google-api-core
1180 # google-cloud-storage
1181 # gpt-oss
1183 - # huggingface-hub
1182 # lightly
1183 # lm-eval
1184 # mistral-common
@@ -1194,7 +1192,6 @@ requests==2.32.5
1192 # starlette-testclient
1193 # tacoreader
1194 # tiktoken
1197 - # transformers
1195 # wandb
1196 resampy==0.4.3
1197 # via -r requirements/test/rocm.in
@@ -1428,7 +1425,7 @@ timm==1.0.17
1425 # segmentation-models-pytorch
1426 # terratorch
1427 # torchgeo
1431 -tokenizers==0.22.0
1428 +tokenizers==0.22.2
1429 # via
1430 # -c requirements/common.txt
1431 # -r requirements/test/../common.txt
@@ -1471,7 +1468,7 @@ tqdm==4.67.3
1468 # tacoreader
1469 # terratorch
1470 # transformers
1474 -transformers==4.57.5
1471 +transformers==5.5.3
1472 # via
1473 # -c requirements/common.txt
1474 # -r requirements/test/../common.txt
@@ -1498,7 +1495,9 @@ typer==0.24.1
1495 # fastapi-cli
1496 # fastapi-cloud-cli
1497 # fastsafetensors
1498 + # huggingface-hub
1499 # perceptron
1500 + # transformers
1501 typeshed-client==2.9.0
1502 # via jsonargparse
1503 typing-extensions==4.15.0
requirements/test/xpu.in
-1
@@ -13,7 +13,6 @@ pytest-shard
13 absl-py
14 accelerate
15 arctic-inference
16 -hf_transfer
16 lm_eval[api]
17 modelscope
18
requirements/test/xpu.txt
+15 -9
@@ -19,7 +19,9 @@ aiosignal==1.4.0
19 albumentations==1.4.6
20 # via -r requirements/test/xpu.in
21 annotated-doc==0.0.4
22 - # via fastapi
22 + # via
23 + # fastapi
24 + # typer
25 annotated-types==0.7.0
26 # via pydantic
27 anyio==4.13.0
@@ -64,6 +66,7 @@ click==8.3.1
66 # jiwer
67 # nltk
68 # schemathesis
69 + # typer
70 # uvicorn
71 colorama==0.4.6
72 # via sacrebleu
@@ -112,7 +115,6 @@ filelock==3.25.2
115 # huggingface-hub
116 # modelscope
117 # torch
115 - # transformers
118 frozenlist==1.8.0
119 # via
120 # aiohttp
@@ -133,9 +135,7 @@ h11==0.16.0
135 # uvicorn
136 harfile==0.4.0
137 # via schemathesis
136 -hf-transfer==0.1.9
137 - # via -r requirements/test/xpu.in
138 -hf-xet==1.4.2
138 +hf-xet==1.4.3
139 # via huggingface-hub
140 html2text==2025.4.15
141 # via gpt-oss
@@ -144,8 +144,9 @@ httpcore==1.0.9
144 httpx==0.28.1
145 # via
146 # datasets
147 + # huggingface-hub
148 # schemathesis
148 -huggingface-hub==0.36.2
149 +huggingface-hub==1.10.2
150 # via
151 # accelerate
152 # datasets
@@ -515,7 +516,6 @@ requests==2.33.1
516 # docker
517 # evaluate
518 # gpt-oss
518 - # huggingface-hub
519 # lm-eval
520 # mistral-common
521 # modelscope
@@ -524,11 +524,11 @@ requests==2.33.1
524 # schemathesis
525 # starlette-testclient
526 # tiktoken
527 - # transformers
527 rich==14.3.3
528 # via
529 # mteb
530 # schemathesis
531 + # typer
532 rouge-score==0.1.2
533 # via lm-eval
534 rpds-py==0.30.0
@@ -572,6 +572,8 @@ setuptools==80.10.2
572 # modelscope
573 # pytablewriter
574 # torch
575 +shellingham==1.5.4
576 + # via typer
577 six==1.17.0
578 # via
579 # -c requirements/common.txt
@@ -665,7 +667,7 @@ tqdm==4.67.3
667 # pqdm
668 # sentence-transformers
669 # transformers
668 -transformers==4.57.6
670 +transformers==5.5.3
671 # via
672 # -c requirements/common.txt
673 # sentence-transformers
@@ -676,6 +678,10 @@ typepy==1.3.4
678 # dataproperty
679 # pytablewriter
680 # tabledata
681 +typer==0.24.1
682 + # via
683 + # huggingface-hub
684 + # transformers
685 typing-extensions==4.15.0
686 # via
687 # -c requirements/common.txt
tests/conftest.py
+9
@@ -410,6 +410,15 @@ class HfRunner:
410 model_name,
411 trust_remote_code=trust_remote_code,
412 )
413 + # HF runner should use the HF config so that it's consistent with the HF model
414 + if self.config.__module__.startswith("vllm.transformers_utils.configs"):
415 + from transformers.models.auto.configuration_auto import CONFIG_MAPPING
416 +
417 + del CONFIG_MAPPING._extra_content[self.config.model_type]
418 + self.config = AutoConfig.from_pretrained(
419 + model_name,
420 + trust_remote_code=trust_remote_code,
421 + )
422 self.device = self.get_default_device()
423 self.dtype = dtype = _get_and_verify_dtype(
424 self.model_name,
tests/lora/conftest.py
+6
@@ -3,6 +3,7 @@
3
4 import tempfile
5 from collections import OrderedDict
6 +from importlib import reload
7 from unittest.mock import MagicMock
8
9 import pytest
@@ -47,6 +48,11 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
48 def maybe_enable_lora_dual_stream(monkeypatch: pytest.MonkeyPatch):
49 if current_platform.is_cuda():
50 monkeypatch.setenv("VLLM_LORA_ENABLE_DUAL_STREAM", "1")
51 + import vllm.lora.layers.base_linear
52 +
53 + if not hasattr(vllm.lora.layers.base_linear, "lora_linear_async"):
54 + # Reload the module to ensure the environment variable takes effect.
55 + reload(vllm.lora.layers.base_linear)
56 yield
57
58
tests/lora/test_minicpmv_tp.py
+11
@@ -1,7 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from importlib.metadata import version
5 +
6 import pytest
7 +from packaging.version import Version
8
9 import vllm
10 from vllm.assets.image import ImageAsset
@@ -10,6 +13,14 @@ from vllm.platforms import current_platform
13
14 from ..utils import multi_gpu_test
15
16 +pytestmark = pytest.mark.skipif(
17 + Version("5.0") <= Version(version("transformers")),
18 + reason=(
19 + "MiniCPMV custom processor uses tokenizer.im_start_id which is not "
20 + "available on TokenizersBackend in transformers v5.0+"
21 + ),
22 +)
23 +
24 MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
25
26 PROMPT_TEMPLATE = (
tests/model_executor/test_weight_utils.py
-18
@@ -1,7 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -import os
4 import tempfile
5
6 import huggingface_hub.constants
@@ -10,26 +9,10 @@ from huggingface_hub.utils import LocalEntryNotFoundError
9
10 from vllm.model_executor.model_loader.weight_utils import (
11 download_weights_from_hf,
13 - enable_hf_transfer,
12 maybe_remap_kv_scale_name,
13 )
14
15
18 -def test_hf_transfer_auto_activation():
19 - if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ:
20 - # in case it is already set, we can't test the auto activation
21 - pytest.skip("HF_HUB_ENABLE_HF_TRANSFER is set, can't test auto activation")
22 - enable_hf_transfer()
23 - try:
24 - # enable hf hub transfer if available
25 - import hf_transfer # type: ignore # noqa
26 -
27 - HF_TRANSFER_ACTIVE = True
28 - except ImportError:
29 - HF_TRANSFER_ACTIVE = False
30 - assert huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER == HF_TRANSFER_ACTIVE
31 -
32 -
16 def test_download_weights_from_hf():
17 with tempfile.TemporaryDirectory() as tmpdir:
18 # assert LocalEntryNotFoundError error is thrown
@@ -178,5 +161,4 @@ class TestMaybeRemapKvScaleName:
161
162
163 if __name__ == "__main__":
181 - test_hf_transfer_auto_activation()
164 test_download_weights_from_hf()
tests/models/language/generation/test_common.py
+5
@@ -143,6 +143,11 @@ def test_models(
143 # in parts of the operators
144 pytest.skip(f"Skipping '{model}' model test with AITER kernel.")
145
146 + if current_platform.is_cpu() and model == "TitanML/tiny-mixtral":
147 + # This untrained model is sensitive to the rounding error
148 + # Fuse ops to reduce bfloat16 rounding
149 + monkeypatch.setenv("VLLM_CPU_CI_ENV", "0")
150 +
151 with hf_runner(model) as hf_model:
152 hf_outputs = hf_model.generate_greedy_logprobs_limit(
153 example_prompts, max_tokens, num_logprobs
tests/models/language/pooling_mteb_test/test_baai.py
+4 -1
@@ -69,7 +69,10 @@ MODELS = [
69 attn_type="decoder",
70 is_prefix_caching_supported=True,
71 is_chunked_prefill_supported=True,
72 - enable_test=True,
72 + # Skip: model's custom tokenizer on HF hub is incompatible with
73 + # transformers v5 (sets attrs before super().__init__, triggering
74 + # AttributeError on 'verbose' in __getattr__).
75 + enable_test=False,
76 ),
77 ]
78
tests/models/language/pooling_mteb_test/test_gte.py
+2 -1
@@ -72,7 +72,8 @@ MODELS = [
72 attn_type="encoder_only",
73 is_prefix_caching_supported=False,
74 is_chunked_prefill_supported=False,
75 - enable_test=True,
75 + # Skip: numerical regression with transformers v5.
76 + enable_test=False,
77 ),
78 ########## ModernBertModel
79 EmbedModelInfo(
tests/models/language/pooling_mteb_test/test_jina.py
+4
@@ -75,6 +75,10 @@ def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
75 mteb_test_rerank_models(vllm_runner, model_info)
76
77
78 +@pytest.mark.skip(
79 + reason="jinaai/jina-embeddings-v3 custom XLMRobertaLoRA model on HF hub "
80 + "is incompatible with transformers v5 (missing all_tied_weights_keys)"
81 +)
82 @pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
83 @pytest.mark.parametrize("dtype", ["half"])
84 @pytest.mark.parametrize("dimensions", [16, 32])
tests/models/multimodal/generation/test_common.py
+38 -6
@@ -186,7 +186,14 @@ VLM_TEST_SETTINGS = {
186 max_num_seqs=2,
187 auto_cls=AutoModel,
188 hf_output_post_proc=model_utils.ultravox_trunc_hf_output,
189 - marks=[pytest.mark.core_model, pytest.mark.cpu_model],
189 + marks=[
190 + pytest.mark.core_model,
191 + pytest.mark.cpu_model,
192 + # TODO: Remove skip once model has been upstreamed to Transformers
193 + pytest.mark.skip(
194 + reason="Custom model code is not compatible with Transformers v5"
195 + ),
196 + ],
197 ),
198 #### Transformers fallback to test
199 ## To reduce test burden, we only test batching arbitrary image size
@@ -397,14 +404,14 @@ VLM_TEST_SETTINGS = {
404 "gemma4": VLMTestInfo(
405 models=["google/gemma-4-E2B-it"],
406 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
400 - prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
407 + prompt_formatter=lambda img_prompt: f"<bos><|turn>user\n{img_prompt}<turn|>\n<|turn>model\n", # noqa: E501
408 single_image_prompts=IMAGE_ASSETS.prompts(
409 {
403 - "stop_sign": "What's the content in the center of the image?",
404 - "cherry_blossom": "What is the season?",
410 + "stop_sign": "<|image|>What's the content in the center of the image?", # noqa: E501
411 + "cherry_blossom": "<|image|>What is the season?",
412 }
413 ),
407 - multi_image_prompt="Describe the two images in detail.",
414 + multi_image_prompt="<|image|><|image|>Describe the two images in detail.", # noqa: E501
415 max_model_len=4096,
416 max_num_seqs=2,
417 auto_cls=AutoModelForImageTextToText,
@@ -533,6 +540,12 @@ VLM_TEST_SETTINGS = {
540 max_model_len=4096,
541 use_tokenizer_eos=True,
542 patch_hf_runner=model_utils.internvl_patch_hf_runner,
543 + # TODO: Remove skip once model has been upstreamed to Transformers
544 + marks=[
545 + pytest.mark.skip(
546 + reason="Custom model code tries to access data from meta-tensor"
547 + )
548 + ],
549 ),
550 "intern_vl-video": VLMTestInfo(
551 models=[
@@ -545,6 +558,12 @@ VLM_TEST_SETTINGS = {
558 use_tokenizer_eos=True,
559 patch_hf_runner=model_utils.internvl_patch_hf_runner,
560 num_logprobs=10 if current_platform.is_rocm() else 5,
561 + # TODO: Remove skip once model has been upstreamed to Transformers
562 + marks=[
563 + pytest.mark.skip(
564 + reason="Custom model code tries to access data from meta-tensor"
565 + )
566 + ],
567 ),
568 "intern_vl-hf": VLMTestInfo(
569 models=["OpenGVLab/InternVL3-1B-hf"],
@@ -591,6 +610,8 @@ VLM_TEST_SETTINGS = {
610 hf_model_kwargs={"device_map": "auto"},
611 patch_hf_runner=model_utils.isaac_patch_hf_runner,
612 image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
613 + # TODO: Remove skip once model has been upstreamed to Transformers
614 + marks=[pytest.mark.skip(reason="Custom model imports deleted object")], # noqa: E501
615 ),
616 "kimi_vl": VLMTestInfo(
617 models=["moonshotai/Kimi-VL-A3B-Instruct"],
@@ -806,7 +827,12 @@ VLM_TEST_SETTINGS = {
827 pytest.mark.skipif(
828 Version(TRANSFORMERS_VERSION) == Version("4.57.3"),
829 reason="This model is broken in Transformers v4.57.3",
809 - )
830 + ),
831 + pytest.mark.skipif(
832 + Version(TRANSFORMERS_VERSION) >= Version("5.0.0"),
833 + reason="Model's custom code uses ROPE_INIT_FUNCTIONS"
834 + "['default'] which was removed in transformers v5",
835 + ),
836 ],
837 ),
838 "phi3v": VLMTestInfo(
@@ -960,6 +986,12 @@ VLM_TEST_SETTINGS = {
986 )
987 for inp in custom_inputs.different_patch_input_cases_internvl()
988 ],
989 + # TODO: Remove skip once model has been upstreamed to Transformers
990 + marks=[
991 + pytest.mark.skip(
992 + reason="Custom model code tries to access data from meta-tensor"
993 + )
994 + ],
995 ),
996 "llava_onevision-multiple-images": VLMTestInfo(
997 models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
tests/models/multimodal/generation/test_nemotron_parse.py
+4
@@ -103,6 +103,10 @@ def run_test(
103 )
104
105
106 +@pytest.mark.skip(
107 + reason="Model's custom MBart decoder has head count mismatch with "
108 + "transformers v5's GQA-aware cross-attention (8 vs 16 heads)"
109 +)
110 @pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.1"])
111 @pytest.mark.parametrize("dtype", ["bfloat16"])
112 @pytest.mark.parametrize("num_logprobs", [5])
tests/models/multimodal/generation/test_phi4siglip.py
+11
@@ -2,9 +2,11 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from collections.abc import Sequence
5 +from importlib.metadata import version
6
7 import pytest
8 import regex as re
9 +from packaging.version import Version
10 from transformers import AutoModelForCausalLM, AutoTokenizer
11
12 from vllm.logprobs import SampleLogprobs
@@ -19,6 +21,15 @@ from ....conftest import (
21 from ....utils import multi_gpu_test
22 from ...utils import check_logprobs_close
23
24 +pytestmark = pytest.mark.skipif(
25 + Version("5.0") <= Version(version("transformers")),
26 + reason=(
27 + "vllm upgraded transformers above v5.4 where HF model custom code uses siglip2 "
28 + "internals (filter_out_non_signature_kwargs) removed by "
29 + "huggingface/transformers#43514"
30 + ),
31 +)
32 +
33 MODEL_ID = "microsoft/Phi-4-reasoning-vision-15B"
34
35 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
tests/models/multimodal/generation/test_voxtral.py
+4
@@ -149,6 +149,10 @@ def test_online_serving(vllm_runner, audio_assets: AudioTestAssets):
149 )
150
151
152 +@pytest.mark.skip(
153 + reason="VoxtralProcessor.apply_chat_template() in transformers v5 "
154 + "doesn't resolve chat_template=None to the default template"
155 +)
156 def test_hf_reference(hf_runner, vllm_runner, audio_assets: AudioTestAssets):
157 """Compare vLLM Mistral-format output against HF Transformers reference.
158
tests/models/multimodal/generation/vlm_utils/core.py
+5
@@ -80,6 +80,11 @@ def run_test(
80 if vllm_runner_kwargs:
81 vllm_runner_kwargs_.update(vllm_runner_kwargs)
82
83 + # Avoid passing limit_mm_per_prompt twice when vllm_runner_kwargs
84 + # already contains it (e.g. gemma4 sets it via vllm_runner_kwargs).
85 + if "limit_mm_per_prompt" in vllm_runner_kwargs_:
86 + limit_mm_per_prompt = vllm_runner_kwargs_.pop("limit_mm_per_prompt")
87 +
88 with vllm_runner(
89 model,
90 max_model_len=max_model_len,
tests/models/multimodal/pooling/test_colqwen3.py
+5
@@ -22,6 +22,11 @@ from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
22
23 from ....conftest import VllmRunner
24
25 +pytestmark = pytest.mark.skip(
26 + reason="ColQwen3 model's weight tying is incompatible with "
27 + "transformers v5 (missing all_tied_weights_keys)"
28 +)
29 +
30 MODELS = [
31 "TomoroAI/tomoro-colqwen3-embed-4b",
32 "OpenSearch-AI/Ops-Colqwen3-4B",
tests/models/multimodal/pooling/test_intern_vit.py
+5
@@ -12,6 +12,11 @@ from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
12
13 from ....conftest import ImageTestAssets
14
15 +pytestmark = pytest.mark.skip(
16 + reason="InternVisionModel's custom code is incompatible with "
17 + "transformers v5 (missing all_tied_weights_keys)"
18 +)
19 +
20 # we use snapshot_download to prevent conflicts between
21 # dynamic_module and trust_remote_code for hf_runner
22 DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
tests/models/multimodal/pooling/test_jinavl_reranker.py
+5
@@ -15,6 +15,11 @@ from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
15
16 from ....conftest import HfRunner, VllmRunner
17
18 +pytestmark = pytest.mark.skip(
19 + reason="jinaai/jina-reranker-m0 custom code is incompatible with "
20 + "transformers v5 (missing all_tied_weights_keys)"
21 +)
22 +
23 MODELS = ["jinaai/jina-reranker-m0"]
24
25 MM_PROCESSOR_KWARGS = {
tests/models/multimodal/processing/test_musicflamingo.py
+7
@@ -17,11 +17,13 @@
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19
20 +from importlib.metadata import version
21 from unittest.mock import MagicMock
22
23 import numpy as np
24 import pytest
25 import torch
26 +from packaging.version import Version
27 from transformers import PretrainedConfig
28
29 from tests.models.registry import HF_EXAMPLE_MODELS
@@ -122,6 +124,11 @@ def test_musicflamingo_dummy_text_uses_plain_audio_tokens(mock_ctx):
124 assert builder.get_dummy_text({"audio": 2}) == "<sound><sound>"
125
126
127 +@pytest.mark.skipif(
128 + Version(version("transformers")) >= Version("5.5"),
129 + reason="transformers v5.5 added native MusicFlamingoForConditionalGeneration "
130 + "with a different get_audio_features signature (requires input_ids)",
131 +)
132 def test_musicflamingo_audio_feature_pipeline_matches_hf_small_config():
133 from transformers.models.musicflamingo import (
134 modeling_musicflamingo as hf_musicflamingo_modeling,
tests/models/registry.py
+130 -9
@@ -335,7 +335,15 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
335 "internlm/internlm2-chat-7b", trust_remote_code=True
336 ),
337 "InternLM2VEForCausalLM": _HfExamplesInfo(
338 - "OpenGVLab/Mono-InternVL-2B", trust_remote_code=True
338 + "OpenGVLab/Mono-InternVL-2B",
339 + trust_remote_code=True,
340 + max_transformers_version="4.57",
341 + transformers_version_reason={
342 + "vllm": (
343 + "Custom config cannot be loaded with Transformers "
344 + "v5 because `vision_config` is not always set"
345 + )
346 + },
347 ),
348 "InternLM3ForCausalLM": _HfExamplesInfo(
349 "internlm/internlm3-8b-instruct", trust_remote_code=True
@@ -475,6 +483,13 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
483 "Plamo2ForCausalLM": _HfExamplesInfo(
484 "pfnet/plamo-2-1b",
485 trust_remote_code=True,
486 + max_transformers_version="4.57",
487 + transformers_version_reason={
488 + "hf": (
489 + "Custom model code uses `_tied_weight_keys: list[str]` but "
490 + "Transformers v5 now expects `_tied_weight_keys: dict[str, str]`"
491 + )
492 + },
493 ),
494 "Plamo3ForCausalLM": _HfExamplesInfo(
495 "pfnet/plamo-3-nict-2b-base",
@@ -515,6 +530,13 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
530 trust_remote_code=True,
531 max_model_len=4096,
532 is_available_online=True,
533 + max_transformers_version="5.3",
534 + transformers_version_reason={
535 + "vllm": (
536 + "vllm upgraded transformers above v5.4 where "
537 + "validate_rope() no longer accepts ignore_keys param"
538 + )
539 + },
540 ),
541 "SeedOssForCausalLM": _HfExamplesInfo(
542 "ByteDance-Seed/Seed-OSS-36B-Instruct",
@@ -553,6 +575,11 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
575 "xverse/XVERSE-7B-Chat",
576 tokenizer="meta-llama/Llama-2-7b",
577 trust_remote_code=True,
578 + max_transformers_version="4.57",
579 + transformers_version_reason={
580 + "vllm": "XVERSE tokenizer is incompatible with transformers v5 "
581 + "(add_prefix_space / prepend_scheme mismatch).",
582 + },
583 ),
584 "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
585 "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
@@ -763,10 +790,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
790 # [Decoder-only]
791 "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
792 "AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
766 - "nvidia/audio-flamingo-3-hf", min_transformers_version="5.0.0"
793 + "nvidia/audio-flamingo-3-hf",
794 + min_transformers_version="5.3.0",
795 + transformers_version_reason={
796 + "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
797 + },
798 ),
799 "MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
769 - "nvidia/music-flamingo-2601-hf", min_transformers_version="5.3.0"
800 + "nvidia/music-flamingo-2601-hf",
801 + min_transformers_version="5.3.0",
802 + transformers_version_reason={
803 + "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
804 + },
805 ),
806 "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),
807 "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
@@ -821,12 +856,30 @@ _MULTIMODAL_EXAMPLE_MODELS = {
856 ),
857 "FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
858 "allendou/FireRedASR2-LLM-vllm",
859 + trust_remote_code=True,
860 + max_transformers_version="5.1",
861 + transformers_version_reason={
862 + "vllm": "Incompatible with transformers v5.2+ "
863 + "(dict object has no attribute '__name__').",
864 + },
865 ),
866 "FireRedLIDForConditionalGeneration": _HfExamplesInfo(
867 "PatchyTisa/FireRedLID-vllm",
868 + trust_remote_code=True,
869 + max_transformers_version="5.1",
870 + transformers_version_reason={
871 + "vllm": "Incompatible with transformers v5.2+ "
872 + "(dict object has no attribute '__name__').",
873 + },
874 ),
875 "FunASRForConditionalGeneration": _HfExamplesInfo(
876 "allendou/Fun-ASR-Nano-2512-vllm",
877 + trust_remote_code=True,
878 + max_transformers_version="5.1",
879 + transformers_version_reason={
880 + "vllm": "Incompatible with transformers v5.2+ "
881 + "(dict object has no attribute '__name__').",
882 + },
883 ),
884 "FunAudioChatForConditionalGeneration": _HfExamplesInfo(
885 "funaudiochat", is_available_online=False
@@ -868,6 +921,13 @@ _MULTIMODAL_EXAMPLE_MODELS = {
921 "HCXVisionForCausalLM": _HfExamplesInfo(
922 "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
923 trust_remote_code=True,
924 + max_transformers_version="4.57",
925 + transformers_version_reason={
926 + "vllm": (
927 + "Custom config cannot be loaded with Transformers "
928 + "v5 because `text_config` is not always set"
929 + )
930 + },
931 ),
932 "HCXVisionV2ForCausalLM": _HfExamplesInfo(
933 "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
@@ -887,7 +947,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
947 extras={"0.2-2B-Preview": "PerceptronAI/Isaac-0.2-2B-Preview"},
948 ),
949 "InternS1ForConditionalGeneration": _HfExamplesInfo(
890 - "internlm/Intern-S1", trust_remote_code=True
950 + "internlm/Intern-S1",
951 + trust_remote_code=True,
952 + max_transformers_version="4.57",
953 + transformers_version_reason={
954 + "vllm": "Custom tokenizer code is not compatible with Transformers v5."
955 + },
956 ),
957 "InternS1ProForConditionalGeneration": _HfExamplesInfo(
958 "internlm/Intern-S1-Pro",
@@ -976,7 +1041,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1041 "MiDashengLMModel": _HfExamplesInfo(
1042 "mispeech/midashenglm-7b", trust_remote_code=True
1043 ),
979 - "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6", trust_remote_code=True),
1044 + "MiniCPMO": _HfExamplesInfo(
1045 + "openbmb/MiniCPM-o-2_6",
1046 + trust_remote_code=True,
1047 + max_transformers_version="4.57",
1048 + transformers_version_reason={
1049 + "hf": "Custom processor code is not compatible with Transformers v5."
1050 + },
1051 + ),
1052 "MiniCPMV": _HfExamplesInfo(
1053 "openbmb/MiniCPM-Llama3-V-2_5",
1054 extras={
@@ -984,6 +1056,13 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1056 "4.0": "openbmb/MiniCPM-V-4",
1057 "4.5": "openbmb/MiniCPM-V-4_5",
1058 },
1059 + max_transformers_version="4.57",
1060 + transformers_version_reason={
1061 + "vllm": (
1062 + "MiniCPMVBatchFeature is incompatible with its base class in "
1063 + "Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78"
1064 + )
1065 + },
1066 trust_remote_code=True,
1067 ),
1068 "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
@@ -1083,13 +1162,25 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1162 trust_remote_code=True,
1163 ),
1164 "OpenCUAForConditionalGeneration": _HfExamplesInfo(
1086 - "xlangai/OpenCUA-7B", trust_remote_code=True
1165 + "xlangai/OpenCUA-7B",
1166 + trust_remote_code=True,
1167 + max_transformers_version="4.57",
1168 + transformers_version_reason={
1169 + "vllm": "Tokenizer cannot be initialised in Transformers v5."
1170 + },
1171 ),
1172 "OpenPanguVLForConditionalGeneration": _HfExamplesInfo(
1173 "FreedomIntelligence/openPangu-VL-7B",
1174 trust_remote_code=True,
1175 max_model_len=4096,
1176 enforce_eager=True,
1177 + max_transformers_version="4.57",
1178 + transformers_version_reason={
1179 + "vllm": (
1180 + "OpenPanguVLVideoProcessorInitKwargs does not specify total=False, "
1181 + "making all kwargs required. See https://huggingface.co/FreedomIntelligence/openPangu-VL-7B/discussions/2"
1182 + )
1183 + },
1184 ),
1185 "Ovis": _HfExamplesInfo(
1186 "AIDC-AI/Ovis2-1B",
@@ -1101,12 +1192,24 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1192 "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B",
1193 },
1194 ),
1104 - "Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B", trust_remote_code=True),
1195 + "Ovis2_5": _HfExamplesInfo(
1196 + "AIDC-AI/Ovis2.5-2B",
1197 + trust_remote_code=True,
1198 + max_transformers_version="4.57",
1199 + transformers_version_reason={
1200 + "vllm": "Custom processor code is not compatible with Transformers v5."
1201 + },
1202 + ),
1203 "Ovis2_6ForCausalLM": _HfExamplesInfo(
1204 "AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
1205 ),
1206 "Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
1109 - "AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True
1207 + "AIDC-AI/Ovis2.6-30B-A3B",
1208 + trust_remote_code=True,
1209 + max_transformers_version="4.57",
1210 + transformers_version_reason={
1211 + "vllm": "Custom processor code is not compatible with Transformers v5."
1212 + },
1213 ),
1214 "PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
1215 "PaddlePaddle/PaddleOCR-VL",
@@ -1126,7 +1229,17 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1229 extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"},
1230 ),
1231 "Phi4ForCausalLMV": _HfExamplesInfo(
1129 - "microsoft/Phi-4-reasoning-vision-15B", trust_remote_code=True
1232 + "microsoft/Phi-4-reasoning-vision-15B",
1233 + trust_remote_code=True,
1234 + max_transformers_version="5.3",
1235 + transformers_version_reason={
1236 + "vllm": (
1237 + "vllm upgraded transformers above v5.4 where HF model "
1238 + "custom code uses siglip2 internals "
1239 + "(filter_out_non_signature_kwargs) removed "
1240 + "by huggingface/transformers#43514"
1241 + )
1242 + },
1243 ),
1244 "Phi4MMForCausalLM": _HfExamplesInfo(
1245 "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
@@ -1223,6 +1336,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1336 "architectures": ["Tarsier2ForConditionalGeneration"],
1337 "model_type": "tarsier2",
1338 },
1339 + max_transformers_version="5.3",
1340 + transformers_version_reason={
1341 + "vllm": (
1342 + "Qwen2VLConfig was split into Qwen2VLConfig + "
1343 + "Qwen2VLTextConfig in transformers v5, breaking "
1344 + "attribute access (num_attention_heads, hidden_size, etc.)"
1345 + )
1346 + },
1347 ),
1348 "VoxtralForConditionalGeneration": _HfExamplesInfo(
1349 "mistralai/Voxtral-Mini-3B-2507",
tests/models/utils.py
+10 -1
@@ -476,7 +476,16 @@ def dummy_hf_overrides(
476 else:
477 # Use minimal layers for testing
478 num_layers = 1
479 - num_hidden_layers = 3 if model_arch == "Gemma3nForConditionalGeneration" else 1
479 + num_hidden_layers = (
480 + 3
481 + if model_arch
482 + in (
483 + "Gemma3nForConditionalGeneration",
484 + "Gemma4ForCausalLM",
485 + "Gemma4ForConditionalGeneration",
486 + )
487 + else 1
488 + )
489
490 update_dict = {
491 "num_layers": num_layers,
tests/reasoning/test_step3p5_reasoning_parser.py
+2 -2
@@ -2,10 +2,10 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import pytest
5 -from transformers import AutoTokenizer
5
6 from tests.reasoning.utils import run_reasoning_extraction
7 from vllm.reasoning import ReasoningParser, ReasoningParserManager
8 +from vllm.tokenizers import get_tokenizer
9
10 parser_name = "step3p5"
11 start_token = "<think>"
@@ -16,7 +16,7 @@ REASONING_MODEL_NAME = "stepfun-ai/Step-3.5-Flash"
16
17 @pytest.fixture(scope="module")
18 def step3p5_tokenizer():
19 - return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
19 + return get_tokenizer(tokenizer_name=REASONING_MODEL_NAME)
20
21
22 SIMPLE_REASONING = {
tests/v1/e2e/spec_decode/test_spec_decode.py
+5 -1
@@ -557,12 +557,16 @@ def test_eagle_correctness_light(
557 "auto",
558 0.8,
559 ),
560 - (
560 + pytest.param(
561 ("eagle3", "Qwen/Qwen3-8B", "AngelSlim/Qwen3-8B_eagle3", 1),
562 False,
563 False,
564 "transformers",
565 0.8,
566 + # TODO(hmellor): figure out why memory usage is so high
567 + marks=pytest.mark.skip(
568 + reason="Feature is experimental and uses too much memory in CI",
569 + ),
570 ),
571 pytest.param(
572 (
vllm/model_executor/model_loader/gguf_loader.py
+12
@@ -265,12 +265,24 @@ class GGUFModelLoader(BaseModelLoader):
265 GGUF tensor name with suffix (e.g., 'mm.soft_emb_norm.weight')
266 or None if no mapping found
267 """
268 + # In transformers v5, multimodal models (e.g. Gemma3) wrap
269 + # all sub-models under an outer 'model.' attribute, producing
270 + # state_dict keys like 'model.language_model.layers.0...' and
271 + # 'model.vision_tower.vision_model...'. Strip this outer
272 + # prefix so the keys match what gguf-py expects.
273 + if is_multimodal and hf_name.startswith("model."):
274 + hf_name = hf_name[6:] # Remove outer 'model.'
275 +
276 # Strip 'language_model.' prefix for multimodal models - gguf-py
277 # tensor mappings expect parameter names without this prefix.
278 # Note: 'model.' prefix should be KEPT for text-only models as
279 # gguf-py expects it.
280 if hf_name.startswith("language_model."):
281 hf_name = hf_name[15:] # Remove 'language_model.'
282 + # Re-add 'model.' prefix because gguf-py text tensor maps
283 + # expect 'model.layers...' format.
284 + if is_multimodal:
285 + hf_name = "model." + hf_name
286
287 # Parse parameter name and suffix
288 if hf_name.endswith((".weight", ".bias")):
vllm/model_executor/models/gemma4_mm.py
+36 -15
@@ -125,8 +125,12 @@ class Gemma4AudioInputs(TensorSchema):
125 """
126
127 type: Literal["audio"] = "audio"
128 - input_features_padded: Annotated[torch.Tensor, TensorShape("bn", "s", "f")]
129 - input_features_mask: Annotated[torch.Tensor, TensorShape("bn", "s")]
128 + input_features_padded: Annotated[
129 + torch.Tensor, TensorShape("bn", "s", "f", dynamic_dims={"s"})
130 + ]
131 + input_features_mask: Annotated[
132 + torch.Tensor, TensorShape("bn", "s", dynamic_dims={"s"})
133 + ]
134
135
136 Gemma4ImageInputs = Gemma4ImagePixelInputs
@@ -510,6 +514,8 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
514 video_timestamps_per_video: list[list[float]] = []
515 video_frame_counts: list[int] = []
516
517 + video_replacements: list[str] = []
518 +
519 for item in videos:
520 video_array, metadata = item
521
@@ -562,10 +568,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
568 video_timestamps_per_video.append(timestamps)
569 video_frame_counts.append(len(frames))
570
565 - # Build expanded replacement text and replace the
566 - # <|video|> placeholder in the prompt.
567 - # Use split(token, 1) to avoid collision — the
568 - # replacement text itself contains <|video|> tokens.
571 + # Build expanded replacement text for this video.
572 ts_strs = [f"{int(s // 60):02d}:{int(s % 60):02d}" for s in timestamps]
573 replacement = " ".join(
574 f"{t} {processor.boi_token}"
@@ -573,9 +576,23 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
576 f"{processor.eoi_token}"
577 for t, n in zip(ts_strs, num_soft_per_frame)
578 )
576 - parts = prompt.split(processor.video_token, 1)
577 - if len(parts) == 2:
578 - prompt = parts[0] + replacement + parts[1]
579 + video_replacements.append(replacement)
580 +
581 + # Replace all <|video|> placeholders at once. We split on
582 + # video_token to get N+1 parts, then interleave with the
583 + # N replacement strings. This avoids the iterative
584 + # split-replace bug where replacement text (which itself
585 + # contains <|video|> tokens) collides with later splits.
586 + vt = processor.video_token
587 + parts = prompt.split(vt, len(video_replacements))
588 +
589 + # NOTE: len(parts) <= len(video_replacements) + 1
590 + parts_with_repl: list[str] = []
591 + for part, repl in zip(parts, video_replacements):
592 + parts_with_repl.extend([part, repl])
593 + parts_with_repl.extend(parts[len(video_replacements) :])
594 +
595 + prompt = "".join(parts_with_repl)
596
597 video_outputs = {
598 "pixel_values_videos": torch.cat(all_video_pixel_values, dim=0),
@@ -638,19 +655,23 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
655 )
656
657 if "input_features" in processed_outputs:
641 - # Keep padded features for batched audio tower execution.
642 - processed_outputs["input_features_padded"] = processed_outputs[
643 - "input_features"
644 - ]
645 - # Unpad per-item so each item's cache entry is self-contained.
658 + # Unpad per-item so each item's cache entry is
659 + # self-contained. The batched() field config in
660 + # _get_mm_fields_config will re-pad all fields to the
661 + # batch's max length at batch time, ensuring consistent
662 + # padding regardless of cache history.
663 + masks = processed_outputs["input_features_mask"]
664 unpadded_features = [
665 f[mask]
666 for f, mask in zip(
667 processed_outputs["input_features"],
650 - processed_outputs["input_features_mask"],
668 + masks,
669 )
670 ]
671 + unpadded_masks = [mask[mask] for mask in masks]
672 processed_outputs["input_features"] = unpadded_features
673 + processed_outputs["input_features_padded"] = unpadded_features
674 + processed_outputs["input_features_mask"] = unpadded_masks
675
676 # Merge video outputs into the final result
677 combined_outputs = dict(processed_outputs, **video_outputs)
vllm/model_executor/models/musicflamingo.py
+1 -1
@@ -32,9 +32,9 @@ from transformers.models.musicflamingo import (
32
33 from vllm.config import VllmConfig
34 from vllm.config.multimodal import BaseDummyOptions
35 +from vllm.inputs import MultiModalDataDict
36 from vllm.multimodal import MULTIMODAL_REGISTRY
37 from vllm.multimodal.inputs import (
37 - MultiModalDataDict,
38 MultiModalFieldConfig,
39 MultiModalKwargsItems,
40 )
vllm/model_executor/models/transformers/base.py
+5
@@ -275,6 +275,11 @@ class Base(
275 )
276 class SupportTorchCompileWrapper(cls): ...
277
278 + # Preserve __module__ so transformers v5's source-file checks
279 + # (e.g. _can_set_experts_implementation) read the original
280 + # model's module instead of this file.
281 + SupportTorchCompileWrapper.__module__ = cls.__module__
282 +
283 # Patch the class in its module
284 module = sys.modules[cls.__module__]
285 setattr(module, cls.__name__, SupportTorchCompileWrapper)
vllm/tokenizers/registry.py
+34 -1
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import contextlib
4 from dataclasses import dataclass, field
5 from functools import lru_cache
6 from pathlib import Path
@@ -10,6 +11,7 @@ from typing_extensions import TypeVar, assert_never
11
12 import vllm.envs as envs
13 from vllm.logger import init_logger
14 +from vllm.transformers_utils.config import get_config
15 from vllm.transformers_utils.gguf_utils import (
16 check_gguf_file,
17 get_gguf_file_path_from_hf,
@@ -31,6 +33,13 @@ if TYPE_CHECKING:
33 logger = init_logger(__name__)
34
35
36 +# Model types whose hub tokenizer_class is incorrect and should be overridden with
37 +# TokenizersBackend (the generic fast tokenizer). Adding a model type here is always a
38 +# temporary workaround and better long term solutions are:
39 +# - Add model type to MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS in transformers (better)
40 +# - Fix tokenizer_class on the hub for the affected models (best)
41 +_MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {"step3_vl"}
42 +
43 _VLLM_TOKENIZERS = {
44 "deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"),
45 "grok2": ("grok2", "Grok2Tokenizer"),
@@ -202,7 +211,31 @@ def get_tokenizer(
211 **kwargs,
212 )
213
205 - if tokenizer_cls == TokenizerLike:
214 + # Ensure that, if the config were to come from vllm.transformers_utils.config, it is
215 + # registered with AutoConfig before the tokenizer is loaded. This is necessary since
216 + # tokenizer_cls_.from_pretrained will call AutoConfig.from_pretrained internally.
217 + # This may fail for paths that don't have a model config (e.g. LoRA adapters),
218 + # which is fine — those don't need custom config registration.
219 + config = None
220 + with contextlib.suppress(ValueError, OSError):
221 + config = get_config(
222 + tokenizer_name,
223 + trust_remote_code=trust_remote_code,
224 + revision=revision,
225 + )
226 +
227 + # Some models have an incorrect tokenizer_class on the hub.
228 + # For these model types, bypass AutoTokenizer and use TokenizersBackend directly.
229 + model_type = getattr(config, "model_type", None) if config else None
230 + if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS:
231 + from transformers.tokenization_utils_tokenizers import TokenizersBackend
232 +
233 + logger.debug(
234 + "Overriding tokenizer_class to TokenizersBackend for model_type=%r",
235 + model_type,
236 + )
237 + tokenizer_cls_ = TokenizersBackend
238 + elif tokenizer_cls == TokenizerLike:
239 tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode)
240 else:
241 tokenizer_cls_ = tokenizer_cls