A high-throughput and memory-efficient inference and serving engine for LLMs
Deprecated 1st generation Qwen and QwenVL models (#45131)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jun 10, 2026 at 15:55 UTC
3cc9fecd5836d03b055f04b40d14c0185de711a2
27 files changed
+6
-1349
.buildkite/test-amd.yaml
-2
@@ -2946,7 +2946,6 @@ steps:
2946
- vllm/model_executor/models/qwen3_5_mtp.py
2947
- vllm/transformers_utils/configs/qwen3_5.py
2948
- vllm/transformers_utils/configs/qwen3_5_moe.py
2949
- - vllm/model_executor/models/qwen.py
2949
- vllm/model_executor/models/qwen2.py
2950
- vllm/model_executor/models/qwen3.py
2951
- vllm/model_executor/models/qwen3_next.py
@@ -3184,7 +3183,6 @@ steps:
3183
- vllm/model_executor/models/qwen3_5_mtp.py
3184
- vllm/transformers_utils/configs/qwen3_5.py
3185
- vllm/transformers_utils/configs/qwen3_5_moe.py
3187
- - vllm/model_executor/models/qwen.py
3186
- vllm/model_executor/models/qwen2.py
3187
- vllm/model_executor/models/qwen3.py
3188
- vllm/model_executor/models/qwen3_next.py
docs/configuration/optimization.md
+1
-1
@@ -296,7 +296,7 @@ llm = LLM(model="Qwen/Qwen3-8B")
296
The `fastokens` Python package (>= 0.2.0) must be installed; if it isn't,
297
vLLM raises a clear `ImportError` at tokenizer load. The override applies to
298
any `--tokenizer-mode` that ends up loading an HF fast tokenizer (`hf`,
299
-`deepseek_v32`, `deepseek_v4`, `qwen_vl`, …). Models that don't use the HF
299
+`deepseek_v32`, `deepseek_v4`, …). Models that don't use the HF
300
fast tokenizer (`mistral`, `grok2`, `kimi_audio`) ignore the flag.
301
302
Tokenizer-bound workloads — long shared prefixes, bursty short prompts,
docs/contributing/model/multimodal.md
-1
@@ -884,4 +884,3 @@ Examples:
884
885
- DeepSeek-VL2: [vllm/model_executor/models/deepseek_vl2.py](../../../vllm/model_executor/models/deepseek_vl2.py)
886
- InternVL: [vllm/model_executor/models/internvl.py](../../../vllm/model_executor/models/internvl.py)
887
-- Qwen-VL: [vllm/model_executor/models/qwen_vl.py](../../../vllm/model_executor/models/qwen_vl.py)
docs/models/supported_models.md
-2
@@ -469,7 +469,6 @@ th {
469
| `PersimmonForCausalLM` | Persimmon | `adept/persimmon-8b-base`, `adept/persimmon-8b-chat`, etc. | | ✅︎ |
470
| `Plamo2ForCausalLM` | PLaMo2 | `pfnet/plamo-2-1b`, `pfnet/plamo-2-8b`, etc. | ✅ | ✅︎ |
471
| `Plamo3ForCausalLM` | PLaMo3 | `pfnet/plamo-3-nict-2b-base`, `pfnet/plamo-3-nict-8b-base`, etc. | ✅ | ✅︎ |
472
-| `QWenLMHeadModel` | Qwen | `Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc. | ✅︎ | ✅︎ |
472
| `Qwen2ForCausalLM` | QwQ, Qwen2 | `Qwen/QwQ-32B-Preview`, `Qwen/Qwen2-7B-Instruct`, `Qwen/Qwen2-7B`, etc. | ✅︎ | ✅︎ |
473
| `Qwen2MoeForCausalLM` | Qwen2MoE | `Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc. | ✅︎ | ✅︎ |
474
| `Qwen3ForCausalLM` | Qwen3 | `Qwen/Qwen3-8B`, etc. | ✅︎ | ✅︎ |
@@ -620,7 +619,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
619
| `Phi4ForCausalLMV` | Phi-4-reasoning-vision | T + I<sup>+</sup> | `microsoft/Phi-4-reasoning-vision-15B`, etc. | | ✅︎ |
620
| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | ✅︎ | ✅︎ |
621
| `QianfanOCRForConditionalGeneration` | QianfanOCR | T + I<sup>E+</sup> | `baidu/Qianfan-OCR`, etc. | ✅︎ | ✅︎ |
623
-| `QwenVLForConditionalGeneration`<sup>^</sup> | Qwen-VL | T + I<sup>E+</sup> | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ |
622
| `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ |
623
| `Qwen2VLForConditionalGeneration` <sup>Q</sup> | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ |
624
| `Qwen2_5_VLForConditionalGeneration` <sup>Q</sup> | Qwen2.5-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ |
examples/generate/multimodal/vision_language_multi_image_offline.py
-44
@@ -1042,49 +1042,6 @@ def load_phi4siglip(question: str, image_urls: list[str]) -> ModelRequestData:
1042
)
1043
1044
1045
-def load_qwen_vl_chat(question: str, image_urls: list[str]) -> ModelRequestData:
1046
- model_name = "Qwen/Qwen-VL-Chat"
1047
- engine_args = EngineArgs(
1048
- model=model_name,
1049
- trust_remote_code=True,
1050
- max_model_len=1024,
1051
- max_num_seqs=2,
1052
- hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
1053
- limit_mm_per_prompt={"image": len(image_urls)},
1054
- )
1055
- placeholders = "".join(
1056
- f"Picture {i}: <img></img>\n" for i, _ in enumerate(image_urls, start=1)
1057
- )
1058
-
1059
- # This model does not have a chat_template attribute on its tokenizer,
1060
- # so we need to explicitly pass it. We use ChatML since it's used in the
1061
- # generation utils of the model:
1062
- # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265
1063
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
1064
-
1065
- # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating
1066
- chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" # noqa: E501
1067
-
1068
- messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]
1069
- prompt = tokenizer.apply_chat_template(
1070
- messages,
1071
- tokenize=False,
1072
- add_generation_prompt=True,
1073
- chat_template=chat_template,
1074
- )
1075
-
1076
- stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
1077
- stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
1078
-
1079
- return ModelRequestData(
1080
- engine_args=engine_args,
1081
- prompt=prompt,
1082
- stop_token_ids=stop_token_ids,
1083
- image_data=[fetch_image(url) for url in image_urls],
1084
- chat_template=chat_template,
1085
- )
1086
-
1087
-
1045
def load_qwen2_vl(question: str, image_urls: list[str]) -> ModelRequestData:
1046
try:
1047
from qwen_vl_utils import smart_resize
@@ -1544,7 +1501,6 @@ model_example_map = {
1501
"phi4_mm": load_phi4mm,
1502
"phi4_siglip": load_phi4siglip,
1503
"pixtral_hf": load_pixtral_hf,
1547
- "qwen_vl_chat": load_qwen_vl_chat,
1504
"qwen2_vl": load_qwen2_vl,
1505
"qwen2_5_vl": load_qwen2_5_vl,
1506
"rvl": load_r_vl,
examples/generate/multimodal/vision_language_offline.py
-22
@@ -1938,27 +1938,6 @@ def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
1938
)
1939
1940
1941
-# Qwen-VL
1942
-def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:
1943
- assert modality == "image"
1944
-
1945
- engine_args = EngineArgs(
1946
- model="Qwen/Qwen-VL",
1947
- trust_remote_code=True,
1948
- max_model_len=1024,
1949
- max_num_seqs=2,
1950
- hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
1951
- limit_mm_per_prompt={modality: 1},
1952
- )
1953
-
1954
- prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]
1955
-
1956
- return ModelRequestData(
1957
- engine_args=engine_args,
1958
- prompts=prompts,
1959
- )
1960
-
1961
-
1941
# Qwen2-VL
1942
def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:
1943
model_name = "Qwen/Qwen2-VL-7B-Instruct"
@@ -2522,7 +2501,6 @@ model_example_map = {
2501
"phi4_mm": run_phi4mm,
2502
"phi4_siglip": run_phi4siglip,
2503
"pixtral_hf": run_pixtral_hf,
2525
- "qwen_vl": run_qwen_vl,
2504
"qwen2_vl": run_qwen2_vl,
2505
"qwen2_5_vl": run_qwen2_5_vl,
2506
"qwen2_5_omni": run_qwen2_5_omni,
tests/distributed/test_pipeline_parallel.py
-1
@@ -192,7 +192,6 @@ MULTIMODAL_MODELS = {
192
"AIDC-AI/Ovis2.5-2B": PPTestSettings.fast(),
193
"microsoft/Phi-3.5-vision-instruct": PPTestSettings.fast(),
194
"mistralai/Pixtral-12B-2409": PPTestSettings.fast(load_format="dummy"),
195
- "Qwen/Qwen-VL-Chat": PPTestSettings.fast(),
195
"Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
196
"Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
197
"fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(),
tests/models/multimodal/conftest.py
-10
@@ -5,21 +5,11 @@
5
import os
6
import warnings
7
8
-import pytest
8
import torch
9
11
-from tests.utils import prewarm_hf_cache
10
from vllm.platforms import current_platform
11
12
15
-@pytest.fixture(scope="session", autouse=True)
16
-def _prewarm_hf_cache():
17
- # tokenization_qwen.py downloads SimSun.ttf from
18
- # qianwen-res.oss-cn-beijing.aliyuncs.com; both Qwen/Qwen-VL and
19
- # Qwen/Qwen-VL-Chat look it up from the Chat repo.
20
- prewarm_hf_cache([("Qwen/Qwen-VL-Chat", "SimSun.ttf")])
21
-
22
-
13
def pytest_configure(config):
14
"""Early ROCm configuration that must happen before test collection."""
15
if not current_platform.is_rocm():
tests/models/multimodal/generation/test_common.py
-10
@@ -976,16 +976,6 @@ VLM_TEST_SETTINGS = {
976
auto_cls=AutoModelForImageTextToText,
977
hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"),
978
),
979
- "qwen_vl": VLMTestInfo(
980
- models=["Qwen/Qwen-VL"],
981
- test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
982
- prompt_formatter=identity,
983
- img_idx_to_prompt=lambda idx: f"Picture {idx}: <img></img>\n",
984
- max_model_len=1024,
985
- max_num_seqs=2,
986
- vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
987
- prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
988
- ),
979
"qwen2_vl": VLMTestInfo(
980
models=["Qwen/Qwen2-VL-2B-Instruct"],
981
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
tests/models/registry.py
-18
@@ -505,14 +505,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
505
"pfnet/plamo-3-nict-2b-base",
506
trust_remote_code=True,
507
),
508
- "QWenLMHeadModel": _HfExamplesInfo(
509
- "Qwen/Qwen-7B-Chat",
510
- max_transformers_version="4.53",
511
- transformers_version_reason={
512
- "hf": "HF model uses remote code that is not compatible with latest Transformers" # noqa: E501
513
- },
514
- trust_remote_code=True,
515
- ),
508
"Qwen2ForCausalLM": _HfExamplesInfo(
509
"Qwen/Qwen2-0.5B-Instruct",
510
extras={
@@ -1297,16 +1289,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1289
"baidu/Qianfan-OCR",
1290
min_transformers_version="5.6.0",
1291
),
1300
- "QwenVLForConditionalGeneration": _HfExamplesInfo(
1301
- "Qwen/Qwen-VL",
1302
- extras={"chat": "Qwen/Qwen-VL-Chat"},
1303
- trust_remote_code=True,
1304
- max_transformers_version="4.53.3",
1305
- transformers_version_reason={
1306
- "hf": "HF model uses deprecated imports which have been removed."
1307
- }, # noqa: E501
1308
- hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
1309
- ),
1292
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
1293
"Qwen/Qwen2-Audio-7B-Instruct"
1294
),
tests/renderers/conftest.py
deleted
-14
@@ -1,14 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import pytest
5
-
6
-from tests.utils import prewarm_hf_cache
7
-
8
-
9
-@pytest.fixture(scope="session", autouse=True)
10
-def _prewarm_hf_cache():
11
- # tokenization_qwen.py downloads SimSun.ttf from
12
- # qianwen-res.oss-cn-beijing.aliyuncs.com; both Qwen/Qwen-VL and
13
- # Qwen/Qwen-VL-Chat look it up from the Chat repo.
14
- prewarm_hf_cache([("Qwen/Qwen-VL-Chat", "SimSun.ttf")])
tests/renderers/test_hf.py
-2
@@ -428,8 +428,6 @@ def test_resolve_content_format_hf_defined(model, expected_format):
428
("deepseek-ai/deepseek-vl2-tiny", "string"),
429
("adept/fuyu-8b", "string"),
430
("google/paligemma-3b-mix-224", "string"),
431
- ("Qwen/Qwen-VL", "string"),
432
- ("Qwen/Qwen-VL-Chat", "string"),
431
],
432
)
433
def test_resolve_content_format_fallbacks(model, expected_format):
tests/tokenizers_/conftest.py
deleted
-14
@@ -1,14 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import pytest
5
-
6
-from tests.utils import prewarm_hf_cache
7
-
8
-
9
-@pytest.fixture(scope="session", autouse=True)
10
-def _prewarm_hf_cache():
11
- # tokenization_qwen.py downloads SimSun.ttf from
12
- # qianwen-res.oss-cn-beijing.aliyuncs.com; both Qwen/Qwen-VL and
13
- # Qwen/Qwen-VL-Chat look it up from the Chat repo.
14
- prewarm_hf_cache([("Qwen/Qwen-VL-Chat", "SimSun.ttf")])
tests/tokenizers_/test_basic.py
-8
@@ -47,14 +47,6 @@ def test_tokenizer_like_protocol():
47
assert "DSV32" in tokenizer.__class__.__name__
48
_assert_tokenizer_like(tokenizer)
49
50
- tokenizer = get_tokenizer(
51
- "Qwen/Qwen-VL",
52
- tokenizer_mode="qwen_vl",
53
- trust_remote_code=True,
54
- )
55
- assert isinstance(tokenizer, HfTokenizer)
56
- assert "WithoutImagePad" in tokenizer.__class__.__name__
57
-
50
51
@pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"])
52
def test_tokenizer_revision(tokenizer_name: str):
vllm/benchmarks/serve.py
-1
@@ -1440,7 +1440,6 @@ def add_cli_args(parser: argparse.ArgumentParser):
1440
- "slow" will always use the slow tokenizer.\n
1441
- "mistral" will always use the tokenizer from `mistral_common`.\n
1442
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
1443
- - "qwen_vl" will always use the tokenizer from `qwen_vl`.\n
1443
- Other custom values can be supported via plugins.""",
1444
)
1445
parser.add_argument("--use-beam-search", action="store_true")
vllm/config/model.py
+1
-5
@@ -135,14 +135,12 @@ class ModelConfig:
135
- "mistral" will always use the tokenizer from `mistral_common`.
136
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
137
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
138
- - "qwen_vl" will always use the tokenizer from `qwen_vl`.
138
- Other custom values can be supported via plugins.
139
140
To swap the Rust BPE backend that powers HF fast tokenizers for the
141
[fastokens](https://github.com/crusoecloud/fastokens) implementation, set
142
`VLLM_USE_FASTOKENS=1` instead — that override applies to any mode that
144
- loads an HF fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`,
145
- `qwen_vl`, …)."""
143
+ loads an HF fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, …)."""
144
trust_remote_code: bool = False
145
"""Trust remote code (e.g., from HuggingFace) when downloading the model
146
and tokenizer."""
@@ -618,8 +616,6 @@ class ModelConfig:
616
self.tokenizer_mode = "grok2"
617
elif arch == "MoonshotKimiaForCausalLM":
618
self.tokenizer_mode = "kimi_audio"
621
- elif arch == "QwenVLForConditionalGeneration":
622
- self.tokenizer_mode = "qwen_vl"
619
elif arch == "DeepseekV32ForCausalLM":
620
self.tokenizer_mode = "deepseek_v32"
621
elif arch == "DeepseekV4ForCausalLM":
vllm/envs.py
+1
-1
@@ -686,7 +686,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
686
# If true, replace the Rust BPE backend that powers HF fast tokenizers
687
# with the `fastokens` (https://github.com/crusoecloud/fastokens) shim.
688
# Applies to any tokenizer mode that loads an HF fast tokenizer
689
- # (`hf`, `deepseek_v32`, `deepseek_v4`, `qwen_vl`, …). The `fastokens`
689
+ # (`hf`, `deepseek_v32`, `deepseek_v4`, …). The `fastokens`
690
# Python package must be installed.
691
"VLLM_USE_FASTOKENS": lambda: bool(int(os.getenv("VLLM_USE_FASTOKENS", "0"))),
692
# Interval in seconds to log a warning message when the ring buffer is full
vllm/model_executor/models/qwen.py
deleted
-377
@@ -1,377 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Adapted from
5
-# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
6
-# Copyright (c) Alibaba Cloud.
7
-# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
8
-"""Inference-only QWen model compatible with HuggingFace weights."""
9
-
10
-import json
11
-from collections.abc import Iterable
12
-from itertools import islice
13
-from typing import Any
14
-
15
-import torch
16
-from torch import nn
17
-from transformers import PretrainedConfig
18
-
19
-from vllm.compilation.decorators import support_torch_compile
20
-from vllm.config import CacheConfig, VllmConfig
21
-from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
22
-from vllm.model_executor.layers.activation import SiluAndMul
23
-from vllm.model_executor.layers.attention import Attention
24
-from vllm.model_executor.layers.layernorm import RMSNorm
25
-from vllm.model_executor.layers.linear import (
26
- MergedColumnParallelLinear,
27
- QKVParallelLinear,
28
- RowParallelLinear,
29
-)
30
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
31
-from vllm.model_executor.layers.quantization import QuantizationConfig
32
-from vllm.model_executor.layers.rotary_embedding import get_rope
33
-from vllm.model_executor.layers.vocab_parallel_embedding import (
34
- ParallelLMHead,
35
- VocabParallelEmbedding,
36
-)
37
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38
-from vllm.sequence import IntermediateTensors
39
-
40
-from .interfaces import SupportsLoRA, SupportsPP
41
-from .utils import (
42
- is_pp_missing_parameter,
43
- make_empty_intermediate_tensors_factory,
44
- make_layers,
45
- maybe_prefix,
46
-)
47
-
48
-
49
-class QWenMLP(nn.Module):
50
- """MLP for the language component of the Qwen model, which contains a
51
- MergedColumnParallelLinear merging 2 outputs via silu activation."""
52
-
53
- def __init__(
54
- self,
55
- hidden_size: int,
56
- intermediate_size: int,
57
- hidden_act: str = "silu",
58
- quant_config: QuantizationConfig | None = None,
59
- prefix: str = "",
60
- ):
61
- super().__init__()
62
- self.gate_up_proj = MergedColumnParallelLinear(
63
- hidden_size,
64
- [intermediate_size] * 2,
65
- bias=False,
66
- quant_config=quant_config,
67
- prefix=f"{prefix}.gate_up_proj",
68
- )
69
- self.c_proj = RowParallelLinear(
70
- intermediate_size,
71
- hidden_size,
72
- bias=False,
73
- quant_config=quant_config,
74
- prefix=f"{prefix}.c_proj",
75
- )
76
- if hidden_act != "silu":
77
- raise ValueError(
78
- f"Unsupported activation: {hidden_act}. Only silu is supported for now."
79
- )
80
- self.act_fn = SiluAndMul()
81
-
82
- def forward(self, x: torch.Tensor) -> torch.Tensor:
83
- gate_up, _ = self.gate_up_proj(x)
84
- x = self.act_fn(gate_up)
85
- x, _ = self.c_proj(x)
86
- return x
87
-
88
-
89
-class QWenAttention(nn.Module):
90
- def __init__(
91
- self,
92
- hidden_size: int,
93
- num_heads: int,
94
- max_position_embeddings: int,
95
- rope_parameters: dict[str, Any] | None = None,
96
- cache_config: CacheConfig | None = None,
97
- quant_config: QuantizationConfig | None = None,
98
- prefix: str = "",
99
- ):
100
- super().__init__()
101
- self.hidden_size = hidden_size
102
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
103
- self.total_num_heads = num_heads
104
- assert self.total_num_heads % tensor_model_parallel_world_size == 0
105
- self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
106
- self.head_dim = hidden_size // self.total_num_heads
107
- self.c_attn = QKVParallelLinear(
108
- hidden_size,
109
- self.head_dim,
110
- self.total_num_heads,
111
- bias=True,
112
- quant_config=quant_config,
113
- prefix=f"{prefix}.c_attn",
114
- )
115
- self.c_proj = RowParallelLinear(
116
- self.total_num_heads * self.head_dim,
117
- hidden_size,
118
- bias=False,
119
- quant_config=quant_config,
120
- prefix=f"{prefix}.c_proj",
121
- )
122
- self.scaling = self.head_dim**-0.5
123
-
124
- self.rotary_emb = get_rope(
125
- self.head_dim,
126
- max_position=max_position_embeddings,
127
- rope_parameters=rope_parameters,
128
- )
129
- self.attn = Attention(
130
- self.num_heads,
131
- self.head_dim,
132
- self.scaling,
133
- cache_config=cache_config,
134
- quant_config=quant_config,
135
- prefix=f"{prefix}.attn",
136
- )
137
-
138
- def forward(
139
- self,
140
- positions: torch.Tensor,
141
- hidden_states: torch.Tensor,
142
- ) -> torch.Tensor:
143
- qkv, _ = self.c_attn(hidden_states)
144
- q, k, v = qkv.chunk(chunks=3, dim=-1)
145
- q, k = self.rotary_emb(positions, q, k)
146
- attn_output = self.attn(q, k, v)
147
- output, _ = self.c_proj(attn_output)
148
- return output
149
-
150
-
151
-class QWenBlock(nn.Module):
152
- def __init__(
153
- self,
154
- config: PretrainedConfig,
155
- cache_config: CacheConfig | None = None,
156
- quant_config: QuantizationConfig | None = None,
157
- prefix: str = "",
158
- ):
159
- super().__init__()
160
- self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
161
-
162
- self.attn = QWenAttention(
163
- config.hidden_size,
164
- config.num_attention_heads,
165
- config.max_position_embeddings,
166
- rope_parameters=config.rope_parameters,
167
- cache_config=cache_config,
168
- quant_config=quant_config,
169
- prefix=f"{prefix}.attn",
170
- )
171
-
172
- self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
173
-
174
- self.mlp = QWenMLP(
175
- config.hidden_size,
176
- config.intermediate_size // 2,
177
- quant_config=quant_config,
178
- prefix=f"{prefix}.mlp",
179
- )
180
-
181
- def forward(
182
- self,
183
- positions: torch.Tensor,
184
- hidden_states: torch.Tensor,
185
- residual: torch.Tensor | None,
186
- ) -> tuple[torch.Tensor, torch.Tensor]:
187
- # Self Attention
188
- if residual is None:
189
- residual = hidden_states
190
- hidden_states = self.ln_1(hidden_states)
191
- else:
192
- hidden_states, residual = self.ln_1(hidden_states, residual)
193
- hidden_states = self.attn(
194
- positions=positions,
195
- hidden_states=hidden_states,
196
- )
197
-
198
- # Fully Connected
199
- hidden_states, residual = self.ln_2(hidden_states, residual)
200
- hidden_states = self.mlp(hidden_states)
201
- return hidden_states, residual
202
-
203
-
204
-@support_torch_compile
205
-class QWenModel(nn.Module):
206
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
207
- super().__init__()
208
-
209
- config = vllm_config.model_config.hf_config
210
- cache_config = vllm_config.cache_config
211
- quant_config = vllm_config.quant_config
212
-
213
- self.config = config
214
- self.vocab_size = config.vocab_size
215
-
216
- self.wte = VocabParallelEmbedding(
217
- config.vocab_size,
218
- config.hidden_size,
219
- )
220
- self.start_layer, self.end_layer, self.h = make_layers(
221
- config.num_hidden_layers,
222
- lambda prefix: QWenBlock(config, cache_config, quant_config, prefix=prefix),
223
- prefix=f"{prefix}.h",
224
- )
225
- self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
226
- self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
227
- ["hidden_states", "residual"], config.hidden_size
228
- )
229
-
230
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
231
- return self.wte(input_ids)
232
-
233
- def forward(
234
- self,
235
- input_ids: torch.Tensor | None,
236
- positions: torch.Tensor,
237
- intermediate_tensors: IntermediateTensors | None,
238
- inputs_embeds: torch.Tensor | None = None,
239
- ) -> torch.Tensor | IntermediateTensors:
240
- if get_pp_group().is_first_rank:
241
- if inputs_embeds is not None:
242
- hidden_states = inputs_embeds
243
- else:
244
- hidden_states = self.embed_input_ids(input_ids)
245
- residual = None
246
- else:
247
- assert intermediate_tensors is not None
248
- hidden_states = intermediate_tensors["hidden_states"]
249
- residual = intermediate_tensors["residual"]
250
-
251
- for layer in islice(self.h, self.start_layer, self.end_layer):
252
- hidden_states, residual = layer(
253
- positions,
254
- hidden_states,
255
- residual,
256
- )
257
- if not get_pp_group().is_last_rank:
258
- return IntermediateTensors(
259
- {"hidden_states": hidden_states, "residual": residual}
260
- )
261
- hidden_states, _ = self.ln_f(hidden_states, residual)
262
- return hidden_states
263
-
264
-
265
-class QWenBaseModel(nn.Module):
266
- def __init__(
267
- self,
268
- *,
269
- vllm_config: VllmConfig,
270
- prefix: str = "",
271
- transformer_type: type[QWenModel] = QWenModel,
272
- ) -> None:
273
- super().__init__()
274
- config = vllm_config.model_config.hf_config
275
- quant_config = vllm_config.quant_config
276
- multimodal_config = vllm_config.model_config.multimodal_config
277
- self.config = config
278
- self.multimodal_config = multimodal_config
279
- self.quant_config = quant_config
280
- self.transformer = transformer_type(
281
- vllm_config=vllm_config, prefix=maybe_prefix(prefix, "transformer")
282
- )
283
- self.lm_head = ParallelLMHead(
284
- config.vocab_size,
285
- config.hidden_size,
286
- quant_config=quant_config,
287
- prefix=maybe_prefix(prefix, "lm_head"),
288
- )
289
- if self.config.tie_word_embeddings:
290
- self.lm_head.weight = self.transformer.wte.weight
291
- self.logits_processor = LogitsProcessor(config.vocab_size)
292
- self.make_empty_intermediate_tensors = (
293
- self.transformer.make_empty_intermediate_tensors
294
- )
295
-
296
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
297
- return self.transformer.wte(input_ids)
298
-
299
- def compute_logits(
300
- self,
301
- hidden_states: torch.Tensor,
302
- ) -> torch.Tensor | None:
303
- logits = self.logits_processor(self.lm_head, hidden_states)
304
- return logits
305
-
306
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
307
- stacked_params_mapping = [
308
- # (param_name, shard_name, shard_id)
309
- ("gate_up_proj", "w2", 0),
310
- ("gate_up_proj", "w1", 1),
311
- ]
312
- params_dict = dict(self.named_parameters())
313
- loaded_params: set[str] = set()
314
- for name, loaded_weight in weights:
315
- if "rotary_emb.inv_freq" in name:
316
- continue
317
- for param_name, weight_name, shard_id in stacked_params_mapping:
318
- if weight_name not in name:
319
- continue
320
- name = name.replace(weight_name, param_name)
321
- # Skip loading extra bias for GPTQ models.
322
- if name.endswith(".bias") and name not in params_dict:
323
- continue
324
- # Skip layers on other devices.
325
- if is_pp_missing_parameter(name, self):
326
- continue
327
- param = params_dict[name]
328
- weight_loader = param.weight_loader
329
- weight_loader(param, loaded_weight, shard_id)
330
- break
331
- else:
332
- # Skip loading extra bias for GPTQ models.
333
- if name.endswith(".bias") and name not in params_dict:
334
- continue
335
- # Skip layers on other devices.
336
- if is_pp_missing_parameter(name, self):
337
- continue
338
- param = params_dict[name]
339
- weight_loader = getattr(param, "weight_loader", default_weight_loader)
340
- weight_loader(param, loaded_weight)
341
- loaded_params.add(name)
342
- return loaded_params
343
-
344
-
345
-class QWenLMHeadModel(QWenBaseModel, SupportsPP, SupportsLoRA):
346
- packed_modules_mapping = {
347
- "c_attn": ["c_attn"],
348
- "gate_up_proj": [
349
- "w2",
350
- "w1",
351
- ],
352
- }
353
-
354
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
355
- config = vllm_config.model_config.hf_config
356
- if hasattr(config, "visual"):
357
- hf_overrides = {"architectures": ["QwenVLForConditionalGeneration"]}
358
- raise RuntimeError(
359
- "The configuration of this model indicates that it supports "
360
- "vision inputs, but you instantiated the text-only version "
361
- "of this model. Please use the vision model by setting "
362
- f"`--hf-overrides '{json.dumps(hf_overrides)}'`"
363
- )
364
-
365
- super().__init__(vllm_config=vllm_config, prefix=prefix)
366
-
367
- def forward(
368
- self,
369
- input_ids: torch.Tensor | None,
370
- positions: torch.Tensor,
371
- intermediate_tensors: IntermediateTensors | None = None,
372
- inputs_embeds: torch.Tensor | None = None,
373
- ) -> torch.Tensor | IntermediateTensors:
374
- hidden_states = self.transformer(
375
- input_ids, positions, intermediate_tensors, inputs_embeds
376
- )
377
- return hidden_states
vllm/model_executor/models/qwen_vl.py
deleted
-688
@@ -1,688 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Adapted from
5
-# https://huggingface.co/Qwen/Qwen-VL/blob/main/modeling_qwen.py
6
-# Copyright (c) Alibaba Cloud.
7
-"""Inference-only Qwen-VL model compatible with HuggingFace weights."""
8
-
9
-import math
10
-from collections.abc import Callable, Mapping, Sequence
11
-from functools import partial
12
-from typing import Annotated, Literal, TypeAlias
13
-
14
-import regex as re
15
-import torch
16
-from torch import nn
17
-from transformers import BatchFeature
18
-
19
-from vllm.config import VllmConfig
20
-from vllm.config.multimodal import BaseDummyOptions
21
-from vllm.inputs import MultiModalDataDict
22
-from vllm.model_executor.layers.activation import get_act_fn
23
-from vllm.model_executor.layers.conv import Conv2dLayer
24
-from vllm.model_executor.layers.linear import (
25
- ColumnParallelLinear,
26
- ReplicatedLinear,
27
- RowParallelLinear,
28
-)
29
-from vllm.model_executor.layers.quantization import QuantizationConfig
30
-from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
31
-from vllm.model_executor.models.module_mapping import MultiModelKeys
32
-from vllm.multimodal import MULTIMODAL_REGISTRY
33
-from vllm.multimodal.inputs import (
34
- MultiModalFieldConfig,
35
- MultiModalKwargsItems,
36
-)
37
-from vllm.multimodal.parse import MultiModalDataItems
38
-from vllm.multimodal.processing import (
39
- BaseDummyInputsBuilder,
40
- BaseMultiModalProcessor,
41
- BaseProcessingInfo,
42
- PromptReplacement,
43
- PromptUpdate,
44
- PromptUpdateDetails,
45
-)
46
-from vllm.sequence import IntermediateTensors
47
-from vllm.transformers_utils.processors.qwen_vl import (
48
- QwenVLImageProcessorFast,
49
- QwenVLProcessor,
50
-)
51
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
52
-
53
-from .interfaces import (
54
- MultiModalEmbeddings,
55
- SupportsLoRA,
56
- SupportsMultiModal,
57
- SupportsPP,
58
-)
59
-from .qwen import QWenBaseModel, QWenBlock, QWenModel
60
-
61
-
62
-class QwenImagePixelInputs(TensorSchema):
63
- """
64
- Dimensions:
65
- - bn: Batch size * number of images
66
- - c: Number of channels (3)
67
- - h: Height
68
- - w: Width
69
-
70
- Note that image_size is the value in the vision config to which we resize
71
- the image to in the normalization transform. Currently multi-image support
72
- can only be leveraged by passing image embeddings directly.
73
- """
74
-
75
- type: Literal["pixel_values"] = "pixel_values"
76
- data: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]
77
-
78
-
79
-class QwenImageEmbeddingInputs(TensorSchema):
80
- """
81
- Dimensions:
82
- - bn: Batch size * number of images
83
- - ifs: Image feature size (256)
84
- - hs: Hidden size
85
-
86
- `hidden_size` must match the hidden size of the language model backbone
87
- and is stored in the visual config of the model if we have one.
88
- """
89
-
90
- type: Literal["image_embeds"] = "image_embeds"
91
- data: Annotated[torch.Tensor, TensorShape("bn", 256, "hs")]
92
-
93
-
94
-QwenImageInputs: TypeAlias = QwenImagePixelInputs | QwenImageEmbeddingInputs
95
-
96
-
97
-class VisualAttention(nn.Module):
98
- """self-attention layer class.
99
- Self-attention layer takes input with size [s, b, h]
100
- and returns output of the same size.
101
- """
102
-
103
- def __init__(
104
- self,
105
- embed_dim: int,
106
- num_heads: int,
107
- bias: bool = True,
108
- kdim: int | None = None,
109
- vdim: int | None = None,
110
- prefix: str = "",
111
- ):
112
- super().__init__()
113
- self.embed_dim = embed_dim
114
- self.kdim = kdim if kdim is not None else embed_dim
115
- self.vdim = vdim if vdim is not None else embed_dim
116
- self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim
117
-
118
- self.num_heads = num_heads
119
-
120
- # Per attention head and per partition values.
121
- assert embed_dim % num_heads == 0
122
- self.hidden_size_per_attention_head = embed_dim // num_heads
123
- self.num_attention_heads_per_partition = num_heads
124
- self.hidden_size_per_partition = embed_dim
125
-
126
- # Strided linear layer.
127
- assert self._qkv_same_embed_dim, (
128
- "Visual Attention implementation only supports self-attention"
129
- )
130
- self.in_proj = ReplicatedLinear(
131
- embed_dim, 3 * embed_dim, prefix=f"{prefix}.in_proj"
132
- )
133
- self.out_proj = ReplicatedLinear(
134
- embed_dim, embed_dim, prefix=f"{prefix}.out_proj"
135
- )
136
- self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
137
-
138
- def forward(
139
- self,
140
- x: torch.Tensor,
141
- attn_mask: torch.Tensor | None = None,
142
- ) -> torch.Tensor:
143
- # query/key/value: [sq, b, h]
144
- sq, b, _ = x.size()
145
- mixed_x_layer, _ = self.in_proj(x)
146
-
147
- # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
148
- new_tensor_shape = mixed_x_layer.size()[:-1] + (
149
- self.num_attention_heads_per_partition,
150
- 3 * self.hidden_size_per_attention_head,
151
- )
152
- mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
153
-
154
- # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
155
- query_layer, key_layer, value_layer = mixed_x_layer.split(
156
- self.hidden_size_per_attention_head, dim=-1
157
- )
158
-
159
- # [sq, b, np, hn] -> [sq, b * np, hn]
160
- query_layer = query_layer.view(
161
- sq,
162
- b * self.num_attention_heads_per_partition,
163
- self.hidden_size_per_attention_head,
164
- ).transpose(0, 1)
165
- # [sk, b, np, hn] -> [sk, b * np, hn]
166
- key_layer = key_layer.view(
167
- sq,
168
- b * self.num_attention_heads_per_partition,
169
- self.hidden_size_per_attention_head,
170
- ).transpose(0, 1)
171
-
172
- q_scaled = query_layer / self.norm_factor
173
- if attn_mask is not None:
174
- attention_probs = torch.baddbmm(
175
- attn_mask, q_scaled, key_layer.transpose(-2, -1)
176
- )
177
- else:
178
- attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1))
179
- attention_probs = attention_probs.softmax(dim=-1)
180
-
181
- value_layer = value_layer.view(
182
- sq,
183
- b * self.num_attention_heads_per_partition,
184
- self.hidden_size_per_attention_head,
185
- ).transpose(0, 1)
186
-
187
- # matmul: [b * np, sq, hn]
188
- context_layer = torch.bmm(attention_probs, value_layer)
189
-
190
- # change view [b, np, sq, hn]
191
- context_layer = context_layer.view(
192
- b,
193
- self.num_attention_heads_per_partition,
194
- sq,
195
- self.hidden_size_per_attention_head,
196
- )
197
-
198
- # [b, np, sq, hn] --> [sq, b, np, hn]
199
- context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
200
-
201
- # [sq, b, np, hn] --> [sq, b, hp]
202
- new_context_layer_shape = context_layer.size()[:-2] + (
203
- self.hidden_size_per_partition,
204
- )
205
- context_layer = context_layer.view(*new_context_layer_shape)
206
-
207
- output, _ = self.out_proj(context_layer)
208
-
209
- return output
210
-
211
-
212
-class QwenVLMLP(nn.Module):
213
- """MLP for the visual component of the Qwen model."""
214
-
215
- def __init__(
216
- self,
217
- hidden_size: int,
218
- intermediate_size: int,
219
- quant_config: QuantizationConfig | None = None,
220
- prefix: str = "",
221
- ):
222
- super().__init__()
223
- self.c_fc = ColumnParallelLinear(
224
- hidden_size,
225
- intermediate_size,
226
- bias=True,
227
- quant_config=quant_config,
228
- prefix=f"{prefix}.c_fc",
229
- )
230
- self.act_fn = get_act_fn("gelu")
231
- self.c_proj = RowParallelLinear(
232
- intermediate_size,
233
- hidden_size,
234
- bias=True,
235
- quant_config=quant_config,
236
- prefix=f"{prefix}.c_proj",
237
- )
238
-
239
- def forward(self, x):
240
- x, _ = self.c_fc(x)
241
- x = self.act_fn(x)
242
- x, _ = self.c_proj(x)
243
- return x
244
-
245
-
246
-class VisualAttentionBlock(nn.Module):
247
- def __init__(
248
- self,
249
- d_model: int,
250
- n_head: int,
251
- mlp_ratio: float = 4.0,
252
- norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
253
- quant_config: QuantizationConfig | None = None,
254
- prefix: str = "",
255
- ):
256
- super().__init__()
257
-
258
- self.ln_1 = norm_layer(d_model)
259
- self.ln_2 = norm_layer(d_model)
260
- mlp_width = int(d_model * mlp_ratio)
261
- self.attn = VisualAttention(d_model, n_head, prefix=f"{prefix}.attn")
262
- self.mlp = QwenVLMLP(
263
- hidden_size=d_model,
264
- intermediate_size=mlp_width,
265
- quant_config=quant_config,
266
- prefix=f"{prefix}.mlp",
267
- )
268
-
269
- def attention(
270
- self,
271
- x: torch.Tensor,
272
- attn_mask: torch.Tensor | None = None,
273
- ) -> torch.Tensor:
274
- attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None
275
- return self.attn(x, attn_mask=attn_mask)
276
-
277
- def forward(
278
- self,
279
- x: torch.Tensor,
280
- attn_mask: torch.Tensor | None = None,
281
- ) -> torch.Tensor:
282
- x = x + self.attention(self.ln_1(x), attn_mask=attn_mask)
283
- x = x + self.mlp(self.ln_2(x))
284
- return x
285
-
286
-
287
-class TransformerBlock(nn.Module):
288
- def __init__(
289
- self,
290
- width: int,
291
- layers: int,
292
- heads: int,
293
- mlp_ratio: float = 4.0,
294
- norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
295
- quant_config: QuantizationConfig | None = None,
296
- prefix: str = "",
297
- ):
298
- super().__init__()
299
- self.width = width
300
- self.layers = layers
301
-
302
- self.resblocks = nn.ModuleList(
303
- [
304
- VisualAttentionBlock(
305
- width,
306
- heads,
307
- mlp_ratio,
308
- norm_layer=norm_layer,
309
- quant_config=quant_config,
310
- prefix=f"{prefix}.resblocks.{i}",
311
- )
312
- for i in range(layers)
313
- ]
314
- )
315
-
316
- def get_cast_dtype(self) -> torch.dtype:
317
- return self.resblocks[0].mlp.c_fc.weight.dtype
318
-
319
- def get_cast_device(self) -> torch.device:
320
- return self.resblocks[0].mlp.c_fc.weight.device
321
-
322
- def forward(
323
- self, x: torch.Tensor, attn_mask: torch.Tensor | None = None
324
- ) -> torch.Tensor:
325
- for r in self.resblocks:
326
- x = r(x, attn_mask=attn_mask)
327
- return x
328
-
329
-
330
-class VisionTransformer(nn.Module):
331
- def __init__(
332
- self,
333
- image_size: int,
334
- patch_size: int,
335
- width: int,
336
- layers: int,
337
- heads: int,
338
- mlp_ratio: float,
339
- n_queries: int = 256,
340
- output_dim: int = 512,
341
- image_start_id: int = 151857,
342
- quant_config: QuantizationConfig | None = None,
343
- prefix: str = "",
344
- **kwargs,
345
- ):
346
- super().__init__()
347
- image_height, image_width = self.image_size = (image_size, image_size)
348
- patch_height, patch_width = self.patch_size = (patch_size, patch_size)
349
- self.grid_size = (image_height // patch_height, image_width // patch_width)
350
- self.output_dim = output_dim
351
- self.conv1 = Conv2dLayer(
352
- in_channels=3,
353
- out_channels=width,
354
- kernel_size=patch_size,
355
- stride=patch_size,
356
- bias=False,
357
- )
358
-
359
- # class embeddings and positional embeddings
360
- scale = width**-0.5
361
- self.positional_embedding = nn.Parameter(scale * torch.randn(256, width))
362
-
363
- norm_layer = partial(nn.LayerNorm, eps=1e-6)
364
-
365
- self.ln_pre = norm_layer(width)
366
- self.transformer = TransformerBlock(
367
- width,
368
- layers,
369
- heads,
370
- mlp_ratio,
371
- norm_layer=norm_layer,
372
- quant_config=quant_config,
373
- prefix=f"{prefix}.transformer",
374
- )
375
-
376
- self.attn_pool = Resampler2(
377
- grid_size=int(math.sqrt(n_queries)),
378
- embed_dim=output_dim,
379
- num_heads=output_dim // 128,
380
- kv_dim=width,
381
- norm_layer=norm_layer,
382
- adaptive=False,
383
- do_post_projection=False,
384
- prefix=f"{prefix}.attn_pool",
385
- ).to(
386
- device=self.positional_embedding.device,
387
- dtype=self.positional_embedding.dtype,
388
- )
389
-
390
- self.ln_post = norm_layer(output_dim)
391
- self.proj = nn.Parameter(
392
- (output_dim**-0.5) * torch.randn(output_dim, output_dim)
393
- )
394
-
395
- self.image_start_id = image_start_id
396
- self.image_end_id = image_start_id + 1
397
- self.image_pad_id = image_start_id + 2
398
-
399
- def forward(self, x: torch.Tensor) -> torch.Tensor:
400
- x = x.to(
401
- dtype=self.transformer.get_cast_dtype(),
402
- device=self.transformer.get_cast_device(),
403
- )
404
-
405
- # to patches
406
- x = self.conv1(x) # shape = [*, width, grid, grid]
407
- x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2]
408
- x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
409
-
410
- x = x + get_abs_pos(self.positional_embedding, int(math.sqrt(x.size(1))))
411
-
412
- x = self.ln_pre(x)
413
-
414
- x = x.permute(1, 0, 2) # NLD -> LND
415
- x = self.transformer(x)
416
- x = x.permute(1, 0, 2) # LND -> NLD
417
-
418
- x = self.attn_pool(x)
419
- x = self.ln_post(x)
420
- x = x @ self.proj
421
-
422
- return x
423
-
424
-
425
-class QwenVLModel(QWenModel):
426
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
427
- super().__init__(vllm_config=vllm_config, prefix=prefix)
428
-
429
- config = vllm_config.model_config.hf_config
430
- quant_config = vllm_config.quant_config
431
-
432
- self.visual = VisionTransformer(
433
- **config.visual, quant_config=quant_config, prefix=f"{prefix}.visual"
434
- )
435
-
436
-
437
-class QwenVLProcessingInfo(BaseProcessingInfo):
438
- def get_image_processor(self, **kwargs):
439
- config = self.get_hf_config()
440
- vision_config = config.visual
441
-
442
- image_size = vision_config["image_size"]
443
- kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
444
- kwargs.setdefault("size", {"width": image_size, "height": image_size})
445
-
446
- return QwenVLImageProcessorFast(**kwargs)
447
-
448
- def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
449
- return QwenVLProcessor(
450
- tokenizer=self.get_tokenizer(),
451
- image_processor=self.get_image_processor(**kwargs),
452
- )
453
-
454
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
455
- return {"image": None}
456
-
457
- def get_num_image_tokens(self) -> int:
458
- hf_config = self.get_hf_config()
459
- vision_config = hf_config.visual
460
-
461
- image_size = vision_config["image_size"]
462
- patch_size = vision_config["patch_size"]
463
- grid_length = image_size // patch_size // 2
464
- return grid_length * grid_length
465
-
466
-
467
-class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):
468
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
469
- num_images = mm_counts.get("image", 0)
470
-
471
- hf_processor = self.info.get_hf_processor()
472
- img_start = hf_processor.image_start_tag
473
- img_end = hf_processor.image_end_tag
474
-
475
- return "".join(
476
- f"Picture {i}: {img_start}{img_end}\n" for i in range(1, num_images + 1)
477
- )
478
-
479
- def get_dummy_mm_data(
480
- self,
481
- seq_len: int,
482
- mm_counts: Mapping[str, int],
483
- mm_options: Mapping[str, BaseDummyOptions],
484
- ) -> MultiModalDataDict:
485
- hf_config = self.info.get_hf_config()
486
- vision_config = hf_config.visual
487
-
488
- target_width = target_height = vision_config["image_size"]
489
- num_images = mm_counts.get("image", 0)
490
-
491
- image_overrides = mm_options.get("image")
492
-
493
- return {
494
- "image": self._get_dummy_images(
495
- width=target_width,
496
- height=target_height,
497
- num_images=num_images,
498
- overrides=image_overrides,
499
- )
500
- }
501
-
502
-
503
-class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
504
- def _call_hf_processor(
505
- self,
506
- prompt: str,
507
- mm_data: Mapping[str, object],
508
- mm_kwargs: Mapping[str, object],
509
- tok_kwargs: Mapping[str, object],
510
- ) -> BatchFeature:
511
- # Drops anything between <img>/</img> tags; encoding with the tokenizer
512
- # will automatically add the image pads for the context.
513
- prompt, num_matched_images = re.subn(
514
- r"(Picture \d*: <img>).*?(<\/img>\n)",
515
- r"\1\2",
516
- prompt,
517
- )
518
-
519
- image_data = mm_data.get("images")
520
- if image_data is not None:
521
- assert isinstance(image_data, list)
522
-
523
- num_images = len(image_data)
524
- assert num_matched_images == num_images
525
-
526
- return super()._call_hf_processor(
527
- prompt=prompt,
528
- mm_data=mm_data,
529
- mm_kwargs=mm_kwargs,
530
- tok_kwargs=tok_kwargs,
531
- )
532
-
533
- def _hf_processor_applies_updates(
534
- self,
535
- prompt_text: str,
536
- mm_items: MultiModalDataItems,
537
- hf_processor_mm_kwargs: Mapping[str, object],
538
- tokenization_kwargs: Mapping[str, object],
539
- ) -> bool:
540
- return False
541
-
542
- def _get_mm_fields_config(
543
- self,
544
- hf_inputs: BatchFeature,
545
- hf_processor_mm_kwargs: Mapping[str, object],
546
- ) -> Mapping[str, MultiModalFieldConfig]:
547
- return dict(
548
- pixel_values=MultiModalFieldConfig.batched("image"),
549
- image_embeds=MultiModalFieldConfig.batched("image"),
550
- )
551
-
552
- def _get_prompt_updates(
553
- self,
554
- mm_items: MultiModalDataItems,
555
- hf_processor_mm_kwargs: Mapping[str, object],
556
- out_mm_kwargs: MultiModalKwargsItems,
557
- ) -> Sequence[PromptUpdate]:
558
- tokenizer = self.info.get_tokenizer()
559
- special_tokens: dict[str, int] = tokenizer.special_tokens # type: ignore
560
-
561
- processor = self.info.get_hf_processor()
562
- img_start_id = special_tokens[processor.image_start_tag]
563
- img_end_id = special_tokens[processor.image_end_tag]
564
- img_pad_id = special_tokens[processor.image_pad_tag]
565
-
566
- num_image_tokens = self.info.get_num_image_tokens()
567
- image_tokens = [img_pad_id] * num_image_tokens
568
-
569
- return [
570
- PromptReplacement(
571
- modality="image",
572
- target=[img_start_id, img_end_id],
573
- replacement=PromptUpdateDetails.select_token_id(
574
- [img_start_id] + image_tokens + [img_end_id],
575
- embed_token_id=img_pad_id,
576
- ),
577
- )
578
- ]
579
-
580
-
581
-@MULTIMODAL_REGISTRY.register_processor(
582
- QwenVLMultiModalProcessor,
583
- info=QwenVLProcessingInfo,
584
- dummy_inputs=QwenVLDummyInputsBuilder,
585
-)
586
-class QwenVLForConditionalGeneration(
587
- QWenBaseModel, SupportsPP, SupportsLoRA, SupportsMultiModal
588
-):
589
- packed_modules_mapping = {
590
- "c_attn": ["c_attn"],
591
- "gate_up_proj": [
592
- "w2",
593
- "w1",
594
- ],
595
- }
596
-
597
- embed_input_ids = SupportsMultiModal.embed_input_ids
598
-
599
- def get_mm_mapping(self) -> MultiModelKeys:
600
- """
601
- Get the module prefix in multimodal models
602
- """
603
- return MultiModelKeys.from_string_field(
604
- language_model="transformer.h",
605
- connector="transformer.visual.attn_pool",
606
- tower_model="transformer.visual.transformer",
607
- )
608
-
609
- @classmethod
610
- def get_placeholder_str(cls, modality: str, i: int) -> str | None:
611
- if modality.startswith("image"):
612
- return f"Picture {i}: <img></img>"
613
-
614
- raise ValueError("Only image modality is supported")
615
-
616
- def __init__(
617
- self,
618
- *,
619
- vllm_config: VllmConfig,
620
- prefix: str = "",
621
- transformer_type: type[QwenVLModel] = QwenVLModel,
622
- ) -> None:
623
- with self._mark_composite_model(
624
- vllm_config,
625
- language_targets=QWenBlock,
626
- tower_targets={"image": VisionTransformer},
627
- ):
628
- super().__init__(
629
- vllm_config=vllm_config,
630
- prefix=prefix,
631
- transformer_type=transformer_type,
632
- )
633
-
634
- self.transformer: QwenVLModel
635
-
636
- def _parse_and_validate_image_input(
637
- self, **kwargs: object
638
- ) -> QwenImageInputs | None:
639
- pixel_values = kwargs.pop("pixel_values", None)
640
- image_embeds = kwargs.pop("image_embeds", None)
641
-
642
- if pixel_values is not None:
643
- expected_h = expected_w = self.config.visual["image_size"]
644
- resolve_bindings = {"h": expected_h, "w": expected_w}
645
-
646
- return QwenImagePixelInputs(
647
- type="pixel_values",
648
- data=pixel_values,
649
- resolve_bindings=resolve_bindings,
650
- )
651
-
652
- if image_embeds is not None:
653
- return QwenImageEmbeddingInputs(
654
- type="image_embeds",
655
- data=image_embeds,
656
- )
657
-
658
- return None
659
-
660
- def _process_image_input(self, image_input: QwenImageInputs) -> torch.Tensor:
661
- if image_input["type"] == "image_embeds":
662
- return image_input["data"]
663
-
664
- return self.transformer.visual(image_input["data"])
665
-
666
- def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
667
- image_input = self._parse_and_validate_image_input(**kwargs)
668
- if image_input is None:
669
- return []
670
-
671
- vision_embeddings = self._process_image_input(image_input)
672
- return vision_embeddings
673
-
674
- def forward(
675
- self,
676
- input_ids: torch.Tensor | None,
677
- positions: torch.Tensor,
678
- intermediate_tensors: IntermediateTensors | None = None,
679
- inputs_embeds: torch.Tensor | None = None,
680
- **kwargs: object,
681
- ) -> torch.Tensor | IntermediateTensors:
682
- if intermediate_tensors is not None:
683
- inputs_embeds = None
684
-
685
- hidden_states = self.transformer(
686
- input_ids, positions, intermediate_tensors, inputs_embeds
687
- )
688
- return hidden_states
vllm/model_executor/models/registry.py
+2
-2
@@ -197,7 +197,6 @@ _TEXT_GENERATION_MODELS = {
197
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
198
"Plamo2ForCausalLM": ("plamo2", "Plamo2ForCausalLM"),
199
"Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
200
- "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
200
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
201
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
202
"Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
@@ -531,7 +530,6 @@ _MULTIMODAL_MODELS = {
530
"qianfan_ocr",
531
"QianfanOCRForConditionalGeneration",
532
),
534
- "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),
533
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
534
"Qwen2_5_VLForConditionalGeneration": (
535
"qwen2_5_vl",
@@ -715,6 +713,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
713
"ErnieModel": "0.23.0",
714
"ErnieForSequenceClassification": "0.23.0",
715
"ErnieForTokenClassification": "0.23.0",
716
+ "QWenLMHeadModel": "0.23.0",
717
+ "QwenVLForConditionalGeneration": "0.23.0",
718
# encoder-decoder models except whisper
719
# have been removed for V0 deprecation.
720
"DonutForConditionalGeneration": "0.10.2",
vllm/renderers/registry.py
-1
@@ -26,7 +26,6 @@ _VLLM_RENDERERS = {
26
"hf": ("hf", "HfRenderer"),
27
"kimi_audio": ("hf", "HfRenderer"),
28
"mistral": ("mistral", "MistralRenderer"),
29
- "qwen_vl": ("hf", "HfRenderer"),
29
"terratorch": ("terratorch", "TerratorchRenderer"),
30
}
31
vllm/tokenizers/fastokens.py
+1
-1
@@ -7,7 +7,7 @@ the inner Rust tokenizer of every HF fast tokenizer loaded afterwards with the
7
fastokens shim and rebinds ``tokenizers.decoders.DecodeStream`` so the
8
streaming detokenizer accepts the shim. The patch is process-global and
9
idempotent, so it applies to any tokenizer mode that ends up loading an HF
10
-fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, `qwen_vl`, …).
10
+fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, …).
11
"""
12
13
from importlib.metadata import PackageNotFoundError, version
vllm/tokenizers/qwen_vl.py
deleted
-71
@@ -1,71 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import copy
4
-import unicodedata
5
-from collections.abc import Collection, Set
6
-
7
-from transformers import AutoTokenizer
8
-
9
-from .hf import HfTokenizer, get_cached_tokenizer
10
-from .protocol import TokenizerLike
11
-
12
-
13
-def get_qwen_vl_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
14
- """
15
- The logic of adding image pad tokens should only be applied in
16
- `QwenVLProcessor`, so they are patched out here.
17
-
18
- The definition of the wrapped tokenizer can be found here:
19
- https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
20
- """
21
- new_tokenizer = copy.copy(tokenizer)
22
-
23
- class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
24
- def tokenize(
25
- self,
26
- text: str,
27
- allowed_special: Set[str] | str = "all",
28
- disallowed_special: Collection[str] | str = (),
29
- **kwargs,
30
- ) -> list[bytes | str]:
31
- text = unicodedata.normalize("NFC", text)
32
-
33
- return [
34
- self.decoder[t]
35
- for t in self.tokenizer.encode(
36
- text,
37
- allowed_special=allowed_special,
38
- disallowed_special=disallowed_special,
39
- )
40
- ]
41
-
42
- def _decode(
43
- self,
44
- token_ids: int | list[int],
45
- skip_special_tokens: bool = False,
46
- errors: str | None = None,
47
- **kwargs,
48
- ) -> str:
49
- if isinstance(token_ids, int):
50
- token_ids = [token_ids]
51
-
52
- return self.tokenizer.decode(
53
- token_ids,
54
- errors=errors or self.errors,
55
- )
56
-
57
- TokenizerWithoutImagePad.__name__ = f"{tokenizer.__class__.__name__}WithoutImagePad"
58
-
59
- new_tokenizer.__class__ = TokenizerWithoutImagePad
60
- return new_tokenizer
61
-
62
-
63
-class QwenVLTokenizer(TokenizerLike):
64
- image_start_tag: str
65
- image_end_tag: str
66
- image_pad_tag: str
67
-
68
- @classmethod
69
- def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
70
- tokenizer = AutoTokenizer.from_pretrained(*args, **kwargs)
71
- return get_cached_tokenizer(get_qwen_vl_tokenizer(tokenizer))
vllm/tokenizers/registry.py
-1
@@ -47,7 +47,6 @@ _VLLM_TOKENIZERS = {
47
"hf": ("hf", "CachedHfTokenizer"),
48
"kimi_audio": ("kimi_audio", "KimiAudioTokenizer"),
49
"mistral": ("mistral", "MistralTokenizer"),
50
- "qwen_vl": ("qwen_vl", "QwenVLTokenizer"),
50
}
51
52
vllm/transformers_utils/chat_templates/registry.py
-8
@@ -13,13 +13,6 @@ CHAT_TEMPLATES_DIR = Path(__file__).parent
13
ChatTemplatePath: TypeAlias = Path | Callable[[str], Path | None]
14
15
16
-def _get_qwen_chat_template_fallback(tokenizer_name_or_path: str) -> Path | None:
17
- if tokenizer_name_or_path.endswith("-Chat"):
18
- return CHAT_TEMPLATES_DIR / "template_chatml.jinja"
19
-
20
- return CHAT_TEMPLATES_DIR / "template_basic.jinja"
21
-
22
-
16
def _get_minicpmv_chat_template_fallback(tokenizer_name_or_path: str) -> Path | None:
17
# MiniCPM-V-4.5 version uses a dedicated template
18
if "4.5" in tokenizer_name_or_path or "4_5" in tokenizer_name_or_path:
@@ -41,7 +34,6 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
34
"minicpmv": _get_minicpmv_chat_template_fallback,
35
"minicpmv4_6": _get_minicpmv_chat_template_fallback,
36
"paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
44
- "qwen": _get_qwen_chat_template_fallback,
37
"siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
38
"siglip2": CHAT_TEMPLATES_DIR / "template_basic.jinja",
39
}
vllm/transformers_utils/processors/__init__.py
-2
@@ -40,7 +40,6 @@ __all__ = [
40
"OpenVLAProcessor",
41
"OvisProcessor",
42
"Ovis2_5Processor",
43
- "QwenVLProcessor",
43
"Qwen3ASRProcessor",
44
"Step3VLProcessor",
45
]
@@ -75,7 +74,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
74
"OpenVLAProcessor": "vllm.transformers_utils.processors.openvla",
75
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
76
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
78
- "QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl",
77
"Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
78
"Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl",
79
}
vllm/transformers_utils/processors/qwen_vl.py
deleted
-42
@@ -1,42 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Adapted from
5
-# https://huggingface.co/Qwen/Qwen-VL/blob/main/modeling_qwen.py
6
-# Copyright (c) Alibaba Cloud.
7
-from transformers.image_processing_utils_fast import BaseImageProcessorFast
8
-from transformers.image_utils import PILImageResampling
9
-from transformers.processing_utils import ProcessorMixin
10
-
11
-from vllm.tokenizers.qwen_vl import QwenVLTokenizer
12
-
13
-
14
-class QwenVLImageProcessorFast(BaseImageProcessorFast):
15
- """
16
- Port of https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
17
- to HF Transformers.
18
- """
19
-
20
- resample = PILImageResampling.BICUBIC
21
- image_mean = [0.48145466, 0.4578275, 0.40821073]
22
- image_std = [0.26862954, 0.26130258, 0.27577711]
23
- size = {"height": 448, "width": 448}
24
- do_resize = True
25
- do_rescale = True
26
- do_normalize = True
27
-
28
-
29
-class QwenVLProcessor(ProcessorMixin):
30
- attributes = ["image_processor", "tokenizer"]
31
-
32
- def __init__(
33
- self,
34
- image_processor: QwenVLImageProcessorFast,
35
- tokenizer: QwenVLTokenizer,
36
- ) -> None:
37
- self.image_processor = image_processor
38
- self.tokenizer = tokenizer
39
-
40
- self.image_start_tag = tokenizer.image_start_tag
41
- self.image_end_tag = tokenizer.image_end_tag
42
- self.image_pad_tag = tokenizer.image_pad_tag