A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[VLM] Separate text-only and vision variants of the same model architecture (#13157)
Cyrus Leung committed
Feb 13, 2025 at 22:19 UTC
1bc3b5e71b26d82e655f09d4d1733b7706ea3ff6
15 files changed
+1729
-1643
docs/source/models/supported_models.md
+8
-9
@@ -699,10 +699,10 @@ See [this page](#generative-models) for more information on how to use generativ
699
*
700
* ✅︎
701
* ✅︎
702
-- * `DeepseekVLV2ForCausalLM`
702
+- * `DeepseekVLV2ForCausalLM`<sup>^</sup>
703
* DeepSeek-VL2
704
* T + I<sup>+</sup>
705
- * `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc. (see note)
705
+ * `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc.
706
*
707
* ✅︎
708
* ✅︎
@@ -713,10 +713,10 @@ See [this page](#generative-models) for more information on how to use generativ
713
*
714
* ✅︎
715
* ✅︎
716
-- * `ChatGLMModel`
716
+- * `GLM4VForCausalLM`<sup>^</sup>
717
* GLM-4V
718
* T + I
719
- * `THUDM/glm-4v-9b` etc.
719
+ * `THUDM/glm-4v-9b`, `THUDM/cogagent-9b-20241220` etc.
720
* ✅︎
721
* ✅︎
722
* ✅︎
@@ -825,7 +825,7 @@ See [this page](#generative-models) for more information on how to use generativ
825
*
826
* ✅︎
827
* ✅︎
828
-- * `QWenLMHeadModel`
828
+- * `QwenVLForConditionalGeneration`<sup>^</sup>
829
* Qwen-VL
830
* T + I<sup>E+</sup>
831
* `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc.
@@ -862,13 +862,12 @@ See [this page](#generative-models) for more information on how to use generativ
862
* ✅︎
863
:::
864
865
+<sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
866
+ • For example, to use DeepSeek-VL2 series models:
867
+ `--hf-overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'`
868
<sup>E</sup> Pre-computed embeddings can be inputted for this modality.
869
<sup>+</sup> Multiple items can be inputted per text prompt for this modality.
870
868
-:::{note}
869
-To use DeepSeek-VL2 series models, you have to pass `--hf_overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'` when running vLLM.
870
-:::
871
-
871
:::{note}
872
H2O-VL series models will be available in V1 once we support backends other than FlashAttention.
873
:::
examples/offline_inference/vision_language.py
+3
@@ -105,7 +105,9 @@ def run_glm4v(question: str, modality: str):
105
max_num_seqs=2,
106
trust_remote_code=True,
107
enforce_eager=True,
108
+ hf_overrides={"architectures": ["GLM4VForCausalLM"]},
109
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
110
+
111
prompt = f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
112
{question}<|assistant|>"
113
@@ -495,6 +497,7 @@ def run_qwen_vl(question: str, modality: str):
497
trust_remote_code=True,
498
max_model_len=1024,
499
max_num_seqs=2,
500
+ hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
501
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
502
)
503
examples/offline_inference/vision_language_multi_image.py
+3
-2
@@ -77,7 +77,7 @@ def load_deepseek_vl2(question: str, image_urls: List[str]):
77
)
78
79
80
-def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
80
+def load_h2ovl(question: str, image_urls: List[str]) -> ModelRequestData:
81
model_name = "h2oai/h2ovl-mississippi-2b"
82
83
llm = LLM(
@@ -302,6 +302,7 @@ def load_qwen_vl_chat(question: str,
302
trust_remote_code=True,
303
max_model_len=1024,
304
max_num_seqs=2,
305
+ hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
306
limit_mm_per_prompt={"image": len(image_urls)},
307
)
308
placeholders = "".join(f"Picture {i}: <img></img>\n"
@@ -452,7 +453,7 @@ def load_qwen2_5_vl(question, image_urls: List[str]) -> ModelRequestData:
453
model_example_map = {
454
"aria": load_aria,
455
"deepseek_vl_v2": load_deepseek_vl2,
455
- "h2ovl_chat": load_h2onvl,
456
+ "h2ovl_chat": load_h2ovl,
457
"idefics3": load_idefics3,
458
"internvl_chat": load_internvl,
459
"mllama": load_mllama,
tests/distributed/test_pipeline_parallel.py
+87
-84
@@ -6,6 +6,7 @@ WARNING: This test runs in both single-node (4 GPUs) and multi-node
6
all workers in a node other than the head node, which can cause the test
7
to fail.
8
"""
9
+import json
10
import os
11
from dataclasses import dataclass
12
from typing import List, Literal, NamedTuple, Optional
@@ -15,6 +16,7 @@ import pytest
16
from vllm.config import TaskOption
17
from vllm.logger import init_logger
18
19
+from ..models.registry import HF_EXAMPLE_MODELS
20
from ..utils import compare_two_settings, fork_new_process_for_each_test
21
22
logger = init_logger("test_pipeline_parallel")
@@ -31,10 +33,7 @@ class ParallelSetup(NamedTuple):
33
34
class PPTestOptions(NamedTuple):
35
multi_node_only: bool
34
- trust_remote_code: bool
35
- tokenizer_mode: Optional[str]
36
load_format: Optional[str] = None
37
- hf_overrides: Optional[str] = None
37
38
39
@dataclass
@@ -64,10 +63,7 @@ class PPTestSettings:
63
pp_base: int = 2,
64
multi_node_only: bool = False,
65
task: TaskOption = "auto",
67
- trust_remote_code: bool = False,
68
- tokenizer_mode: Optional[str] = None,
66
load_format: Optional[str] = None,
70
- hf_overrides: Optional[str] = None,
67
):
68
return PPTestSettings(
69
parallel_setups=[
@@ -97,10 +93,7 @@ class PPTestSettings:
93
vllm_major_versions=["0", "0", "1"],
94
task=task,
95
test_options=PPTestOptions(multi_node_only=multi_node_only,
100
- trust_remote_code=trust_remote_code,
101
- tokenizer_mode=tokenizer_mode,
102
- load_format=load_format,
103
- hf_overrides=hf_overrides),
96
+ load_format=load_format),
97
)
98
99
@staticmethod
@@ -110,10 +103,7 @@ class PPTestSettings:
103
pp_base: int = 2,
104
task: TaskOption = "auto",
105
multi_node_only: bool = False,
113
- trust_remote_code: bool = False,
114
- tokenizer_mode: Optional[str] = None,
106
load_format: Optional[str] = None,
116
- hf_overrides: Optional[str] = None,
107
):
108
return PPTestSettings(
109
parallel_setups=[
@@ -126,19 +116,16 @@ class PPTestSettings:
116
vllm_major_versions=["0"],
117
task=task,
118
test_options=PPTestOptions(multi_node_only=multi_node_only,
129
- trust_remote_code=trust_remote_code,
130
- tokenizer_mode=tokenizer_mode,
131
- load_format=load_format,
132
- hf_overrides=hf_overrides),
119
+ load_format=load_format),
120
)
121
135
- def iter_params(self, model_name: str):
122
+ def iter_params(self, model_id: str):
123
opts = self.test_options
124
125
for parallel_setup in self.parallel_setups:
126
for backend, vllm_major_version in zip(self.distributed_backends,
127
self.vllm_major_versions):
141
- yield (model_name, parallel_setup, backend, vllm_major_version,
128
+ yield (model_id, parallel_setup, backend, vllm_major_version,
129
self.task, opts)
130
131
@@ -150,16 +137,16 @@ TEXT_GENERATION_MODELS = {
137
# [Decoder-only]
138
# Uses Llama
139
# "BAAI/AquilaChat-7B": PPTestSettings.fast(),
153
- "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(tp_base=8, trust_remote_code=True), # noqa: E501
154
- "baichuan-inc/Baichuan-7B": PPTestSettings.fast(trust_remote_code=True),
155
- "baichuan-inc/Baichuan2-13B-Chat": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
140
+ "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(load_format="dummy"), # noqa: E501
141
+ "baichuan-inc/Baichuan-7B": PPTestSettings.fast(),
142
+ "baichuan-inc/Baichuan2-13B-Chat": PPTestSettings.fast(),
143
"bigscience/bloomz-1b1": PPTestSettings.fast(),
157
- "THUDM/chatglm3-6b": PPTestSettings.fast(trust_remote_code=True),
158
- "CohereForAI/c4ai-command-r-v01": PPTestSettings.fast(tp_base=2, trust_remote_code=True), # noqa: E501
159
- "databricks/dbrx-instruct": PPTestSettings.fast(tp_base=8),
160
- "Deci/DeciLM-7B-instruct": PPTestSettings.fast(trust_remote_code=True),
144
+ "THUDM/chatglm3-6b": PPTestSettings.fast(),
145
+ "CohereForAI/c4ai-command-r-v01": PPTestSettings.fast(load_format="dummy"),
146
+ "databricks/dbrx-instruct": PPTestSettings.fast(load_format="dummy"),
147
+ "Deci/DeciLM-7B-instruct": PPTestSettings.fast(),
148
"deepseek-ai/deepseek-llm-7b-chat": PPTestSettings.fast(),
162
- "deepseek-ai/DeepSeek-V2-Lite-Chat": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
149
+ "deepseek-ai/DeepSeek-V2-Lite-Chat": PPTestSettings.fast(),
150
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct": PPTestSettings.fast(),
151
"tiiuae/falcon-7b": PPTestSettings.fast(),
152
"google/gemma-2b": PPTestSettings.fast(),
@@ -172,36 +159,36 @@ TEXT_GENERATION_MODELS = {
159
"ibm/PowerMoE-3b": PPTestSettings.fast(),
160
# Uses Llama
161
# "internlm/internlm-chat-7b": PPTestSettings.fast(),
175
- "internlm/internlm2-chat-7b": PPTestSettings.fast(trust_remote_code=True),
162
+ "internlm/internlm2-chat-7b": PPTestSettings.fast(),
163
"inceptionai/jais-13b-chat": PPTestSettings.fast(),
164
"ai21labs/Jamba-tiny-dev": PPTestSettings.fast(),
165
"meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
179
- "openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(trust_remote_code=True),
180
- "openbmb/MiniCPM3-4B": PPTestSettings.fast(trust_remote_code=True),
166
+ "openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(),
167
+ "openbmb/MiniCPM3-4B": PPTestSettings.fast(),
168
# Uses Llama
169
# "mistralai/Mistral-7B-Instruct-v0.1": PPTestSettings.fast(),
170
"state-spaces/mamba-130m-hf": PPTestSettings.fast(),
184
- "mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(tp_base=4),
171
+ "mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(load_format="dummy"), # noqa: E501
172
"mosaicml/mpt-7b": PPTestSettings.fast(),
173
"nvidia/Minitron-8B-Base": PPTestSettings.fast(),
174
"allenai/OLMo-1B-hf": PPTestSettings.fast(),
175
"shanearora/OLMo-7B-1124-hf": PPTestSettings.fast(),
176
"allenai/OLMoE-1B-7B-0924-Instruct": PPTestSettings.fast(),
177
"facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
191
- "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
178
+ "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(),
179
"adept/persimmon-8b-chat": PPTestSettings.fast(),
180
"microsoft/phi-2": PPTestSettings.fast(),
194
- "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
195
- "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True, load_format="dummy", hf_overrides='{"num_hidden_layers": 4, "hidden_size": 512, "intermediate_size": 800, "num_attention_heads": 4, "num_key_value_heads": 1}'), # noqa: E501
196
- "Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
181
+ "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(),
182
+ "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(multi_node_only=True, load_format="dummy"), # noqa: E501
183
+ "Qwen/Qwen-7B-Chat": PPTestSettings.fast(),
184
"Qwen/Qwen2-7B-Instruct": PPTestSettings.fast(),
185
"Qwen/Qwen1.5-MoE-A2.7B-Chat": PPTestSettings.fast(),
186
"stabilityai/stablelm-3b-4e1t": PPTestSettings.fast(),
187
"bigcode/starcoder2-3b": PPTestSettings.fast(),
201
- "upstage/solar-pro-preview-instruct": PPTestSettings.fast(tp_base=2),
188
+ "upstage/solar-pro-preview-instruct": PPTestSettings.fast(load_format="dummy"), # noqa: E501
189
# FIXME: Cannot load tokenizer in latest transformers version.
190
# Need to use tokenizer from `meta-llama/Llama-2-7b-chat-hf`
204
- # "xverse/XVERSE-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
191
+ # "xverse/XVERSE-7B-Chat": PPTestSettings.fast(),
192
# [Encoder-only]
193
# TODO: Implement PP
194
# "facebook/bart-base": PPTestSettings.fast(),
@@ -211,7 +198,7 @@ EMBEDDING_MODELS = { # type: ignore[var-annotated]
198
# [Text-only]
199
"intfloat/e5-mistral-7b-instruct": PPTestSettings.fast(),
200
"BAAI/bge-multilingual-gemma2": PPTestSettings.fast(),
214
- "Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(tp_base=4, trust_remote_code=True), # noqa: E501
201
+ "Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(load_format="dummy"),
202
}
203
204
MULTIMODAL_MODELS = {
@@ -219,20 +206,20 @@ MULTIMODAL_MODELS = {
206
"Salesforce/blip2-opt-2.7b": PPTestSettings.fast(),
207
"facebook/chameleon-7b": PPTestSettings.fast(),
208
"adept/fuyu-8b": PPTestSettings.fast(),
222
- "THUDM/glm-4v-9b": PPTestSettings.fast(trust_remote_code=True),
223
- "OpenGVLab/InternVL2-1B": PPTestSettings.fast(trust_remote_code=True),
209
+ "THUDM/glm-4v-9b": PPTestSettings.fast(),
210
+ "OpenGVLab/InternVL2-1B": PPTestSettings.fast(),
211
"llava-hf/llava-1.5-7b-hf": PPTestSettings.fast(),
212
"llava-hf/llava-v1.6-mistral-7b-hf": PPTestSettings.fast(),
213
"llava-hf/LLaVA-NeXT-Video-7B-hf": PPTestSettings.fast(),
214
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf": PPTestSettings.fast(),
228
- "openbmb/MiniCPM-Llama3-V-2_5": PPTestSettings.fast(trust_remote_code=True),
229
- "allenai/Molmo-7B-D-0924": PPTestSettings.fast(trust_remote_code=True),
230
- "microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
231
- "mistralai/Pixtral-12B-2409": PPTestSettings.fast(tp_base=2, tokenizer_mode="mistral"), # noqa: E501
232
- "Qwen/Qwen-VL-Chat": PPTestSettings.fast(trust_remote_code=True),
215
+ "openbmb/MiniCPM-Llama3-V-2_5": PPTestSettings.fast(),
216
+ "allenai/Molmo-7B-D-0924": PPTestSettings.fast(),
217
+ "microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(),
218
+ "mistralai/Pixtral-12B-2409": PPTestSettings.fast(load_format="dummy"),
219
+ "Qwen/Qwen-VL-Chat": PPTestSettings.fast(),
220
"Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
221
"Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
235
- "fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
222
+ "fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(),
223
# [Encoder-decoder]
224
# TODO: Implement PP
225
# "meta-llama/Llama-3.2-11B-Vision-Instruct": PPTestSettings.fast(),
@@ -258,7 +245,7 @@ TEST_MODELS = [
245
246
247
def _compare_tp(
261
- model_name: str,
248
+ model_id: str,
249
parallel_setup: ParallelSetup,
250
distributed_backend: str,
251
vllm_major_version: str,
@@ -267,6 +254,7 @@ def _compare_tp(
254
num_gpus_available: int,
255
*,
256
method: Literal["generate", "encode"],
257
+ is_multimodal: bool,
258
):
259
(
260
tp_size,
@@ -274,13 +262,32 @@ def _compare_tp(
262
eager_mode,
263
chunked_prefill,
264
) = parallel_setup
277
- (
278
- multi_node_only,
279
- trust_remote_code,
280
- tokenizer_mode,
281
- load_format,
282
- hf_overrides,
283
- ) = test_options
265
+
266
+ multi_node_only, load_format = test_options
267
+
268
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
269
+ model_info.check_transformers_version(on_fail="skip")
270
+
271
+ trust_remote_code = model_info.trust_remote_code
272
+ tokenizer_mode = model_info.tokenizer_mode
273
+ hf_overrides = model_info.hf_overrides
274
+
275
+ if load_format == "dummy":
276
+ # Avoid OOM
277
+ text_overrides = {
278
+ "num_layers": 1,
279
+ "num_hidden_layers": 1,
280
+ "num_experts": 2,
281
+ "num_experts_per_tok": 2,
282
+ "num_local_experts": 2,
283
+ }
284
+
285
+ if is_multimodal:
286
+ hf_overrides.update({"text_config": text_overrides})
287
+ else:
288
+ hf_overrides.update(text_overrides)
289
+ else:
290
+ model_info.check_available_online(on_fail="skip")
291
292
if num_gpus_available < tp_size * pp_size:
293
pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
@@ -312,7 +319,7 @@ def _compare_tp(
319
if load_format:
320
common_args.extend(["--load-format", load_format])
321
if hf_overrides:
315
- common_args.extend(["--hf-overrides", hf_overrides])
322
+ common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
323
324
specific_case = tp_size == 2 and pp_size == 2 and chunked_prefill
325
if distributed_backend == "ray" and (vllm_major_version == "1"
@@ -355,11 +362,7 @@ def _compare_tp(
362
]
363
364
try:
358
- compare_two_settings(model_name,
359
- pp_args,
360
- tp_args,
361
- pp_env,
362
- method=method)
365
+ compare_two_settings(model_id, pp_args, tp_args, pp_env, method=method)
366
except Exception:
367
if pp_env is None:
368
raise
@@ -369,17 +372,16 @@ def _compare_tp(
372
373
374
@pytest.mark.parametrize(
372
- ("model_name", "parallel_setup", "distributed_backend",
373
- "vllm_major_version", "task", "test_options"),
375
+ ("model_id", "parallel_setup", "distributed_backend", "vllm_major_version",
376
+ "task", "test_options"),
377
[
375
- params for model_name, settings in TEXT_GENERATION_MODELS.items()
376
- for params in settings.iter_params(model_name)
377
- if model_name in TEST_MODELS
378
+ params for model_id, settings in TEXT_GENERATION_MODELS.items()
379
+ for params in settings.iter_params(model_id) if model_id in TEST_MODELS
380
],
381
)
382
@fork_new_process_for_each_test
383
def test_tp_language_generation(
382
- model_name: str,
384
+ model_id: str,
385
parallel_setup: ParallelSetup,
386
distributed_backend: str,
387
vllm_major_version: str,
@@ -387,28 +389,28 @@ def test_tp_language_generation(
389
test_options: PPTestOptions,
390
num_gpus_available,
391
):
390
- _compare_tp(model_name,
392
+ _compare_tp(model_id,
393
parallel_setup,
394
distributed_backend,
395
vllm_major_version,
396
task,
397
test_options,
398
num_gpus_available,
397
- method="generate")
399
+ method="generate",
400
+ is_multimodal=False)
401
402
403
@pytest.mark.parametrize(
401
- ("model_name", "parallel_setup", "distributed_backend",
402
- "vllm_major_version", "task", "test_options"),
404
+ ("model_id", "parallel_setup", "distributed_backend", "vllm_major_version",
405
+ "task", "test_options"),
406
[
404
- params for model_name, settings in EMBEDDING_MODELS.items()
405
- for params in settings.iter_params(model_name)
406
- if model_name in TEST_MODELS
407
+ params for model_id, settings in EMBEDDING_MODELS.items()
408
+ for params in settings.iter_params(model_id) if model_id in TEST_MODELS
409
],
410
)
411
@fork_new_process_for_each_test
412
def test_tp_language_embedding(
411
- model_name: str,
413
+ model_id: str,
414
parallel_setup: ParallelSetup,
415
distributed_backend: str,
416
vllm_major_version: str,
@@ -416,28 +418,28 @@ def test_tp_language_embedding(
418
test_options: PPTestOptions,
419
num_gpus_available,
420
):
419
- _compare_tp(model_name,
421
+ _compare_tp(model_id,
422
parallel_setup,
423
distributed_backend,
424
vllm_major_version,
425
task,
426
test_options,
427
num_gpus_available,
426
- method="encode")
428
+ method="encode",
429
+ is_multimodal=False)
430
431
432
@pytest.mark.parametrize(
430
- ("model_name", "parallel_setup", "distributed_backend",
431
- "vllm_major_version", "task", "test_options"),
433
+ ("model_id", "parallel_setup", "distributed_backend", "vllm_major_version",
434
+ "task", "test_options"),
435
[
433
- params for model_name, settings in MULTIMODAL_MODELS.items()
434
- for params in settings.iter_params(model_name)
435
- if model_name in TEST_MODELS
436
+ params for model_id, settings in MULTIMODAL_MODELS.items()
437
+ for params in settings.iter_params(model_id) if model_id in TEST_MODELS
438
],
439
)
440
@fork_new_process_for_each_test
441
def test_tp_multimodal_generation(
440
- model_name: str,
442
+ model_id: str,
443
parallel_setup: ParallelSetup,
444
distributed_backend: str,
445
vllm_major_version: str,
@@ -445,11 +447,12 @@ def test_tp_multimodal_generation(
447
test_options: PPTestOptions,
448
num_gpus_available,
449
):
448
- _compare_tp(model_name,
450
+ _compare_tp(model_id,
451
parallel_setup,
452
distributed_backend,
453
vllm_major_version,
454
task,
455
test_options,
456
num_gpus_available,
455
- method="generate")
457
+ method="generate",
458
+ is_multimodal=True)
tests/models/decoder_only/vision_language/test_models.py
+3
-8
@@ -155,10 +155,7 @@ VLM_TEST_SETTINGS = {
155
auto_cls=AutoModelForVision2Seq,
156
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
157
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
158
- marks=[pytest.mark.skipif(
159
- TRANSFORMERS_VERSION < "4.49.0",
160
- reason="HF model requires transformers>=4.49.0",
161
- ), pytest.mark.core_model, pytest.mark.cpu_model],
158
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
159
),
160
#### Extended model tests
161
"aria": VLMTestInfo(
@@ -215,7 +212,6 @@ VLM_TEST_SETTINGS = {
212
"cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
213
}),
214
multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
218
- vllm_runner_kwargs={"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]}}, # noqa: E501
215
patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
216
postprocess_inputs=model_utils.cast_dtype_post_processor("images"),
217
hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
@@ -240,7 +236,7 @@ VLM_TEST_SETTINGS = {
236
num_logprobs=10,
237
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
238
),
243
- "glm4": VLMTestInfo(
239
+ "glm4v": VLMTestInfo(
240
models=["THUDM/glm-4v-9b"],
241
test_type=VLMTestType.IMAGE,
242
prompt_formatter=identity,
@@ -351,7 +347,6 @@ VLM_TEST_SETTINGS = {
347
postprocess_inputs=model_utils.cast_dtype_post_processor(
348
"pixel_values"
349
),
354
- vllm_runner_kwargs={"hf_overrides": {"architectures": ["MantisForConditionalGeneration"]}}, # noqa: E501
350
get_stop_token_ids=lambda tok: [128009],
351
auto_cls=AutoModelForVision2Seq,
352
vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
@@ -437,7 +432,7 @@ VLM_TEST_SETTINGS = {
432
auto_cls=AutoModelForVision2Seq,
433
marks=[large_gpu_mark(min_gb=48)],
434
),
440
- "qwen": VLMTestInfo(
435
+ "qwen_vl": VLMTestInfo(
436
models=["Qwen/Qwen-VL"],
437
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
438
prompt_formatter=identity,
tests/models/decoder_only/vision_language/vlm_utils/core.py
+39
-23
@@ -4,12 +4,14 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
4
5
import torch
6
from PIL.Image import Image
7
-from transformers import AutoTokenizer, BatchEncoding, PreTrainedTokenizerBase
7
+from transformers import BatchEncoding
8
from transformers.models.auto.auto_factory import _BaseAutoModelClass
9
10
from vllm.config import TaskOption
11
+from vllm.transformers_utils.tokenizer import AnyTokenizer
12
13
from .....conftest import HfRunner, VllmRunner
14
+from ....registry import HF_EXAMPLE_MODELS
15
from .types import RunnerOutput
16
17
@@ -31,10 +33,8 @@ def run_test(
33
use_tokenizer_eos: bool,
34
postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],
35
comparator: Callable[..., None],
34
- get_stop_token_ids: Optional[Callable[[PreTrainedTokenizerBase],
35
- List[int]]],
36
+ get_stop_token_ids: Optional[Callable[[AnyTokenizer], list[int]]],
37
stop_str: Optional[List[str]],
37
- tokenizer_mode: str,
38
limit_mm_per_prompt: Dict[str, int],
39
vllm_runner_kwargs: Optional[Dict[str, Any]],
40
hf_model_kwargs: Optional[Dict[str, Any]],
@@ -48,7 +48,10 @@ def run_test(
48
"""Modality agnostic test test executor for comparing HF/vLLM outputs."""
49
# In the case of embeddings, vLLM takes separate input tensors
50
vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs
51
- tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
51
+
52
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
53
+ model_info.check_available_online(on_fail="skip")
54
+ model_info.check_transformers_version(on_fail="skip")
55
56
vllm_outputs_per_mm = []
57
hf_outputs_per_mm = []
@@ -57,17 +60,19 @@ def run_test(
60
# vLLM needs a fresh new process without cuda initialization.
61
# if we run HF first, the cuda initialization will be done and it
62
# will hurt multiprocessing backend with fork method (the default method).
60
- vllm_kwargs: Dict[str, Any] = {}
61
- if get_stop_token_ids is not None:
62
- vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
63
- if stop_str:
64
- vllm_kwargs["stop"] = stop_str
63
66
- if vllm_runner_kwargs is None:
67
- vllm_runner_kwargs = {}
64
+ vllm_runner_kwargs_: Dict[str, Any] = {}
65
+ if model_info.tokenizer:
66
+ vllm_runner_kwargs_["tokenizer"] = model_info.tokenizer
67
+ if model_info.tokenizer_mode:
68
+ vllm_runner_kwargs_["tokenizer_mode"] = model_info.tokenizer_mode
69
+ if model_info.hf_overrides:
70
+ vllm_runner_kwargs_["hf_overrides"] = model_info.hf_overrides
71
+
72
+ if vllm_runner_kwargs:
73
+ vllm_runner_kwargs_.update(vllm_runner_kwargs)
74
75
with vllm_runner(model,
70
- tokenizer_mode=tokenizer_mode,
76
max_model_len=max_model_len,
77
max_num_seqs=max_num_seqs,
78
dtype=dtype,
@@ -76,7 +81,15 @@ def run_test(
81
distributed_executor_backend=distributed_executor_backend,
82
enforce_eager=enforce_eager,
83
task=task,
79
- **vllm_runner_kwargs) as vllm_model:
84
+ **vllm_runner_kwargs_) as vllm_model:
85
+ tokenizer = vllm_model.model.get_tokenizer()
86
+
87
+ vllm_kwargs: Dict[str, Any] = {}
88
+ if get_stop_token_ids is not None:
89
+ vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
90
+ if stop_str:
91
+ vllm_kwargs["stop"] = stop_str
92
+
93
for prompts, media in vllm_inputs:
94
vllm_kwargs[runner_mm_key] = media
95
vllm_output = vllm_model.generate_greedy_logprobs(
@@ -93,16 +106,19 @@ def run_test(
106
if patch_hf_runner is not None:
107
hf_model = patch_hf_runner(hf_model)
108
96
- # Some models need to explicitly pass the eos_token_id off the tokenizer or
97
- # processor for a good comparison; currently assume processor/tokenizer
98
- # agree on the EOS, and pull it off the tokenizer if requested.
99
- hf_kwargs = {}
100
- if use_tokenizer_eos:
101
- hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
102
- if stop_str:
103
- hf_kwargs["stop_strings"] = stop_str
104
-
109
with hf_model, torch.no_grad():
110
+ tokenizer = hf_model.tokenizer
111
+
112
+ # Some models need to explicitly pass the eos_token_id off the tokenizer
113
+ # or processor for a good comparison;
114
+ # currently assume processor/tokenizer agree on the EOS, and pull it off
115
+ # the tokenizer if requested.
116
+ hf_kwargs = {}
117
+ if use_tokenizer_eos:
118
+ hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
119
+ if stop_str:
120
+ hf_kwargs["stop_strings"] = stop_str
121
+
122
for prompts, media in inputs:
123
hf_kwargs[runner_mm_key] = media
124
hf_output = hf_model.generate_greedy_logprobs_limit(
tests/models/decoder_only/vision_language/vlm_utils/types.py
+3
-7
@@ -8,12 +8,12 @@ from typing import (Any, Callable, Dict, Iterable, List, NamedTuple, Optional,
8
import torch
9
from PIL.Image import Image
10
from pytest import MarkDecorator
11
-from transformers import (AutoModelForCausalLM, BatchEncoding,
12
- PreTrainedTokenizerBase)
11
+from transformers import AutoModelForCausalLM, BatchEncoding
12
from transformers.models.auto.auto_factory import _BaseAutoModelClass
13
14
from vllm.config import TaskOption
15
from vllm.sequence import SampleLogprobs
16
+from vllm.transformers_utils.tokenizer import AnyTokenizer
17
from vllm.utils import identity
18
19
from .....conftest import IMAGE_ASSETS, HfRunner, ImageAsset, _ImageAssets
@@ -100,8 +100,7 @@ class VLMTestInfo(NamedTuple):
100
vllm_runner_kwargs: Optional[Dict[str, Any]] = None
101
102
# Optional callable which gets a list of token IDs from the model tokenizer
103
- get_stop_token_ids: Optional[Callable[[PreTrainedTokenizerBase],
104
- List[int]]] = None
103
+ get_stop_token_ids: Optional[Callable[[AnyTokenizer], list[int]]] = None
104
# Optional list of strings to stop generation, useful when stop tokens are
105
# not special tokens in the tokenizer
106
stop_str: Optional[List[str]] = None
@@ -156,8 +155,6 @@ class VLMTestInfo(NamedTuple):
155
156
marks: Optional[List[MarkDecorator]] = None
157
159
- tokenizer_mode: str = "auto"
160
-
158
def get_non_parametrized_runner_kwargs(self):
159
"""Returns a dictionary of expandable kwargs for items that are used
160
in all test types, which are NOT used when creating the parametrized
@@ -180,7 +177,6 @@ class VLMTestInfo(NamedTuple):
177
"hf_model_kwargs": self.hf_model_kwargs,
178
"stop_str": self.stop_str,
179
"patch_hf_runner": self.patch_hf_runner,
183
- "tokenizer_mode": self.tokenizer_mode
180
}
181
182
tests/models/registry.py
+22
-15
@@ -104,7 +104,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
104
trust_remote_code=True),
105
"BambaForCausalLM": _HfExamplesInfo("ibm-ai-platform/Bamba-9B"),
106
"BloomForCausalLM": _HfExamplesInfo("bigscience/bloomz-1b1"),
107
- # ChatGLMModel supports multimodal
107
+ "ChatGLMModel": _HfExamplesInfo("THUDM/chatglm3-6b",
108
+ trust_remote_code=True),
109
"CohereForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r-v01",
110
trust_remote_code=True),
111
"Cohere2ForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r7b-12-2024", # noqa: E501
@@ -138,7 +139,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
139
"InternLM3ForCausalLM": _HfExamplesInfo("internlm/internlm3-8b-instruct",
140
trust_remote_code=True),
141
"JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
141
- "JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini"),
142
+ "JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini",
143
+ extras={"tiny": "ai21labs/Jamba-tiny-dev"}), # noqa: E501
144
"LlamaForCausalLM": _HfExamplesInfo("meta-llama/Meta-Llama-3-8B"),
145
"LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
146
is_available_online=False),
@@ -167,7 +169,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
169
trust_remote_code=True),
170
"PhiMoEForCausalLM": _HfExamplesInfo("microsoft/Phi-3.5-MoE-instruct",
171
trust_remote_code=True),
170
- # QWenLMHeadModel supports multimodal
172
+ "QWenLMHeadModel": _HfExamplesInfo("Qwen/Qwen-7B-Chat",
173
+ trust_remote_code=True),
174
"Qwen2ForCausalLM": _HfExamplesInfo("Qwen/Qwen2-7B-Instruct"),
175
"Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
176
"RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b",
@@ -232,18 +235,19 @@ _MULTIMODAL_EXAMPLE_MODELS = {
235
"AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
236
"Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b"), # noqa: E501
237
"ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501
235
- "ChatGLMModel": _HfExamplesInfo("THUDM/glm-4v-9b",
236
- extras={"text_only": "THUDM/chatglm3-6b"},
237
- trust_remote_code=True),
238
- "ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
239
- is_available_online=False),
238
"DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny", # noqa: E501
239
hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]}), # noqa: E501
240
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
243
- "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
241
+ "GLM4VForCausalLM": _HfExamplesInfo("THUDM/glm-4v-9b",
242
+ trust_remote_code=True,
243
+ hf_overrides={"architectures": ["GLM4VForCausalLM"]}), # noqa: E501
244
+ "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m",
245
+ extras={"2b": "h2oai/h2ovl-mississippi-2b"}), # noqa: E501
246
"InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
247
+ extras={"2B": "OpenGVLab/InternVL2-2B"}, # noqa: E501
248
trust_remote_code=True),
246
- "Idefics3ForConditionalGeneration": _HfExamplesInfo("HuggingFaceM4/Idefics3-8B-Llama3"), # noqa: E501
249
+ "Idefics3ForConditionalGeneration": _HfExamplesInfo("HuggingFaceM4/Idefics3-8B-Llama3", # noqa: E501
250
+ {"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"}), # noqa: E501
251
"LlavaForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-1.5-7b-hf",
252
extras={"mistral": "mistral-community/pixtral-12b"}), # noqa: E501
253
"LlavaNextForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-v1.6-mistral-7b-hf"), # noqa: E501
@@ -253,21 +257,24 @@ _MULTIMODAL_EXAMPLE_MODELS = {
257
hf_overrides={"architectures": ["MantisForConditionalGeneration"]}), # noqa: E501
258
"MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6",
259
trust_remote_code=True),
256
- "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-V-2_6",
260
+ "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
261
+ extras={"2.6": "openbmb/MiniCPM-V-2_6"}, # noqa: E501
262
trust_remote_code=True),
263
"MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
264
extras={"olmo": "allenai/Molmo-7B-O-0924"}, # noqa: E501
265
trust_remote_code=True),
266
"NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B",
267
trust_remote_code=True),
263
- "PaliGemmaForConditionalGeneration": _HfExamplesInfo("google/paligemma-3b-pt-224"), # noqa: E501
268
+ "PaliGemmaForConditionalGeneration": _HfExamplesInfo("google/paligemma-3b-mix-224", # noqa: E501
269
+ extras={"v2": "google/paligemma2-3b-ft-docci-448"}), # noqa: E501
270
"Phi3VForCausalLM": _HfExamplesInfo("microsoft/Phi-3-vision-128k-instruct",
271
trust_remote_code=True),
272
"PixtralForConditionalGeneration": _HfExamplesInfo("mistralai/Pixtral-12B-2409", # noqa: E501
273
tokenizer_mode="mistral"),
268
- "QWenLMHeadModel": _HfExamplesInfo("Qwen/Qwen-VL-Chat",
269
- extras={"text_only": "Qwen/Qwen-7B-Chat"}, # noqa: E501
270
- trust_remote_code=True),
274
+ "QwenVLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen-VL",
275
+ extras={"chat": "Qwen/Qwen-VL-Chat"}, # noqa: E501
276
+ trust_remote_code=True,
277
+ hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]}), # noqa: E501
278
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
279
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
280
"Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
tests/models/test_initialization.py
+1
-2
@@ -18,8 +18,7 @@ def test_can_initialize(model_arch):
18
19
# Avoid OOM
20
def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
21
- if hf_config.model_type == "deepseek_vl_v2":
22
- hf_config.update({"architectures": ["DeepseekVLV2ForCausalLM"]})
21
+ hf_config.update(model_info.hf_overrides)
22
23
if hasattr(hf_config, "text_config"):
24
text_config: PretrainedConfig = hf_config.text_config
vllm/model_executor/models/chatglm.py
+58
-362
@@ -1,20 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
-
2
# Adapted from
4
-# https://github.com/THUDM/CogAgent
5
-"""Inference-only CogAgent model compatible with THUDM weights."""
6
-from argparse import Namespace
7
-from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
8
- Union)
3
+# https://github.com/THUDM/ChatGLM2-6B
4
+"""Inference-only ChatGLM model compatible with THUDM weights."""
5
+from typing import Iterable, List, Optional, Set, Tuple, Union
6
7
import torch
8
from torch import nn
9
from torch.nn import LayerNorm
13
-from torchvision import transforms
14
-from torchvision.transforms import InterpolationMode
15
-from transformers import PreTrainedTokenizer, TensorType
16
-from transformers.image_utils import ImageInput
17
-from transformers.tokenization_utils_base import TextInput
10
11
from vllm.attention import Attention, AttentionMetadata
12
from vllm.config import CacheConfig, VllmConfig
@@ -31,204 +23,14 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
ParallelLMHead, VocabParallelEmbedding)
25
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34
-from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
35
-from vllm.model_executor.models.module_mapping import MultiModelKeys
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
37
-from vllm.multimodal import MULTIMODAL_REGISTRY
38
-from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
39
-from vllm.multimodal.parse import MultiModalDataItems
40
-from vllm.multimodal.processing import (BaseMultiModalProcessor,
41
- BaseProcessingInfo, BatchFeature,
42
- MultiModalFieldConfig,
43
- PromptReplacement)
44
-from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
27
from vllm.sequence import IntermediateTensors
28
from vllm.transformers_utils.configs import ChatGLMConfig
29
48
-from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
30
+from .interfaces import SupportsLoRA, SupportsPP
31
from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
32
make_empty_intermediate_tensors_factory, make_layers,
51
- maybe_prefix, merge_multimodal_embeddings)
52
-
53
-
54
-class GLMImagePixelInputs(TypedDict):
55
- pixel_values: torch.Tensor
56
- """Shape: `(batch_size, num_channels, height, width)`"""
57
-
58
-
59
-class GLM4VProcessor:
60
- """
61
- This model doesn't define its own HF processor,
62
- so we implement our own one here.
63
-
64
- """
65
-
66
- def __init__(
67
- self,
68
- config: ChatGLMConfig,
69
- tokenizer: PreTrainedTokenizer,
70
- ) -> None:
71
- super().__init__()
72
-
73
- self.config = config
74
- self.tokenizer = tokenizer
75
-
76
- if vision_config := getattr(config, "vision_config", None):
77
- image_size = vision_config["image_size"]
78
-
79
- self.image_transform = transforms.Compose([
80
- transforms.Resize(
81
- (image_size, image_size),
82
- interpolation=InterpolationMode.BICUBIC,
83
- ),
84
- transforms.ToTensor(),
85
- transforms.Normalize(
86
- mean=(0.48145466, 0.4578275, 0.40821073),
87
- std=(0.26862954, 0.26130258, 0.27577711),
88
- ),
89
- ])
90
- else:
91
- self.image_transform = None
92
-
93
- def __call__(
94
- self,
95
- text: Optional[Union[TextInput, list[TextInput]]] = None,
96
- images: Optional[Union[ImageInput, list[ImageInput]]] = None,
97
- return_tensors: Optional[Union[str, TensorType]] = None,
98
- ) -> BatchFeature:
99
- if text is None:
100
- text = []
101
- if not isinstance(text, list):
102
- text = [text]
103
- if images is None:
104
- images = []
105
- if not isinstance(images, list):
106
- images = [images]
107
- text_inputs = self.tokenizer(text)
108
- if len(images) == 0:
109
- image_inputs = {}
110
- else:
111
- if self.image_transform is None:
112
- raise ValueError("This model does not support image inputs")
113
-
114
- pixel_values = [self.image_transform(image) for image in images]
115
- image_inputs = {"pixel_values": torch.stack(pixel_values)}
116
-
117
- return BatchFeature(
118
- {
119
- **text_inputs,
120
- **image_inputs,
121
- },
122
- tensor_type=return_tensors,
123
- )
124
-
125
-
126
-class GLM4VProcessingInfo(BaseProcessingInfo):
127
-
128
- def get_tokenizer(self):
129
- tokenizer = self.ctx.tokenizer
130
- assert isinstance(tokenizer, PreTrainedTokenizer)
131
- return tokenizer
132
-
133
- def get_hf_config(self):
134
- return self.ctx.get_hf_config(ChatGLMConfig)
135
-
136
- def get_hf_processor(self) -> GLM4VProcessor:
137
- return GLM4VProcessor(
138
- self.get_hf_config(),
139
- self.get_tokenizer(),
140
- )
141
-
142
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
143
- return {"image": 1}
144
-
145
- def get_mm_max_tokens_per_item(
146
- self,
147
- seq_len: int,
148
- mm_counts: Mapping[str, int],
149
- ) -> Mapping[str, int]:
150
- return {"image": self.get_num_image_feature_tokens()}
151
-
152
- def get_num_image_tokens(self) -> int:
153
- hf_config = self.get_hf_config()
154
- if not (vision_config := getattr(hf_config, "vision_config", None)):
155
- return 0
156
-
157
- image_size = vision_config["image_size"]
158
- patch_size = vision_config["patch_size"]
159
- grid_length = image_size // patch_size // 2
160
- return grid_length * grid_length
161
-
162
- def get_num_image_feature_tokens(self) -> int:
163
- # EVA2CLIPModel has embeddings for boi and eoi tokens as well
164
- return self.get_num_image_tokens() + 2
165
-
166
-
167
-class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
168
-
169
- def get_dummy_processor_inputs(
170
- self,
171
- seq_len: int,
172
- mm_counts: Mapping[str, int],
173
- ) -> ProcessorInputs:
174
- hf_config = self.info.get_hf_config()
175
- if not (vision_config := getattr(hf_config, "vision_config", None)):
176
- return ProcessorInputs(prompt_text="", mm_data={})
177
-
178
- target_width = target_height = vision_config["image_size"]
179
- num_images = mm_counts.get("image", 0)
180
-
181
- mm_data = {
182
- "image":
183
- self._get_dummy_images(width=target_width,
184
- height=target_height,
185
- num_images=num_images)
186
- }
187
-
188
- base_text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
189
-
190
- return ProcessorInputs(
191
- prompt_text=base_text * num_images,
192
- mm_data=mm_data,
193
- )
194
-
195
-
196
-class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
197
-
198
- def _get_mm_fields_config(
199
- self,
200
- hf_inputs: BatchFeature,
201
- hf_processor_mm_kwargs: Mapping[str, object],
202
- ) -> Mapping[str, MultiModalFieldConfig]:
203
- return dict(pixel_values=MultiModalFieldConfig.batched("image"))
204
-
205
- def _get_prompt_replacements(
206
- self,
207
- mm_items: MultiModalDataItems,
208
- hf_processor_mm_kwargs: Mapping[str, object],
209
- out_mm_kwargs: MultiModalKwargs,
210
- ) -> list[PromptReplacement]:
211
- hf_config = self.info.get_hf_config()
212
- if not hasattr(hf_config, "vision_config"):
213
- return []
214
-
215
- boi_token_id = hf_config.boi_token_id
216
- image_token_id = hf_config.pad_token_id
217
- eoi_token_id = hf_config.eoi_token_id
218
-
219
- def get_replacement(item_idx: int):
220
- num_image_tokens = self.info.get_num_image_tokens()
221
- image_tokens = [image_token_id] * num_image_tokens
222
-
223
- return [boi_token_id] + image_tokens + [eoi_token_id]
224
-
225
- return [
226
- PromptReplacement(
227
- modality="image",
228
- target=[boi_token_id, image_token_id, eoi_token_id],
229
- replacement=get_replacement,
230
- ),
231
- ]
33
+ maybe_prefix)
34
35
36
class GLMAttention(nn.Module):
@@ -489,7 +291,7 @@ class GLMTransformer(nn.Module):
291
position_ids: torch.Tensor,
292
kv_caches: List[torch.Tensor],
293
attn_metadata: AttentionMetadata,
492
- ) -> torch.Tensor:
294
+ ) -> Union[torch.Tensor, IntermediateTensors]:
295
for i in range(self.start_layer, self.end_layer):
296
layer = self.layers[i]
297
hidden_states = layer(
@@ -498,8 +300,12 @@ class GLMTransformer(nn.Module):
300
kv_cache=kv_caches[i - self.start_layer],
301
attn_metadata=attn_metadata,
302
)
303
+
304
+ if not get_pp_group().is_last_rank:
305
+ return IntermediateTensors({"hidden_states": hidden_states})
306
+
307
# Final layer norm.
502
- if get_pp_group().is_last_rank and self.post_layer_norm:
308
+ if self.post_layer_norm:
309
hidden_states = self.final_layernorm(hidden_states)
310
311
return hidden_states
@@ -534,61 +340,11 @@ class ChatGLMModel(nn.Module):
340
quant_config=quant_config,
341
prefix=f"{prefix}.output_layer")
342
537
- vision_config_flag = getattr(config, 'vision_config', None)
538
- if vision_config_flag is not None:
539
- self.vision_config = Namespace(**config.vision_config)
540
- self.vision = EVA2CLIPModel(self.config,
541
- quant_config,
542
- prefix=f"{prefix}.vision")
543
- else:
544
- self.vision = None
545
-
343
self.make_empty_intermediate_tensors = (
344
self.encoder.make_empty_intermediate_tensors)
345
549
- def _parse_and_validate_image_input(
550
- self, **kwargs: object) -> GLMImagePixelInputs:
551
-
552
- pixel_values = kwargs.pop("pixel_values", None)
553
- if pixel_values is not None and self.vision is not None:
554
- if isinstance(pixel_values, torch.Tensor):
555
- if pixel_values.ndim > 2:
556
- pixel_values = torch.concat(list(pixel_values))
557
- elif isinstance(pixel_values, list):
558
- return torch.concat(pixel_values)
559
- else:
560
- raise TypeError("""pixel_values must be a torch.Tensor
561
- or a list of torch.Tensor
562
- """)
563
- return GLMImagePixelInputs(pixel_values=pixel_values)
564
-
565
- def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
566
- image_input = self._parse_and_validate_image_input(**kwargs)
567
- if image_input["pixel_values"] is None:
568
- return None
569
- pixel_values = image_input["pixel_values"].to(
570
- dtype=self.config.torch_dtype)
571
- vision_embeddings = self.vision(pixel_values)
572
- return vision_embeddings
573
-
574
- def get_input_embeddings(
575
- self,
576
- input_ids: torch.Tensor,
577
- multimodal_embeddings: Optional[NestedTensors] = None,
578
- ) -> torch.Tensor:
579
- inputs_embeds = self.embedding(input_ids)
580
- if multimodal_embeddings is not None:
581
- inputs_embeds = merge_multimodal_embeddings(
582
- input_ids=input_ids,
583
- inputs_embeds=inputs_embeds,
584
- multimodal_embeddings=multimodal_embeddings,
585
- placeholder_token_id=[
586
- self.config.boi_token_id,
587
- self.config.pad_token_id,
588
- self.config.eoi_token_id,
589
- ],
590
- )
591
- return inputs_embeds
346
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
347
+ return self.embedding(input_ids)
348
349
def forward(
350
self,
@@ -599,26 +355,24 @@ class ChatGLMModel(nn.Module):
355
intermediate_tensors: Optional[IntermediateTensors] = None,
356
inputs_embeds: Optional[torch.Tensor] = None,
357
**kwargs: object,
602
- ) -> torch.Tensor:
358
+ ) -> Union[torch.Tensor, IntermediateTensors]:
359
+ if get_pp_group().is_first_rank:
360
+ if inputs_embeds is not None:
361
+ hidden_states = inputs_embeds
362
+ else:
363
+ hidden_states = self.get_input_embeddings(input_ids)
364
+ else:
365
+ assert intermediate_tensors is not None
366
+ hidden_states = intermediate_tensors["hidden_states"]
367
604
- # NOTE: In v1, inputs_embeds is always generated at model runner, this
605
- # condition is for v0 compatibility.
606
- if intermediate_tensors is not None:
607
- inputs_embeds = intermediate_tensors["hidden_states"]
608
- elif inputs_embeds is None:
609
- vision_embeddings = self.get_multimodal_embeddings(**kwargs)
610
- inputs_embeds = self.get_input_embeddings(input_ids,
611
- vision_embeddings)
368
# Run encoder.
369
hidden_states = self.encoder(
614
- hidden_states=inputs_embeds,
370
+ hidden_states=hidden_states,
371
position_ids=positions,
372
kv_caches=kv_caches,
373
attn_metadata=attn_metadata,
374
)
375
620
- if not get_pp_group().is_last_rank:
621
- return IntermediateTensors({"hidden_states": hidden_states})
376
return hidden_states
377
378
def load_weights(self, weights: Iterable[Tuple[str,
@@ -660,12 +414,18 @@ class ChatGLMModel(nn.Module):
414
return loaded_params
415
416
663
-class ChatGLMBaseModel(nn.Module, SupportsLoRA, SupportsPP):
417
+class ChatGLMBaseModel(nn.Module):
418
419
hf_to_vllm_mapper = WeightsMapper(
420
orig_to_new_substr={".word_embeddings": ""}, )
421
668
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
422
+ def __init__(
423
+ self,
424
+ *,
425
+ vllm_config: VllmConfig,
426
+ prefix: str = "",
427
+ transformer_type: type[ChatGLMModel] = ChatGLMModel,
428
+ ) -> None:
429
super().__init__()
430
config = vllm_config.model_config.hf_config
431
quant_config = vllm_config.quant_config
@@ -678,27 +438,17 @@ class ChatGLMBaseModel(nn.Module, SupportsLoRA, SupportsPP):
438
self.quant_config = quant_config
439
self.max_position_embeddings = getattr(config, "max_sequence_length",
440
8192)
681
- self.transformer = ChatGLMModel(vllm_config=vllm_config,
682
- prefix=maybe_prefix(
683
- prefix, "transformer"))
441
+ self.transformer = transformer_type(vllm_config=vllm_config,
442
+ prefix=maybe_prefix(
443
+ prefix, "transformer"))
444
if self.config.tie_word_embeddings:
445
self.transformer.output_layer.weight = (
446
self.transformer.embedding.weight)
447
self.lm_head = self.transformer.output_layer
448
self.logits_processor = LogitsProcessor(config.padded_vocab_size)
449
self.sampler = get_sampler()
690
-
691
- def forward(self,
692
- input_ids: torch.Tensor,
693
- positions: torch.Tensor,
694
- kv_caches: List[torch.Tensor],
695
- attn_metadata: AttentionMetadata,
696
- intermediate_tensors: Optional[IntermediateTensors] = None,
697
- **kwargs) -> torch.Tensor:
698
- hidden_states = self.transformer(input_ids, positions, kv_caches,
699
- attn_metadata, intermediate_tensors,
700
- **kwargs)
701
- return hidden_states
450
+ self.make_empty_intermediate_tensors = (
451
+ self.transformer.make_empty_intermediate_tensors)
452
453
def compute_logits(
454
self,
@@ -722,7 +472,7 @@ class ChatGLMBaseModel(nn.Module, SupportsLoRA, SupportsPP):
472
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
473
474
725
-class ChatGLM(ChatGLMBaseModel):
475
+class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP):
476
packed_modules_mapping = {
477
"query_key_value": ["query_key_value"],
478
"dense_h_to_4h": ["dense_h_to_4h"]
@@ -738,82 +488,28 @@ class ChatGLM(ChatGLMBaseModel):
488
embedding_modules = {}
489
embedding_padding_modules = []
490
491
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
492
+ config = vllm_config.model_config.hf_config
493
+ if hasattr(config, "vision_config"):
494
+ hf_overrides = {"architectures": ["GLM4VForCausalLM"]}
495
+ raise RuntimeError(
496
+ "The configuration of this model indicates that it supports "
497
+ "vision inputs, but you instantiated the text-only version "
498
+ "of this model. Please use the vision model by setting "
499
+ f"`--hf-overrides {hf_overrides!r}`")
500
742
-class ChatGLMV(ChatGLMBaseModel, SupportsMultiModal):
743
-
744
- packed_modules_mapping = {
745
- "query_key_value": ["query_key_value"],
746
- "dense_h_to_4h": ["dense_h_to_4h"],
747
- "merged_proj": ["gate_proj", "dense_h_to_4h"]
748
- }
749
- # LoRA specific attributes
750
- supported_lora_modules = [
751
- "query_key_value",
752
- "dense",
753
- "dense_h_to_4h",
754
- "dense_4h_to_h",
755
- # vision
756
- "fc1",
757
- "fc2",
758
- "merged_proj",
759
- "linear_proj"
760
- ]
761
-
762
- embedding_modules = {}
763
- embedding_padding_modules = []
764
-
765
- def get_mm_mapping(self) -> MultiModelKeys:
766
- """
767
- Get the module prefix in multimodal models
768
- """
769
- return MultiModelKeys.from_string_field(
770
- language_model="transformer.encoder",
771
- connector="transformer.vision.linear_proj",
772
- tower_model="transformer.vision.transformer")
773
-
774
- def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
775
- return self.transformer.get_multimodal_embeddings(**kwargs)
501
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
502
777
- def get_input_embeddings(
503
+ def forward(
504
self,
505
input_ids: torch.Tensor,
780
- multimodal_embeddings: Optional[NestedTensors] = None,
781
- ) -> torch.Tensor:
782
- return self.transformer.get_input_embeddings(input_ids,
783
- multimodal_embeddings)
784
-
785
-
786
-@MULTIMODAL_REGISTRY.register_processor(GLM4VMultiModalProcessor,
787
- info=GLM4VProcessingInfo,
788
- dummy_inputs=GLM4VDummyInputsBuilder)
789
-class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
790
- SupportsMultiModal):
791
- # Ensure that the LoRA support check passes when the class is not
792
- # initialized, but set all these attributes to empty.
793
- # These will be updated when an instance class is selected
794
- packed_modules_mapping = {}
795
- supported_lora_modules = []
796
- embedding_modules = {}
797
- embedding_padding_modules = []
798
-
799
- def __new__(
800
- cls,
801
- vllm_config: VllmConfig,
802
- prefix: str = "",
803
- ) -> None:
804
- config = vllm_config.model_config.hf_config
805
-
806
- # Initialize VL
807
- if hasattr(config, "vision_config"): # noqa: SIM108
808
- instance_cls = ChatGLMV
809
- # Initialize LLM
810
- else:
811
- instance_cls = ChatGLM
812
-
813
- # quant_config references base class members,
814
- # so update values before init is called
815
- cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
816
- cls.supported_lora_modules += instance_cls.supported_lora_modules
817
- cls.embedding_modules.update(instance_cls.embedding_modules)
818
- cls.embedding_padding_modules += instance_cls.embedding_padding_modules
819
- return instance_cls(vllm_config=vllm_config, prefix=prefix)
506
+ positions: torch.Tensor,
507
+ kv_caches: List[torch.Tensor],
508
+ attn_metadata: AttentionMetadata,
509
+ intermediate_tensors: Optional[IntermediateTensors] = None,
510
+ inputs_embeds: Optional[torch.Tensor] = None,
511
+ ) -> Union[torch.Tensor, IntermediateTensors]:
512
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
513
+ attn_metadata, intermediate_tensors,
514
+ inputs_embeds)
515
+ return hidden_states
vllm/model_executor/models/glm4_vision_encoder.py
deleted
-312
@@ -1,312 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
3
-# Adapted from
4
-# https://github.com/THUDM/GLM-4
5
-"""Inference-only GLM-4v model visual encoder compatible with THUDM weights."""
6
-from argparse import Namespace
7
-from typing import Optional
8
-
9
-import torch
10
-from torch import nn
11
-from torch.nn import LayerNorm
12
-
13
-from vllm.attention.layer import MultiHeadAttention
14
-from vllm.distributed import get_tensor_model_parallel_world_size
15
-from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
16
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
- MergedColumnParallelLinear,
18
- QKVParallelLinear,
19
- ReplicatedLinear,
20
- RowParallelLinear)
21
-from vllm.model_executor.layers.quantization.base_config import (
22
- QuantizationConfig)
23
-
24
-
25
-class PatchEmbedding(nn.Module):
26
-
27
- def __init__(self, config):
28
- super().__init__()
29
- self.proj = nn.Conv2d(config.in_channels,
30
- config.hidden_size,
31
- kernel_size=config.patch_size,
32
- stride=config.patch_size)
33
- self.cls_embedding = nn.Parameter(torch.zeros(1, config.hidden_size))
34
- self.position_embedding = nn.Embedding(config.num_positions,
35
- config.hidden_size)
36
-
37
- def forward(self, images: torch.Tensor) -> torch.Tensor:
38
- """
39
- Parameters:
40
- images : torch.Tensor
41
- Input image tensor with shape (B, C, H, W)
42
-
43
- Returns:
44
- torch.Tensor
45
- Transformed tensor with shape (B, L, D)
46
- """
47
- images = images.to(device=self.proj.weight.device,
48
- dtype=self.proj.weight.dtype)
49
- x = self.proj(images)
50
- x = x.flatten(2).transpose(1, 2)
51
- cls_token = self.cls_embedding.expand(x.shape[0], -1, -1)
52
- x = torch.cat((cls_token, x), dim=1)
53
- x += self.position_embedding.weight.unsqueeze(0)
54
- return x
55
-
56
-
57
-class Attention(nn.Module):
58
-
59
- def __init__(
60
- self,
61
- config,
62
- quant_config: Optional[QuantizationConfig] = None,
63
- prefix: str = '',
64
- ):
65
- super().__init__()
66
- self.hidden_size = config.hidden_size
67
- self.tp_size = get_tensor_model_parallel_world_size()
68
- self.num_heads_per_rank = config.num_heads // self.tp_size
69
- self.head_dim = config.hidden_size // config.num_heads
70
- self.scale = self.head_dim**-0.5
71
-
72
- self.query_key_value = QKVParallelLinear(
73
- config.hidden_size,
74
- self.head_dim,
75
- config.num_heads,
76
- quant_config=quant_config,
77
- prefix=f"{prefix}.query_key_value",
78
- )
79
- self.dense = RowParallelLinear(
80
- config.hidden_size,
81
- config.hidden_size,
82
- quant_config=quant_config,
83
- prefix=f"{prefix}.dense",
84
- )
85
-
86
- self.attn = MultiHeadAttention(self.num_heads_per_rank, self.head_dim,
87
- self.scale)
88
- self.output_dropout = torch.nn.Dropout(config.dropout_prob)
89
-
90
- def forward(self, x: torch.Tensor) -> torch.Tensor:
91
- qkv, _ = self.query_key_value(x) # B, L, 3 * H * D
92
- q, k, v = qkv.chunk(3, dim=-1)
93
-
94
- out = self.attn(q, k, v)
95
- output, _ = self.dense(out)
96
- output = self.output_dropout(output)
97
- return output
98
-
99
-
100
-class MLP(nn.Module):
101
-
102
- def __init__(
103
- self,
104
- config,
105
- quant_config: Optional[QuantizationConfig] = None,
106
- prefix: str = '',
107
- ):
108
- super().__init__()
109
- self.config = config
110
- self.activation_fn = get_act_fn(config.hidden_act)
111
- self.fc1 = ColumnParallelLinear(
112
- config.hidden_size,
113
- config.intermediate_size,
114
- quant_config=quant_config,
115
- prefix=f"{prefix}.fc1",
116
- )
117
- self.fc2 = RowParallelLinear(
118
- config.intermediate_size,
119
- config.hidden_size,
120
- quant_config=quant_config,
121
- prefix=f"{prefix}.fc2",
122
- )
123
-
124
- def forward(self, x: torch.Tensor) -> torch.Tensor:
125
- x, _ = self.fc1(x)
126
- x = self.activation_fn(x)
127
- x, _ = self.fc2(x)
128
- return x
129
-
130
-
131
-class TransformerLayer(nn.Module):
132
-
133
- def __init__(
134
- self,
135
- config,
136
- quant_config: Optional[QuantizationConfig] = None,
137
- prefix: str = '',
138
- ):
139
- super().__init__()
140
- self.input_layernorm = LayerNorm(config.hidden_size,
141
- eps=config.layer_norm_eps)
142
- self.attention = Attention(config,
143
- quant_config=quant_config,
144
- prefix=f"{prefix}.attention")
145
- self.mlp = MLP(config,
146
- quant_config=quant_config,
147
- prefix=f"{prefix}.mlp")
148
- self.post_attention_layernorm = LayerNorm(config.hidden_size,
149
- eps=config.layer_norm_eps)
150
-
151
- def forward(self, hidden_states):
152
- attention_input = hidden_states
153
- attention_output = self.input_layernorm(
154
- self.attention(attention_input))
155
- hidden_states = attention_input + attention_output
156
- mlp_input = hidden_states
157
- mlp_output = self.post_attention_layernorm(self.mlp(mlp_input))
158
- output = mlp_input + mlp_output
159
- return output
160
-
161
-
162
-class Transformer(nn.Module):
163
-
164
- def __init__(
165
- self,
166
- config,
167
- quant_config: Optional[QuantizationConfig] = None,
168
- prefix: str = '',
169
- ):
170
- super().__init__()
171
- self.layers = nn.ModuleList([
172
- TransformerLayer(config,
173
- quant_config=quant_config,
174
- prefix=f"{prefix}.layers.{layer_idx}")
175
- for layer_idx in range(config.num_hidden_layers)
176
- ])
177
-
178
- def forward(self, hidden_states):
179
- for layer_module in self.layers:
180
- hidden_states = layer_module(hidden_states)
181
- return hidden_states
182
-
183
-
184
-class GLU(nn.Module):
185
-
186
- def __init__(
187
- self,
188
- config,
189
- in_features,
190
- quant_config: Optional[QuantizationConfig] = None,
191
- prefix: str = '',
192
- ):
193
- """
194
- The original implementation is the same as:
195
- ```python
196
- self.dense_h_to_4h = ColumnParallelLinear(
197
- config.hidden_size,
198
- config.ffn_hidden_size,
199
- bias=False,
200
- quant_config=quant_config
201
- )
202
-
203
- self.gate_proj = ColumnParallelLinear(
204
- config.hidden_size,
205
- config.ffn_hidden_size,
206
- bias=False,
207
- quant_config=quant_config
208
- )
209
- ```
210
- ```
211
- gate_proj_output, _ = self.gate_proj(x)
212
- dense_h_to_4h_output, _ = self.dense_h_to_4h(x)
213
- x = torch.cat([gate_proj_output, dense_h_to_4h_output], dim=-1)
214
- ```
215
-
216
- We merge two ColumnParallelLinear into one MergedColumnParallelLinear:
217
- ```
218
- self.merged_proj = MergedColumnParallelLinear(
219
- config.hidden_size,
220
- [config.ffn_hidden_size] * 2,
221
- bias=False,
222
- quant_config=quant_config
223
- )
224
- ```
225
- ```
226
- x, _ = self.merged_proj(x)
227
- ```
228
- """
229
- super().__init__()
230
- self.linear_proj = ReplicatedLinear(in_features,
231
- config.hidden_size,
232
- bias=False,
233
- quant_config=quant_config,
234
- prefix=f"{prefix}.linear_proj")
235
- self.norm1 = nn.LayerNorm(config.hidden_size)
236
- self.act1 = nn.GELU()
237
- self.act2 = SiluAndMul()
238
-
239
- self.merged_proj = MergedColumnParallelLinear(
240
- config.hidden_size, [config.ffn_hidden_size] * 2,
241
- bias=False,
242
- quant_config=quant_config,
243
- prefix=f"{prefix}.merged_proj")
244
-
245
- self.dense_4h_to_h = RowParallelLinear(
246
- config.ffn_hidden_size,
247
- config.hidden_size,
248
- bias=False,
249
- quant_config=quant_config,
250
- prefix=f"{prefix}.dense_4h_to_h")
251
-
252
- def forward(self, x):
253
- x, _ = self.linear_proj(x)
254
- x = self.act1(self.norm1(x))
255
- x, _ = self.merged_proj(x)
256
- x = self.act2(x)
257
- x, _ = self.dense_4h_to_h(x)
258
- return x
259
-
260
-
261
-class EVA2CLIPModel(nn.Module):
262
-
263
- def __init__(
264
- self,
265
- config,
266
- quant_config: Optional[QuantizationConfig] = None,
267
- prefix: str = '',
268
- ):
269
- super().__init__()
270
- vision_config = Namespace(**config.vision_config)
271
- self.patch_embedding = PatchEmbedding(vision_config)
272
- self.transformer = Transformer(vision_config,
273
- quant_config=quant_config,
274
- prefix=f"{prefix}.transformer")
275
- self.linear_proj = GLU(config,
276
- in_features=config.hidden_size,
277
- quant_config=quant_config,
278
- prefix=f"{prefix}.linear_proj")
279
- self.conv = nn.Conv2d(in_channels=vision_config.hidden_size,
280
- out_channels=config.hidden_size,
281
- kernel_size=2,
282
- stride=2)
283
- self.boi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
284
- self.eoi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
285
- self.scaling_factor = vision_config.scaling_factor
286
-
287
- def forward(self, images: torch.Tensor) -> torch.Tensor:
288
- """
289
- Parameters:
290
- images : torch.Tensor
291
- Input image tensor with shape (B, C, H, W)
292
-
293
- Returns:
294
- torch.Tensor
295
- Transformed tensor with shape (B, L, D)
296
- """
297
- x = self.patch_embedding(images)
298
- x = self.transformer(x)
299
- x = x[:, 1:]
300
-
301
- b, s, h = x.shape
302
- grid_size = int(s**0.5)
303
- x = x.view(b, grid_size, grid_size, h).permute(0, 3, 1, 2)
304
- x = self.conv(x)
305
-
306
- x = x.flatten(2).transpose(1, 2)
307
- x = self.linear_proj(x)
308
- boi = self.boi.expand(x.shape[0], -1, -1)
309
- eoi = self.eoi.expand(x.shape[0], -1, -1)
310
- x = torch.cat((boi, x, eoi), dim=1)
311
- x = x / self.scaling_factor
312
- return x
vllm/model_executor/models/glm4v.py
new
+662
@@ -0,0 +1,662 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# Adapted from
4
+# https://github.com/THUDM/CogAgent
5
+"""Inference-only CogAgent model compatible with THUDM weights."""
6
+from argparse import Namespace
7
+from typing import List, Literal, Mapping, Optional, TypedDict, Union
8
+
9
+import torch
10
+from torch import nn
11
+from torch.nn import LayerNorm
12
+from torchvision import transforms
13
+from torchvision.transforms import InterpolationMode
14
+from transformers import PreTrainedTokenizer, TensorType
15
+from transformers.image_utils import ImageInput
16
+from transformers.tokenization_utils_base import TextInput
17
+
18
+from vllm.attention import AttentionMetadata
19
+from vllm.attention.layer import MultiHeadAttention
20
+from vllm.config import VllmConfig
21
+from vllm.distributed import get_tensor_model_parallel_world_size
22
+from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
23
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24
+ MergedColumnParallelLinear,
25
+ QKVParallelLinear,
26
+ ReplicatedLinear,
27
+ RowParallelLinear)
28
+from vllm.model_executor.layers.quantization import QuantizationConfig
29
+from vllm.model_executor.models.module_mapping import MultiModelKeys
30
+from vllm.multimodal import MULTIMODAL_REGISTRY
31
+from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
32
+from vllm.multimodal.parse import MultiModalDataItems
33
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
34
+ BaseProcessingInfo, BatchFeature,
35
+ MultiModalFieldConfig,
36
+ PromptReplacement)
37
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38
+from vllm.sequence import IntermediateTensors
39
+from vllm.transformers_utils.configs import ChatGLMConfig
40
+
41
+from .chatglm import ChatGLMBaseModel, ChatGLMModel
42
+from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
43
+from .utils import flatten_bn, merge_multimodal_embeddings
44
+
45
+
46
+class GLMVImagePixelInputs(TypedDict):
47
+ type: Literal["pixel_values"]
48
+ data: torch.Tensor
49
+ """Shape: `(batch_size, num_channels, height, width)`"""
50
+
51
+
52
+class EVA2CLIPPatchEmbedding(nn.Module):
53
+
54
+ def __init__(self, config):
55
+ super().__init__()
56
+ self.proj = nn.Conv2d(config.in_channels,
57
+ config.hidden_size,
58
+ kernel_size=config.patch_size,
59
+ stride=config.patch_size)
60
+ self.cls_embedding = nn.Parameter(torch.zeros(1, config.hidden_size))
61
+ self.position_embedding = nn.Embedding(config.num_positions,
62
+ config.hidden_size)
63
+
64
+ def forward(self, images: torch.Tensor) -> torch.Tensor:
65
+ """
66
+ Parameters:
67
+ images : torch.Tensor
68
+ Input image tensor with shape (B, C, H, W)
69
+
70
+ Returns:
71
+ torch.Tensor
72
+ Transformed tensor with shape (B, L, D)
73
+ """
74
+ images = images.to(device=self.proj.weight.device,
75
+ dtype=self.proj.weight.dtype)
76
+ x = self.proj(images)
77
+ x = x.flatten(2).transpose(1, 2)
78
+ cls_token = self.cls_embedding.expand(x.shape[0], -1, -1)
79
+ x = torch.cat((cls_token, x), dim=1)
80
+ x += self.position_embedding.weight.unsqueeze(0)
81
+ return x
82
+
83
+
84
+class EVA2CLIPAttention(nn.Module):
85
+
86
+ def __init__(
87
+ self,
88
+ config,
89
+ quant_config: Optional[QuantizationConfig] = None,
90
+ prefix: str = '',
91
+ ):
92
+ super().__init__()
93
+ self.hidden_size = config.hidden_size
94
+ self.tp_size = get_tensor_model_parallel_world_size()
95
+ self.num_heads_per_rank = config.num_heads // self.tp_size
96
+ self.head_dim = config.hidden_size // config.num_heads
97
+ self.scale = self.head_dim**-0.5
98
+
99
+ self.query_key_value = QKVParallelLinear(
100
+ config.hidden_size,
101
+ self.head_dim,
102
+ config.num_heads,
103
+ quant_config=quant_config,
104
+ prefix=f"{prefix}.query_key_value",
105
+ )
106
+ self.dense = RowParallelLinear(
107
+ config.hidden_size,
108
+ config.hidden_size,
109
+ quant_config=quant_config,
110
+ prefix=f"{prefix}.dense",
111
+ )
112
+
113
+ self.attn = MultiHeadAttention(self.num_heads_per_rank, self.head_dim,
114
+ self.scale)
115
+ self.output_dropout = torch.nn.Dropout(config.dropout_prob)
116
+
117
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
118
+ qkv, _ = self.query_key_value(x) # B, L, 3 * H * D
119
+ q, k, v = qkv.chunk(3, dim=-1)
120
+
121
+ out = self.attn(q, k, v)
122
+ output, _ = self.dense(out)
123
+ output = self.output_dropout(output)
124
+ return output
125
+
126
+
127
+class EVA2CLIPMLP(nn.Module):
128
+
129
+ def __init__(
130
+ self,
131
+ config,
132
+ quant_config: Optional[QuantizationConfig] = None,
133
+ prefix: str = '',
134
+ ):
135
+ super().__init__()
136
+ self.config = config
137
+ self.activation_fn = get_act_fn(config.hidden_act)
138
+ self.fc1 = ColumnParallelLinear(
139
+ config.hidden_size,
140
+ config.intermediate_size,
141
+ quant_config=quant_config,
142
+ prefix=f"{prefix}.fc1",
143
+ )
144
+ self.fc2 = RowParallelLinear(
145
+ config.intermediate_size,
146
+ config.hidden_size,
147
+ quant_config=quant_config,
148
+ prefix=f"{prefix}.fc2",
149
+ )
150
+
151
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
152
+ x, _ = self.fc1(x)
153
+ x = self.activation_fn(x)
154
+ x, _ = self.fc2(x)
155
+ return x
156
+
157
+
158
+class EVA2CLIPTransformerLayer(nn.Module):
159
+
160
+ def __init__(
161
+ self,
162
+ config,
163
+ quant_config: Optional[QuantizationConfig] = None,
164
+ prefix: str = '',
165
+ ):
166
+ super().__init__()
167
+ self.input_layernorm = LayerNorm(config.hidden_size,
168
+ eps=config.layer_norm_eps)
169
+ self.attention = EVA2CLIPAttention(config,
170
+ quant_config=quant_config,
171
+ prefix=f"{prefix}.attention")
172
+ self.mlp = EVA2CLIPMLP(config,
173
+ quant_config=quant_config,
174
+ prefix=f"{prefix}.mlp")
175
+ self.post_attention_layernorm = LayerNorm(config.hidden_size,
176
+ eps=config.layer_norm_eps)
177
+
178
+ def forward(self, hidden_states):
179
+ attention_input = hidden_states
180
+ attention_output = self.input_layernorm(
181
+ self.attention(attention_input))
182
+ hidden_states = attention_input + attention_output
183
+ mlp_input = hidden_states
184
+ mlp_output = self.post_attention_layernorm(self.mlp(mlp_input))
185
+ output = mlp_input + mlp_output
186
+ return output
187
+
188
+
189
+class EVA2CLIPTransformer(nn.Module):
190
+
191
+ def __init__(
192
+ self,
193
+ config,
194
+ quant_config: Optional[QuantizationConfig] = None,
195
+ prefix: str = '',
196
+ ):
197
+ super().__init__()
198
+ self.layers = nn.ModuleList([
199
+ EVA2CLIPTransformerLayer(config,
200
+ quant_config=quant_config,
201
+ prefix=f"{prefix}.layers.{layer_idx}")
202
+ for layer_idx in range(config.num_hidden_layers)
203
+ ])
204
+
205
+ def forward(self, hidden_states):
206
+ for layer_module in self.layers:
207
+ hidden_states = layer_module(hidden_states)
208
+ return hidden_states
209
+
210
+
211
+class EVA2CLIPGLU(nn.Module):
212
+
213
+ def __init__(
214
+ self,
215
+ config,
216
+ in_features,
217
+ quant_config: Optional[QuantizationConfig] = None,
218
+ prefix: str = '',
219
+ ):
220
+ """
221
+ The original implementation is the same as:
222
+ ```python
223
+ self.dense_h_to_4h = ColumnParallelLinear(
224
+ config.hidden_size,
225
+ config.ffn_hidden_size,
226
+ bias=False,
227
+ quant_config=quant_config
228
+ )
229
+
230
+ self.gate_proj = ColumnParallelLinear(
231
+ config.hidden_size,
232
+ config.ffn_hidden_size,
233
+ bias=False,
234
+ quant_config=quant_config
235
+ )
236
+ ```
237
+ ```
238
+ gate_proj_output, _ = self.gate_proj(x)
239
+ dense_h_to_4h_output, _ = self.dense_h_to_4h(x)
240
+ x = torch.cat([gate_proj_output, dense_h_to_4h_output], dim=-1)
241
+ ```
242
+
243
+ We merge two ColumnParallelLinear into one MergedColumnParallelLinear:
244
+ ```
245
+ self.merged_proj = MergedColumnParallelLinear(
246
+ config.hidden_size,
247
+ [config.ffn_hidden_size] * 2,
248
+ bias=False,
249
+ quant_config=quant_config
250
+ )
251
+ ```
252
+ ```
253
+ x, _ = self.merged_proj(x)
254
+ ```
255
+ """
256
+ super().__init__()
257
+ self.linear_proj = ReplicatedLinear(in_features,
258
+ config.hidden_size,
259
+ bias=False,
260
+ quant_config=quant_config,
261
+ prefix=f"{prefix}.linear_proj")
262
+ self.norm1 = nn.LayerNorm(config.hidden_size)
263
+ self.act1 = nn.GELU()
264
+ self.act2 = SiluAndMul()
265
+
266
+ self.merged_proj = MergedColumnParallelLinear(
267
+ config.hidden_size, [config.ffn_hidden_size] * 2,
268
+ bias=False,
269
+ quant_config=quant_config,
270
+ prefix=f"{prefix}.merged_proj")
271
+
272
+ self.dense_4h_to_h = RowParallelLinear(
273
+ config.ffn_hidden_size,
274
+ config.hidden_size,
275
+ bias=False,
276
+ quant_config=quant_config,
277
+ prefix=f"{prefix}.dense_4h_to_h")
278
+
279
+ def forward(self, x):
280
+ x, _ = self.linear_proj(x)
281
+ x = self.act1(self.norm1(x))
282
+ x, _ = self.merged_proj(x)
283
+ x = self.act2(x)
284
+ x, _ = self.dense_4h_to_h(x)
285
+ return x
286
+
287
+
288
+class EVA2CLIPModel(nn.Module):
289
+
290
+ def __init__(
291
+ self,
292
+ config,
293
+ quant_config: Optional[QuantizationConfig] = None,
294
+ prefix: str = '',
295
+ ):
296
+ super().__init__()
297
+ vision_config = Namespace(**config.vision_config)
298
+ self.patch_embedding = EVA2CLIPPatchEmbedding(vision_config)
299
+ self.transformer = EVA2CLIPTransformer(vision_config,
300
+ quant_config=quant_config,
301
+ prefix=f"{prefix}.transformer")
302
+ self.linear_proj = EVA2CLIPGLU(config,
303
+ in_features=config.hidden_size,
304
+ quant_config=quant_config,
305
+ prefix=f"{prefix}.linear_proj")
306
+ self.conv = nn.Conv2d(in_channels=vision_config.hidden_size,
307
+ out_channels=config.hidden_size,
308
+ kernel_size=2,
309
+ stride=2)
310
+ self.boi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
311
+ self.eoi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
312
+ self.scaling_factor = vision_config.scaling_factor
313
+
314
+ def forward(self, images: torch.Tensor) -> torch.Tensor:
315
+ """
316
+ Parameters:
317
+ images : torch.Tensor
318
+ Input image tensor with shape (B, C, H, W)
319
+
320
+ Returns:
321
+ torch.Tensor
322
+ Transformed tensor with shape (B, L, D)
323
+ """
324
+ x = self.patch_embedding(images)
325
+ x = self.transformer(x)
326
+ x = x[:, 1:]
327
+
328
+ b, s, h = x.shape
329
+ grid_size = int(s**0.5)
330
+ x = x.view(b, grid_size, grid_size, h).permute(0, 3, 1, 2)
331
+ x = self.conv(x)
332
+
333
+ x = x.flatten(2).transpose(1, 2)
334
+ x = self.linear_proj(x)
335
+ boi = self.boi.expand(x.shape[0], -1, -1)
336
+ eoi = self.eoi.expand(x.shape[0], -1, -1)
337
+ x = torch.cat((boi, x, eoi), dim=1)
338
+ x = x / self.scaling_factor
339
+ return x
340
+
341
+
342
+class GLM4VModel(ChatGLMModel):
343
+
344
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
345
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
346
+
347
+ quant_config = vllm_config.quant_config
348
+
349
+ self.vision = EVA2CLIPModel(self.config,
350
+ quant_config,
351
+ prefix=f"{prefix}.vision")
352
+
353
+
354
+class GLM4VProcessor:
355
+ """
356
+ This model doesn't define its own HF processor,
357
+ so we implement our own one here.
358
+ """
359
+
360
+ def __init__(
361
+ self,
362
+ config: ChatGLMConfig,
363
+ tokenizer: PreTrainedTokenizer,
364
+ ) -> None:
365
+ super().__init__()
366
+
367
+ self.config = config
368
+ self.tokenizer = tokenizer
369
+
370
+ vision_config = config.vision_config
371
+ image_size = vision_config["image_size"]
372
+
373
+ self.image_transform = transforms.Compose([
374
+ transforms.Resize(
375
+ (image_size, image_size),
376
+ interpolation=InterpolationMode.BICUBIC,
377
+ ),
378
+ transforms.ToTensor(),
379
+ transforms.Normalize(
380
+ mean=(0.48145466, 0.4578275, 0.40821073),
381
+ std=(0.26862954, 0.26130258, 0.27577711),
382
+ ),
383
+ ])
384
+
385
+ def __call__(
386
+ self,
387
+ text: Optional[Union[TextInput, list[TextInput]]] = None,
388
+ images: Optional[Union[ImageInput, list[ImageInput]]] = None,
389
+ return_tensors: Optional[Union[str, TensorType]] = None,
390
+ ) -> BatchFeature:
391
+ if text is None:
392
+ text = []
393
+ if not isinstance(text, list):
394
+ text = [text]
395
+ if images is None:
396
+ images = []
397
+ if not isinstance(images, list):
398
+ images = [images]
399
+
400
+ text_inputs = self.tokenizer(text)
401
+
402
+ if len(images) == 0:
403
+ image_inputs = {}
404
+ else:
405
+ pixel_values = [self.image_transform(image) for image in images]
406
+ image_inputs = {"pixel_values": torch.stack(pixel_values)}
407
+
408
+ return BatchFeature(
409
+ {
410
+ **text_inputs,
411
+ **image_inputs,
412
+ },
413
+ tensor_type=return_tensors,
414
+ )
415
+
416
+
417
+class GLM4VProcessingInfo(BaseProcessingInfo):
418
+
419
+ def get_tokenizer(self):
420
+ tokenizer = self.ctx.tokenizer
421
+ assert isinstance(tokenizer, PreTrainedTokenizer)
422
+ return tokenizer
423
+
424
+ def get_hf_config(self):
425
+ return self.ctx.get_hf_config(ChatGLMConfig)
426
+
427
+ def get_hf_processor(self) -> GLM4VProcessor:
428
+ return GLM4VProcessor(
429
+ self.get_hf_config(),
430
+ self.get_tokenizer(),
431
+ )
432
+
433
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
434
+ return {"image": 1}
435
+
436
+ def get_mm_max_tokens_per_item(
437
+ self,
438
+ seq_len: int,
439
+ mm_counts: Mapping[str, int],
440
+ ) -> Mapping[str, int]:
441
+ return {"image": self.get_num_image_feature_tokens()}
442
+
443
+ def get_num_image_tokens(self) -> int:
444
+ hf_config = self.get_hf_config()
445
+ vision_config = hf_config.vision_config
446
+
447
+ image_size = vision_config["image_size"]
448
+ patch_size = vision_config["patch_size"]
449
+ grid_length = image_size // patch_size // 2
450
+ return grid_length * grid_length
451
+
452
+ def get_num_image_feature_tokens(self) -> int:
453
+ # EVA2CLIPModel has embeddings for boi and eoi tokens as well
454
+ return self.get_num_image_tokens() + 2
455
+
456
+
457
+class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
458
+
459
+ def get_dummy_processor_inputs(
460
+ self,
461
+ seq_len: int,
462
+ mm_counts: Mapping[str, int],
463
+ ) -> ProcessorInputs:
464
+ hf_config = self.info.get_hf_config()
465
+ vision_config = hf_config.vision_config
466
+
467
+ target_width = target_height = vision_config["image_size"]
468
+ num_images = mm_counts.get("image", 0)
469
+
470
+ mm_data = {
471
+ "image":
472
+ self._get_dummy_images(width=target_width,
473
+ height=target_height,
474
+ num_images=num_images)
475
+ }
476
+
477
+ base_text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
478
+
479
+ return ProcessorInputs(
480
+ prompt_text=base_text * num_images,
481
+ mm_data=mm_data,
482
+ )
483
+
484
+
485
+class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
486
+
487
+ def _get_mm_fields_config(
488
+ self,
489
+ hf_inputs: BatchFeature,
490
+ hf_processor_mm_kwargs: Mapping[str, object],
491
+ ) -> Mapping[str, MultiModalFieldConfig]:
492
+ return dict(pixel_values=MultiModalFieldConfig.batched("image"))
493
+
494
+ def _get_prompt_replacements(
495
+ self,
496
+ mm_items: MultiModalDataItems,
497
+ hf_processor_mm_kwargs: Mapping[str, object],
498
+ out_mm_kwargs: MultiModalKwargs,
499
+ ) -> list[PromptReplacement]:
500
+ hf_config = self.info.get_hf_config()
501
+
502
+ boi_token_id = hf_config.boi_token_id
503
+ image_token_id = hf_config.pad_token_id
504
+ eoi_token_id = hf_config.eoi_token_id
505
+
506
+ def get_replacement(item_idx: int):
507
+ num_image_tokens = self.info.get_num_image_tokens()
508
+ image_tokens = [image_token_id] * num_image_tokens
509
+
510
+ return [boi_token_id] + image_tokens + [eoi_token_id]
511
+
512
+ return [
513
+ PromptReplacement(
514
+ modality="image",
515
+ target=[boi_token_id, image_token_id, eoi_token_id],
516
+ replacement=get_replacement,
517
+ ),
518
+ ]
519
+
520
+
521
+@MULTIMODAL_REGISTRY.register_processor(GLM4VMultiModalProcessor,
522
+ info=GLM4VProcessingInfo,
523
+ dummy_inputs=GLM4VDummyInputsBuilder)
524
+class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
525
+ SupportsMultiModal):
526
+
527
+ packed_modules_mapping = {
528
+ "query_key_value": ["query_key_value"],
529
+ "dense_h_to_4h": ["dense_h_to_4h"],
530
+ "merged_proj": ["gate_proj", "dense_h_to_4h"]
531
+ }
532
+ # LoRA specific attributes
533
+ supported_lora_modules = [
534
+ "query_key_value",
535
+ "dense",
536
+ "dense_h_to_4h",
537
+ "dense_4h_to_h",
538
+ # vision
539
+ "fc1",
540
+ "fc2",
541
+ "merged_proj",
542
+ "linear_proj"
543
+ ]
544
+
545
+ embedding_modules = {}
546
+ embedding_padding_modules = []
547
+
548
+ def get_mm_mapping(self) -> MultiModelKeys:
549
+ """
550
+ Get the module prefix in multimodal models
551
+ """
552
+ return MultiModelKeys.from_string_field(
553
+ language_model="transformer.encoder",
554
+ connector="transformer.vision.linear_proj",
555
+ tower_model="transformer.vision.transformer")
556
+
557
+ def __init__(
558
+ self,
559
+ *,
560
+ vllm_config: VllmConfig,
561
+ prefix: str = "",
562
+ transformer_type: type[GLM4VModel] = GLM4VModel,
563
+ ) -> None:
564
+ super().__init__(
565
+ vllm_config=vllm_config,
566
+ prefix=prefix,
567
+ transformer_type=transformer_type,
568
+ )
569
+
570
+ self.transformer: GLM4VModel
571
+
572
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
573
+ h = w = self.config.vision_config["image_size"]
574
+ expected_dims = (3, h, w)
575
+ actual_dims = tuple(data.shape[1:])
576
+
577
+ if actual_dims != expected_dims:
578
+ expected_expr = ("batch_size", *map(str, expected_dims))
579
+ raise ValueError(
580
+ f"The expected shape of pixel values is {expected_expr}. "
581
+ f"You supplied {tuple(data.shape)}.")
582
+
583
+ return data
584
+
585
+ def _parse_and_validate_image_input(
586
+ self, **kwargs: object) -> Optional[GLMVImagePixelInputs]:
587
+ pixel_values = kwargs.pop("pixel_values", None)
588
+
589
+ if pixel_values is not None:
590
+ if not isinstance(pixel_values, torch.Tensor):
591
+ raise ValueError("Incorrect type of pixel values. "
592
+ f"Got type: {type(pixel_values)}")
593
+
594
+ return GLMVImagePixelInputs(
595
+ type="pixel_values",
596
+ data=self._validate_pixel_values(
597
+ flatten_bn(pixel_values, concat=True)),
598
+ )
599
+
600
+ return None
601
+
602
+ def _process_image_input(
603
+ self, image_input: GLMVImagePixelInputs) -> torch.Tensor:
604
+ pixel_values = image_input["data"].to(dtype=self.config.torch_dtype)
605
+
606
+ return self.transformer.vision(pixel_values)
607
+
608
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
609
+ image_input = self._parse_and_validate_image_input(**kwargs)
610
+ if image_input is None:
611
+ return None
612
+
613
+ vision_embeddings = self._process_image_input(image_input)
614
+ return vision_embeddings
615
+
616
+ def get_input_embeddings(
617
+ self,
618
+ input_ids: torch.Tensor,
619
+ multimodal_embeddings: Optional[NestedTensors] = None,
620
+ ) -> torch.Tensor:
621
+ inputs_embeds = self.transformer.get_input_embeddings(input_ids)
622
+
623
+ if multimodal_embeddings is not None:
624
+ inputs_embeds = merge_multimodal_embeddings(
625
+ input_ids=input_ids,
626
+ inputs_embeds=inputs_embeds,
627
+ multimodal_embeddings=multimodal_embeddings,
628
+ placeholder_token_id=[
629
+ self.config.boi_token_id,
630
+ self.config.pad_token_id,
631
+ self.config.eoi_token_id,
632
+ ],
633
+ )
634
+
635
+ return inputs_embeds
636
+
637
+ def forward(
638
+ self,
639
+ input_ids: torch.Tensor,
640
+ positions: torch.Tensor,
641
+ kv_caches: List[torch.Tensor],
642
+ attn_metadata: AttentionMetadata,
643
+ intermediate_tensors: Optional[IntermediateTensors] = None,
644
+ inputs_embeds: Optional[torch.Tensor] = None,
645
+ **kwargs: object,
646
+ ) -> Union[torch.Tensor, IntermediateTensors]:
647
+ if intermediate_tensors is not None:
648
+ inputs_embeds = None
649
+
650
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
651
+ # condition is for v0 compatibility.
652
+ elif inputs_embeds is None:
653
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
654
+ inputs_embeds = self.get_input_embeddings(input_ids,
655
+ vision_embeddings)
656
+ input_ids = None
657
+
658
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
659
+ attn_metadata, intermediate_tensors,
660
+ inputs_embeds)
661
+
662
+ return hidden_states
vllm/model_executor/models/qwen.py
+42
-814
@@ -6,381 +6,35 @@
6
# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
7
"""Inference-only QWen model compatible with HuggingFace weights."""
8
9
-import copy
10
-import math
11
-import re
12
-import unicodedata
13
-from functools import lru_cache, partial
14
-from typing import (AbstractSet, Any, Callable, Collection, Dict, Iterable,
15
- List, Literal, Mapping, Optional, Set, Tuple, TypedDict,
16
- Union)
9
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
10
11
import torch
12
from torch import nn
20
-from torchvision import transforms
21
-from torchvision.transforms import InterpolationMode
22
-from transformers import (BatchFeature, PretrainedConfig, PreTrainedTokenizer,
23
- TensorType)
24
-from transformers.image_utils import ImageInput
25
-from transformers.tokenization_utils_base import TextInput
13
+from transformers import PretrainedConfig
14
15
from vllm.attention import Attention, AttentionMetadata
16
from vllm.compilation.decorators import support_torch_compile
17
from vllm.config import CacheConfig, VllmConfig
18
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
31
-from vllm.logger import init_logger
32
-from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
19
+from vllm.model_executor.layers.activation import SiluAndMul
20
from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35
- MergedColumnParallelLinear,
21
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
22
QKVParallelLinear,
37
- ReplicatedLinear,
23
RowParallelLinear)
24
from vllm.model_executor.layers.logits_processor import LogitsProcessor
25
from vllm.model_executor.layers.quantization import QuantizationConfig
41
-from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
26
from vllm.model_executor.layers.rotary_embedding import get_rope
27
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
28
from vllm.model_executor.layers.vocab_parallel_embedding import (
29
ParallelLMHead, VocabParallelEmbedding)
30
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47
-from vllm.model_executor.models.module_mapping import MultiModelKeys
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
-from vllm.multimodal import MULTIMODAL_REGISTRY
50
-from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
51
- NestedTensors)
52
-from vllm.multimodal.parse import MultiModalDataItems
53
-from vllm.multimodal.processing import (BaseMultiModalProcessor,
54
- BaseProcessingInfo, PromptReplacement,
55
- PromptReplacementDetails)
56
-from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32
from vllm.sequence import IntermediateTensors
33
59
-from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
60
-from .utils import (flatten_bn, is_pp_missing_parameter,
34
+from .interfaces import SupportsLoRA, SupportsPP
35
+from .utils import (is_pp_missing_parameter,
36
make_empty_intermediate_tensors_factory, make_layers,
62
- maybe_prefix, merge_multimodal_embeddings)
63
-
64
-logger = init_logger(__name__)
65
-
66
-
67
-class QwenImagePixelInputs(TypedDict):
68
- type: Literal["pixel_values"]
69
- data: torch.Tensor
70
- """
71
- Shape: `(batch_size * num_images, 3, image_size, image_size)`
72
-
73
- Note that image_size is the value in the vision config to which we resize
74
- the image to in the normalization transform. Currently multi-image support
75
- can only be leveraged by passing image embeddings directly.
76
- """
77
-
78
-
79
-class QwenImageEmbeddingInputs(TypedDict):
80
- type: Literal["image_embeds"]
81
- data: torch.Tensor
82
- """Shape: `(batch_size * num_images, 256, hidden_size)`
83
-
84
- `hidden_size` must match the hidden size of the language model backbone
85
- and is stored in the visual config of the model if we have one.
86
- """
87
-
88
-
89
-QwenImageInputs = Union[QwenImagePixelInputs, QwenImageEmbeddingInputs]
90
-
91
-
92
-class VisualAttention(nn.Module):
93
- """self-attention layer class.
94
- Self-attention layer takes input with size [s, b, h]
95
- and returns output of the same size.
96
- """
97
-
98
- def __init__(
99
- self,
100
- embed_dim: int,
101
- num_heads: int,
102
- bias: bool = True,
103
- kdim: Optional[int] = None,
104
- vdim: Optional[int] = None,
105
- ):
106
- super().__init__()
107
- self.embed_dim = embed_dim
108
- self.kdim = kdim if kdim is not None else embed_dim
109
- self.vdim = vdim if vdim is not None else embed_dim
110
- self._qkv_same_embed_dim = self.kdim == embed_dim \
111
- and self.vdim == embed_dim
112
-
113
- self.num_heads = num_heads
114
-
115
- # Per attention head and per partition values.
116
- assert embed_dim % num_heads == 0
117
- self.hidden_size_per_attention_head = embed_dim // num_heads
118
- self.num_attention_heads_per_partition = num_heads
119
- self.hidden_size_per_partition = embed_dim
120
-
121
- # Strided linear layer.
122
- assert self._qkv_same_embed_dim, \
123
- 'Visual Attention implementation only supports self-attention'
124
- self.in_proj = ReplicatedLinear(embed_dim, 3 * embed_dim)
125
- self.out_proj = ReplicatedLinear(embed_dim, embed_dim)
126
- self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
127
-
128
- def forward(
129
- self,
130
- x: torch.Tensor,
131
- attn_mask: Optional[torch.Tensor] = None,
132
- ) -> torch.Tensor:
133
- # query/key/value: [sq, b, h]
134
- sq, b, _ = x.size()
135
- mixed_x_layer, _ = self.in_proj(x)
136
-
137
- # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
138
- new_tensor_shape = mixed_x_layer.size()[:-1] + \
139
- (self.num_attention_heads_per_partition,
140
- 3 * self.hidden_size_per_attention_head)
141
- mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
142
-
143
- # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
144
- query_layer, key_layer, value_layer = mixed_x_layer.split(
145
- self.hidden_size_per_attention_head, dim=-1)
146
-
147
- # [sq, b, np, hn] -> [sq, b * np, hn]
148
- query_layer = query_layer.view(
149
- sq, b * self.num_attention_heads_per_partition,
150
- self.hidden_size_per_attention_head).transpose(0, 1)
151
- # [sk, b, np, hn] -> [sk, b * np, hn]
152
- key_layer = key_layer.view(
153
- sq, b * self.num_attention_heads_per_partition,
154
- self.hidden_size_per_attention_head).transpose(0, 1)
155
-
156
- q_scaled = query_layer / self.norm_factor
157
- if attn_mask is not None:
158
- attention_probs = torch.baddbmm(attn_mask, q_scaled,
159
- key_layer.transpose(-2, -1))
160
- else:
161
- attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1))
162
- attention_probs = attention_probs.softmax(dim=-1)
163
-
164
- value_layer = value_layer.view(
165
- sq, b * self.num_attention_heads_per_partition,
166
- self.hidden_size_per_attention_head).transpose(0, 1)
167
-
168
- # matmul: [b * np, sq, hn]
169
- context_layer = torch.bmm(attention_probs, value_layer)
170
-
171
- # change view [b, np, sq, hn]
172
- context_layer = context_layer.view(
173
- b, self.num_attention_heads_per_partition, sq,
174
- self.hidden_size_per_attention_head)
175
-
176
- # [b, np, sq, hn] --> [sq, b, np, hn]
177
- context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
178
-
179
- # [sq, b, np, hn] --> [sq, b, hp]
180
- new_context_layer_shape = context_layer.size()[:-2] + \
181
- (self.hidden_size_per_partition,)
182
- context_layer = context_layer.view(*new_context_layer_shape)
183
-
184
- output, _ = self.out_proj(context_layer)
185
-
186
- return output
187
-
188
-
189
-class QwenVMLP(nn.Module):
190
- """MLP for the visual component of the Qwen model."""
191
-
192
- def __init__(
193
- self,
194
- hidden_size: int,
195
- intermediate_size: int,
196
- quant_config: Optional[QuantizationConfig] = None,
197
- ):
198
- super().__init__()
199
- self.c_fc = ColumnParallelLinear(hidden_size,
200
- intermediate_size,
201
- bias=True,
202
- quant_config=quant_config)
203
- self.act_fn = get_act_fn("gelu")
204
- self.c_proj = RowParallelLinear(
205
- intermediate_size,
206
- hidden_size,
207
- bias=True,
208
- quant_config=quant_config,
209
- )
210
-
211
- def forward(self, x):
212
- x, _ = self.c_fc(x)
213
- x = self.act_fn(x)
214
- x, _ = self.c_proj(x)
215
- return x
216
-
217
-
218
-class VisualAttentionBlock(nn.Module):
219
-
220
- def __init__(
221
- self,
222
- d_model: int,
223
- n_head: int,
224
- mlp_ratio: float = 4.0,
225
- norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
226
- quant_config: Optional[QuantizationConfig] = None,
227
- ):
228
- super().__init__()
229
-
230
- self.ln_1 = norm_layer(d_model)
231
- self.ln_2 = norm_layer(d_model)
232
- mlp_width = int(d_model * mlp_ratio)
233
- self.attn = VisualAttention(d_model, n_head)
234
- self.mlp = QwenVMLP(
235
- hidden_size=d_model,
236
- intermediate_size=mlp_width,
237
- quant_config=quant_config,
238
- )
239
-
240
- def attention(
241
- self,
242
- x: torch.Tensor,
243
- attn_mask: Optional[torch.Tensor] = None,
244
- ) -> torch.Tensor:
245
- attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None
246
- return self.attn(x, attn_mask=attn_mask)
247
-
248
- def forward(
249
- self,
250
- x: torch.Tensor,
251
- attn_mask: Optional[torch.Tensor] = None,
252
- ) -> torch.Tensor:
253
- x = x + self.attention(self.ln_1(x), attn_mask=attn_mask)
254
- x = x + self.mlp(self.ln_2(x))
255
- return x
256
-
257
-
258
-class TransformerBlock(nn.Module):
259
-
260
- def __init__(
261
- self,
262
- width: int,
263
- layers: int,
264
- heads: int,
265
- mlp_ratio: float = 4.0,
266
- norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
267
- quant_config: Optional[QuantizationConfig] = None,
268
- ):
269
- super().__init__()
270
- self.width = width
271
- self.layers = layers
272
-
273
- self.resblocks = nn.ModuleList([
274
- VisualAttentionBlock(width,
275
- heads,
276
- mlp_ratio,
277
- norm_layer=norm_layer,
278
- quant_config=quant_config)
279
- for _ in range(layers)
280
- ])
281
-
282
- def get_cast_dtype(self) -> torch.dtype:
283
- return self.resblocks[0].mlp.c_fc.weight.dtype
284
-
285
- def get_cast_device(self) -> torch.device:
286
- return self.resblocks[0].mlp.c_fc.weight.device
287
-
288
- def forward(self,
289
- x: torch.Tensor,
290
- attn_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
291
- for r in self.resblocks:
292
- x = r(x, attn_mask=attn_mask)
293
- return x
294
-
295
-
296
-class VisionTransformer(nn.Module):
297
-
298
- def __init__(self,
299
- image_size: int,
300
- patch_size: int,
301
- width: int,
302
- layers: int,
303
- heads: int,
304
- mlp_ratio: float,
305
- n_queries: int = 256,
306
- output_dim: int = 512,
307
- image_start_id: int = 151857,
308
- quant_config: Optional[QuantizationConfig] = None,
309
- **kwargs):
310
- super().__init__()
311
- image_height, image_width = self.image_size = (image_size, image_size)
312
- patch_height, patch_width = self.patch_size = (patch_size, patch_size)
313
- self.grid_size = (image_height // patch_height,
314
- image_width // patch_width)
315
- self.output_dim = output_dim
316
- self.conv1 = nn.Conv2d(in_channels=3,
317
- out_channels=width,
318
- kernel_size=patch_size,
319
- stride=patch_size,
320
- bias=False)
321
-
322
- # class embeddings and positional embeddings
323
- scale = width**-0.5
324
- self.positional_embedding = nn.Parameter(scale *
325
- torch.randn(256, width))
326
-
327
- norm_layer = partial(nn.LayerNorm, eps=1e-6)
328
-
329
- self.ln_pre = norm_layer(width)
330
- self.transformer = TransformerBlock(width,
331
- layers,
332
- heads,
333
- mlp_ratio,
334
- norm_layer=norm_layer,
335
- quant_config=quant_config)
336
-
337
- self.attn_pool = Resampler2(
338
- grid_size=int(math.sqrt(n_queries)),
339
- embed_dim=output_dim,
340
- num_heads=output_dim // 128,
341
- kv_dim=width,
342
- norm_layer=norm_layer,
343
- adaptive=False,
344
- do_post_projection=False,
345
- ).to(
346
- device=self.positional_embedding.device,
347
- dtype=self.positional_embedding.dtype,
348
- )
349
-
350
- self.ln_post = norm_layer(output_dim)
351
- self.proj = nn.Parameter(
352
- (output_dim**-0.5) * torch.randn(output_dim, output_dim))
353
-
354
- self.image_start_id = image_start_id
355
- self.image_end_id = image_start_id + 1
356
- self.image_pad_id = image_start_id + 2
357
-
358
- def forward(self, x: torch.Tensor) -> torch.Tensor:
359
- x = x.to(
360
- dtype=self.transformer.get_cast_dtype(),
361
- device=self.transformer.get_cast_device(),
362
- )
363
-
364
- # to patches
365
- x = self.conv1(x) # shape = [*, width, grid, grid]
366
- x = x.reshape(x.shape[0], x.shape[1],
367
- -1) # shape = [*, width, grid ** 2]
368
- x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
369
-
370
- x = x + get_abs_pos(self.positional_embedding, int(math.sqrt(
371
- x.size(1))))
372
-
373
- x = self.ln_pre(x)
374
-
375
- x = x.permute(1, 0, 2) # NLD -> LND
376
- x = self.transformer(x)
377
- x = x.permute(1, 0, 2) # LND -> NLD
378
-
379
- x = self.attn_pool(x)
380
- x = self.ln_post(x)
381
- x = x @ self.proj
382
-
383
- return x
37
+ maybe_prefix)
38
39
40
class QWenMLP(nn.Module):
@@ -564,12 +218,6 @@ class QWenModel(nn.Module):
218
make_empty_intermediate_tensors_factory(
219
["hidden_states", "residual"], config.hidden_size))
220
567
- if (vision_config := getattr(config, "visual", None)):
568
- self.visual = VisionTransformer(**vision_config,
569
- quant_config=quant_config)
570
- else:
571
- self.visual = None
572
-
221
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
222
return self.wte(input_ids)
223
@@ -592,6 +240,7 @@ class QWenModel(nn.Module):
240
assert intermediate_tensors is not None
241
hidden_states = intermediate_tensors["hidden_states"]
242
residual = intermediate_tensors["residual"]
243
+
244
for i in range(self.start_layer, self.end_layer):
245
layer = self.h[i]
246
hidden_states, residual = layer(
@@ -610,302 +259,25 @@ class QWenModel(nn.Module):
259
return hidden_states
260
261
613
-@lru_cache(maxsize=1)
614
-def _get_tokenizer_without_image_pad(
615
- tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
616
- """
617
- The logic of adding image pad tokens should only be applied in
618
- :class:`QWenVLProcessor`, so they are patched out here.
619
-
620
- The definition of the wrapped tokenizer can be found here:
621
- https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
622
- """
623
- new_tokenizer = copy.deepcopy(tokenizer)
624
-
625
- class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
626
-
627
- def tokenize(
628
- self,
629
- text: str,
630
- allowed_special: Union[AbstractSet[str], str] = "all",
631
- disallowed_special: Union[Collection[str], str] = (),
632
- **kwargs,
633
- ) -> list[Union[bytes, str]]:
634
- text = unicodedata.normalize("NFC", text)
635
-
636
- return [
637
- self.decoder[t] for t in self.tokenizer.encode(
638
- text,
639
- allowed_special=allowed_special,
640
- disallowed_special=disallowed_special,
641
- )
642
- ]
643
-
644
- def _decode(
645
- self,
646
- token_ids: Union[int, List[int]],
647
- skip_special_tokens: bool = False,
648
- errors: Optional[str] = None,
649
- **kwargs,
650
- ) -> str:
651
- if isinstance(token_ids, int):
652
- token_ids = [token_ids]
653
-
654
- return self.tokenizer.decode(
655
- token_ids,
656
- errors=errors or self.errors,
657
- )
658
-
659
- TokenizerWithoutImagePad.__name__ = \
660
- f"{tokenizer.__class__.__name__}WithoutImagePad"
661
-
662
- new_tokenizer.__class__ = TokenizerWithoutImagePad
663
- return new_tokenizer
664
-
665
-
666
-class QWenVLProcessor:
667
- """
668
- This model doesn't define its own HF processor,
669
- so we implement our own one here.
670
-
671
- We call the wrapped tokenizer to automatically insert image pad tokens:
672
- https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py#L245
673
-
674
- The image processor is defined here:
675
- https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
676
- """
262
+class QWenBaseModel(nn.Module):
263
264
def __init__(
265
self,
680
- config: PretrainedConfig,
681
- tokenizer: PreTrainedTokenizer,
266
+ *,
267
+ vllm_config: VllmConfig,
268
+ prefix: str = "",
269
+ transformer_type: type[QWenModel] = QWenModel,
270
) -> None:
271
super().__init__()
684
-
685
- self.config = config
686
- self.tokenizer = tokenizer
687
-
688
- if vision_config := getattr(self.config, "visual", None):
689
- image_size = vision_config["image_size"]
690
-
691
- self.image_transform = transforms.Compose([
692
- transforms.Resize(
693
- (image_size, image_size),
694
- interpolation=InterpolationMode.BICUBIC,
695
- ),
696
- transforms.ToTensor(),
697
- transforms.Normalize(
698
- mean=(0.48145466, 0.4578275, 0.40821073),
699
- std=(0.26862954, 0.26130258, 0.27577711),
700
- ),
701
- ])
702
- else:
703
- self.image_transform = None
704
-
705
- @property
706
- def image_start_tag(self) -> str:
707
- return self.tokenizer.image_start_tag # type: ignore
708
-
709
- @property
710
- def image_end_tag(self) -> str:
711
- return self.tokenizer.image_end_tag # type: ignore
712
-
713
- @property
714
- def image_pad_tag(self) -> str:
715
- return self.tokenizer.image_pad_tag # type: ignore
716
-
717
- def __call__(
718
- self,
719
- text: Optional[Union[TextInput, list[TextInput]]] = None,
720
- images: Optional[Union[ImageInput, list[ImageInput]]] = None,
721
- return_tensors: Optional[Union[str, TensorType]] = None,
722
- ) -> BatchFeature:
723
- if text is None:
724
- text = []
725
- if not isinstance(text, list):
726
- text = [text]
727
- if images is None:
728
- images = []
729
- if not isinstance(images, list):
730
- images = [images]
731
-
732
- text_inputs = self.tokenizer(text)
733
-
734
- if len(images) == 0:
735
- image_inputs = {}
736
- else:
737
- if self.image_transform is None:
738
- raise ValueError("This model does not support image inputs")
739
-
740
- pixel_values = [self.image_transform(image) for image in images]
741
- image_inputs = {"pixel_values": torch.stack(pixel_values)}
742
-
743
- return BatchFeature(
744
- {
745
- **text_inputs,
746
- **image_inputs,
747
- },
748
- tensor_type=return_tensors,
749
- )
750
-
751
-
752
-class QWenVLProcessingInfo(BaseProcessingInfo):
753
-
754
- def get_tokenizer(self) -> PreTrainedTokenizer:
755
- tokenizer = self.ctx.tokenizer
756
- assert isinstance(tokenizer, PreTrainedTokenizer)
757
-
758
- return _get_tokenizer_without_image_pad(tokenizer)
759
-
760
- def get_hf_processor(self) -> QWenVLProcessor:
761
- tokenizer = self.ctx.tokenizer
762
- assert isinstance(tokenizer, PreTrainedTokenizer)
763
-
764
- return QWenVLProcessor(self.get_hf_config(), tokenizer)
765
-
766
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
767
- return {"image": None}
768
-
769
- def get_mm_max_tokens_per_item(
770
- self,
771
- seq_len: int,
772
- mm_counts: Mapping[str, int],
773
- ) -> Mapping[str, int]:
774
- return {"image": self.get_num_image_tokens()}
775
-
776
- def get_num_image_tokens(self) -> int:
777
- hf_config = self.get_hf_config()
778
- if not (vision_config := getattr(hf_config, "visual", None)):
779
- return 0
780
-
781
- image_size = vision_config["image_size"]
782
- patch_size = vision_config["patch_size"]
783
- grid_length = image_size // patch_size // 2
784
- return grid_length * grid_length
785
-
786
-
787
-class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
788
-
789
- def get_dummy_processor_inputs(
790
- self,
791
- seq_len: int,
792
- mm_counts: Mapping[str, int],
793
- ) -> ProcessorInputs:
794
- hf_config = self.info.get_hf_config()
795
- if not (vision_config := getattr(hf_config, "visual", None)):
796
- return ProcessorInputs(prompt_text="", mm_data={})
797
-
798
- processor = self.info.get_hf_processor()
799
- img_start = processor.image_start_tag
800
- img_end = processor.image_end_tag
801
-
802
- target_width = target_height = vision_config["image_size"]
803
- num_images = mm_counts.get("image", 0)
804
-
805
- mm_data = {
806
- "image":
807
- self._get_dummy_images(width=target_width,
808
- height=target_height,
809
- num_images=num_images)
810
- }
811
-
812
- return ProcessorInputs(
813
- prompt_text="".join(f"Picture {i}: {img_start}{img_end}\n"
814
- for i in range(1, num_images + 1)),
815
- mm_data=mm_data,
816
- )
817
-
818
-
819
-class QWenVLMultiModalProcessor(BaseMultiModalProcessor[QWenVLProcessingInfo]):
820
-
821
- def _call_hf_processor(
822
- self,
823
- prompt: str,
824
- mm_data: Mapping[str, object],
825
- mm_kwargs: Mapping[str, object],
826
- ) -> BatchFeature:
827
- # Drops anything between <img>/</img> tags; encoding with the tokenizer
828
- # will automatically add the image pads for the context.
829
- prompt, num_matched_images = re.subn(
830
- r"(Picture \d*: <img>).*?(<\/img>\n)",
831
- r"\1\2",
832
- prompt,
833
- )
834
-
835
- image_data = mm_data.get("images")
836
- if image_data is not None:
837
- assert isinstance(image_data, list)
838
-
839
- num_images = len(image_data)
840
- if num_matched_images != num_images:
841
- logger.warning(
842
- "Number of matched image placeholders %s doesn't match "
843
- "the number of expected images %s; check your placeholder "
844
- "formatting.", num_matched_images, num_images)
845
-
846
- return super()._call_hf_processor(
847
- prompt=prompt,
848
- mm_data=mm_data,
849
- mm_kwargs=mm_kwargs,
850
- )
851
-
852
- def _get_mm_fields_config(
853
- self,
854
- hf_inputs: BatchFeature,
855
- hf_processor_mm_kwargs: Mapping[str, object],
856
- ) -> Mapping[str, MultiModalFieldConfig]:
857
- return dict(
858
- pixel_values=MultiModalFieldConfig.batched("image"),
859
- image_embeds=MultiModalFieldConfig.batched("image"),
860
- )
861
-
862
- def _get_prompt_replacements(
863
- self,
864
- mm_items: MultiModalDataItems,
865
- hf_processor_mm_kwargs: Mapping[str, object],
866
- out_mm_kwargs: MultiModalKwargs,
867
- ) -> list[PromptReplacement]:
868
- hf_config = self.info.get_hf_config()
869
- if not hasattr(hf_config, "visual"):
870
- return []
871
-
872
- tokenizer = self.info.get_tokenizer()
873
- special_tokens: dict[str,
874
- int] = tokenizer.special_tokens # type: ignore
875
-
876
- processor = self.info.get_hf_processor()
877
- img_start_id = special_tokens[processor.image_start_tag]
878
- img_end_id = special_tokens[processor.image_end_tag]
879
- img_pad_id = special_tokens[processor.image_pad_tag]
880
-
881
- num_image_tokens = self.info.get_num_image_tokens()
882
- image_tokens = [img_pad_id] * num_image_tokens
883
-
884
- return [
885
- PromptReplacement(
886
- modality="image",
887
- target=[img_start_id, img_end_id],
888
- replacement=PromptReplacementDetails(
889
- full=[img_start_id] + image_tokens + [img_end_id],
890
- features=image_tokens,
891
- ),
892
- )
893
- ]
894
-
895
-
896
-class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
897
-
898
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
899
- super().__init__()
272
config = vllm_config.model_config.hf_config
273
quant_config = vllm_config.quant_config
274
multimodal_config = vllm_config.model_config.multimodal_config
275
self.config = config
276
self.multimodal_config = multimodal_config
277
self.quant_config = quant_config
906
- self.transformer = QWenModel(vllm_config=vllm_config,
907
- prefix=maybe_prefix(
908
- prefix, "transformer"))
278
+ self.transformer = transformer_type(vllm_config=vllm_config,
279
+ prefix=maybe_prefix(
280
+ prefix, "transformer"))
281
self.lm_head = ParallelLMHead(config.vocab_size,
282
config.hidden_size,
283
quant_config=quant_config)
@@ -916,104 +288,6 @@ class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
288
self.make_empty_intermediate_tensors = (
289
self.transformer.make_empty_intermediate_tensors)
290
919
- def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
920
- h = w = self.config.visual["image_size"]
921
- expected_dims = (3, h, w)
922
- actual_dims = tuple(data.shape[1:])
923
-
924
- if actual_dims != expected_dims:
925
- expected_expr = ("batch_size", *map(str, expected_dims))
926
- raise ValueError(
927
- f"The expected shape of pixel values is {expected_expr}. "
928
- f"You supplied {tuple(data.shape)}.")
929
-
930
- return data
931
-
932
- def _parse_and_validate_image_input(
933
- self, **kwargs: object) -> Optional[QwenImageInputs]:
934
- pixel_values = kwargs.pop("pixel_values", None)
935
- image_embeds = kwargs.pop("image_embeds", None)
936
-
937
- if pixel_values is not None:
938
- if not isinstance(pixel_values, torch.Tensor):
939
- raise ValueError("Incorrect type of pixel values. "
940
- f"Got type: {type(pixel_values)}")
941
-
942
- return QwenImagePixelInputs(
943
- type="pixel_values",
944
- data=self._validate_pixel_values(
945
- flatten_bn(pixel_values, concat=True)),
946
- )
947
-
948
- if image_embeds is not None:
949
- if not isinstance(image_embeds, torch.Tensor):
950
- raise ValueError("Incorrect type of image embeddings. "
951
- f"Got type: {type(image_embeds)}")
952
-
953
- return QwenImageEmbeddingInputs(
954
- type="image_embeds",
955
- data=flatten_bn(image_embeds),
956
- )
957
-
958
- return None
959
-
960
- def _process_image_input(self,
961
- image_input: QwenImageInputs) -> torch.Tensor:
962
- if image_input["type"] == "image_embeds":
963
- return image_input["data"]
964
-
965
- assert self.transformer.visual is not None
966
- return self.transformer.visual(image_input["data"])
967
-
968
- def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
969
- image_input = self._parse_and_validate_image_input(**kwargs)
970
- if image_input is None:
971
- return None
972
-
973
- vision_embeddings = self._process_image_input(image_input)
974
- return vision_embeddings
975
-
976
- def get_input_embeddings(
977
- self,
978
- input_ids: torch.Tensor,
979
- multimodal_embeddings: Optional[NestedTensors] = None,
980
- ) -> torch.Tensor:
981
- inputs_embeds = self.transformer.get_input_embeddings(input_ids)
982
-
983
- if multimodal_embeddings is not None:
984
- assert self.transformer.visual is not None
985
- inputs_embeds = merge_multimodal_embeddings(
986
- input_ids, inputs_embeds, multimodal_embeddings,
987
- self.transformer.visual.image_pad_id)
988
-
989
- return inputs_embeds
990
-
991
- def forward(
992
- self,
993
- input_ids: torch.Tensor,
994
- positions: torch.Tensor,
995
- kv_caches: List[torch.Tensor],
996
- attn_metadata: AttentionMetadata,
997
- intermediate_tensors: Optional[IntermediateTensors] = None,
998
- inputs_embeds: Optional[torch.Tensor] = None,
999
- **kwargs: object,
1000
- ) -> Union[torch.Tensor, IntermediateTensors]:
1001
- if intermediate_tensors is not None:
1002
- inputs_embeds = None
1003
-
1004
- # NOTE: In v1, inputs_embeds is always generated at model runner, this
1005
- # condition is for v0 compatibility.
1006
- elif inputs_embeds is None:
1007
- vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1008
- inputs_embeds = self.get_input_embeddings(input_ids,
1009
- vision_embeddings)
1010
- input_ids = None
1011
-
1012
- hidden_states = self.transformer(input_ids, positions, kv_caches,
1013
- attn_metadata, intermediate_tensors,
1014
- inputs_embeds)
1015
- return hidden_states
1016
-
291
def compute_logits(
292
self,
293
hidden_states: torch.Tensor,
@@ -1072,26 +346,7 @@ class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
346
return loaded_params
347
348
1075
-class QWenLLM(QWenBaseModel):
1076
- packed_modules_mapping = {
1077
- "c_attn": ["c_attn"],
1078
- "gate_up_proj": [
1079
- "w2",
1080
- "w1",
1081
- ],
1082
- }
1083
- # LoRA specific attributes
1084
- supported_lora_modules = [
1085
- "c_attn",
1086
- "gate_up_proj",
1087
- "c_proj",
1088
- ]
1089
-
1090
- embedding_modules = {}
1091
- embedding_padding_modules = []
1092
-
1093
-
1094
-class QWenVL(QWenBaseModel, SupportsMultiModal):
349
+class QWenLMHeadModel(QWenBaseModel, SupportsPP, SupportsLoRA):
350
packed_modules_mapping = {
351
"c_attn": ["c_attn"],
352
"gate_up_proj": [
@@ -1104,62 +359,35 @@ class QWenVL(QWenBaseModel, SupportsMultiModal):
359
"c_attn",
360
"gate_up_proj",
361
"c_proj",
1107
- # visual module
1108
- "out_proj",
1109
- "in_proj",
1110
- "c_fc",
1111
- # resampler
1112
- "kv_proj",
362
]
363
364
embedding_modules = {}
365
embedding_padding_modules = []
366
1118
- def get_mm_mapping(self) -> MultiModelKeys:
1119
- """
1120
- Get the module prefix in multimodal models
1121
- """
1122
- return MultiModelKeys.from_string_field(
1123
- language_model="transformer.h",
1124
- connector="transformer.visual.attn_pool",
1125
- tower_model="transformer.visual.transformer")
1126
-
1127
-
1128
-@MULTIMODAL_REGISTRY.register_processor(QWenVLMultiModalProcessor,
1129
- info=QWenVLProcessingInfo,
1130
- dummy_inputs=QWenVLDummyInputsBuilder)
1131
-class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1132
- """
1133
- QWenLMHeadModel is not only applicable to LLM but also to VL, which is not
1134
- conducive to the current integration logic of LoRA in vLLM. Therefore, it
1135
- is necessary to separate them.
1136
- """
1137
- # Ensure that the LoRA support check passes when the class is not
1138
- # initialized, but set all these attributes to empty.
1139
- # These will be updated when an instance class is selected
1140
- packed_modules_mapping = {}
1141
- supported_lora_modules = []
1142
- embedding_modules = {}
1143
- embedding_padding_modules = []
1144
-
1145
- def __new__(
1146
- cls,
1147
- vllm_config: VllmConfig,
1148
- prefix: str = "",
1149
- ) -> QWenBaseModel:
367
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
368
config = vllm_config.model_config.hf_config
369
+ if hasattr(config, "visual"):
370
+ hf_overrides = {
371
+ "architectures": ["QwenVLForConditionalGeneration"]
372
+ }
373
+ raise RuntimeError(
374
+ "The configuration of this model indicates that it supports "
375
+ "vision inputs, but you instantiated the text-only version "
376
+ "of this model. Please use the vision model by setting "
377
+ f"`--hf-overrides {hf_overrides!r}`")
378
+
379
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
380
1152
- # Initialize VL
1153
- if hasattr(config, "visual"): # noqa: SIM108
1154
- instance_cls = QWenVL
1155
- # Initialize LLM
1156
- else:
1157
- instance_cls = QWenLLM
1158
-
1159
- # quant_config references base class members,
1160
- # so update values before init is called
1161
- cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
1162
- cls.supported_lora_modules += instance_cls.supported_lora_modules
1163
- cls.embedding_modules.update(instance_cls.embedding_modules)
1164
- cls.embedding_padding_modules += instance_cls.embedding_padding_modules
1165
- return instance_cls(vllm_config=vllm_config, prefix=prefix)
381
+ def forward(
382
+ self,
383
+ input_ids: torch.Tensor,
384
+ positions: torch.Tensor,
385
+ kv_caches: List[torch.Tensor],
386
+ attn_metadata: AttentionMetadata,
387
+ intermediate_tensors: Optional[IntermediateTensors] = None,
388
+ inputs_embeds: Optional[torch.Tensor] = None,
389
+ ) -> Union[torch.Tensor, IntermediateTensors]:
390
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
391
+ attn_metadata, intermediate_tensors,
392
+ inputs_embeds)
393
+ return hidden_states
vllm/model_executor/models/qwen_vl.py
new
+794
@@ -0,0 +1,794 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# Adapted from
4
+# https://huggingface.co/Qwen/Qwen-VL/blob/main/modeling_qwen.py
5
+# Copyright (c) Alibaba Cloud.
6
+"""Inference-only Qwen-VL model compatible with HuggingFace weights."""
7
+
8
+import copy
9
+import math
10
+import re
11
+import unicodedata
12
+from functools import lru_cache, partial
13
+from typing import (AbstractSet, Callable, Collection, List, Literal, Mapping,
14
+ Optional, TypedDict, Union)
15
+
16
+import torch
17
+from torch import nn
18
+from torchvision import transforms
19
+from torchvision.transforms import InterpolationMode
20
+from transformers import (BatchFeature, PretrainedConfig, PreTrainedTokenizer,
21
+ TensorType)
22
+from transformers.image_utils import ImageInput
23
+from transformers.tokenization_utils_base import TextInput
24
+
25
+from vllm.attention import AttentionMetadata
26
+from vllm.config import VllmConfig
27
+from vllm.model_executor.layers.activation import get_act_fn
28
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
+ ReplicatedLinear,
30
+ RowParallelLinear)
31
+from vllm.model_executor.layers.quantization import QuantizationConfig
32
+from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
33
+from vllm.model_executor.models.module_mapping import MultiModelKeys
34
+from vllm.multimodal import MULTIMODAL_REGISTRY
35
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
36
+ NestedTensors)
37
+from vllm.multimodal.parse import MultiModalDataItems
38
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
+ BaseProcessingInfo, PromptReplacement,
40
+ PromptReplacementDetails)
41
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
42
+from vllm.sequence import IntermediateTensors
43
+
44
+from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
45
+from .qwen import QWenBaseModel, QWenModel
46
+from .utils import flatten_bn, merge_multimodal_embeddings
47
+
48
+
49
+class QwenImagePixelInputs(TypedDict):
50
+ type: Literal["pixel_values"]
51
+ data: torch.Tensor
52
+ """
53
+ Shape: `(batch_size * num_images, 3, image_size, image_size)`
54
+
55
+ Note that image_size is the value in the vision config to which we resize
56
+ the image to in the normalization transform. Currently multi-image support
57
+ can only be leveraged by passing image embeddings directly.
58
+ """
59
+
60
+
61
+class QwenImageEmbeddingInputs(TypedDict):
62
+ type: Literal["image_embeds"]
63
+ data: torch.Tensor
64
+ """Shape: `(batch_size * num_images, 256, hidden_size)`
65
+
66
+ `hidden_size` must match the hidden size of the language model backbone
67
+ and is stored in the visual config of the model if we have one.
68
+ """
69
+
70
+
71
+QwenImageInputs = Union[QwenImagePixelInputs, QwenImageEmbeddingInputs]
72
+
73
+
74
+class VisualAttention(nn.Module):
75
+ """self-attention layer class.
76
+ Self-attention layer takes input with size [s, b, h]
77
+ and returns output of the same size.
78
+ """
79
+
80
+ def __init__(
81
+ self,
82
+ embed_dim: int,
83
+ num_heads: int,
84
+ bias: bool = True,
85
+ kdim: Optional[int] = None,
86
+ vdim: Optional[int] = None,
87
+ ):
88
+ super().__init__()
89
+ self.embed_dim = embed_dim
90
+ self.kdim = kdim if kdim is not None else embed_dim
91
+ self.vdim = vdim if vdim is not None else embed_dim
92
+ self._qkv_same_embed_dim = self.kdim == embed_dim \
93
+ and self.vdim == embed_dim
94
+
95
+ self.num_heads = num_heads
96
+
97
+ # Per attention head and per partition values.
98
+ assert embed_dim % num_heads == 0
99
+ self.hidden_size_per_attention_head = embed_dim // num_heads
100
+ self.num_attention_heads_per_partition = num_heads
101
+ self.hidden_size_per_partition = embed_dim
102
+
103
+ # Strided linear layer.
104
+ assert self._qkv_same_embed_dim, \
105
+ 'Visual Attention implementation only supports self-attention'
106
+ self.in_proj = ReplicatedLinear(embed_dim, 3 * embed_dim)
107
+ self.out_proj = ReplicatedLinear(embed_dim, embed_dim)
108
+ self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
109
+
110
+ def forward(
111
+ self,
112
+ x: torch.Tensor,
113
+ attn_mask: Optional[torch.Tensor] = None,
114
+ ) -> torch.Tensor:
115
+ # query/key/value: [sq, b, h]
116
+ sq, b, _ = x.size()
117
+ mixed_x_layer, _ = self.in_proj(x)
118
+
119
+ # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
120
+ new_tensor_shape = mixed_x_layer.size()[:-1] + \
121
+ (self.num_attention_heads_per_partition,
122
+ 3 * self.hidden_size_per_attention_head)
123
+ mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
124
+
125
+ # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
126
+ query_layer, key_layer, value_layer = mixed_x_layer.split(
127
+ self.hidden_size_per_attention_head, dim=-1)
128
+
129
+ # [sq, b, np, hn] -> [sq, b * np, hn]
130
+ query_layer = query_layer.view(
131
+ sq, b * self.num_attention_heads_per_partition,
132
+ self.hidden_size_per_attention_head).transpose(0, 1)
133
+ # [sk, b, np, hn] -> [sk, b * np, hn]
134
+ key_layer = key_layer.view(
135
+ sq, b * self.num_attention_heads_per_partition,
136
+ self.hidden_size_per_attention_head).transpose(0, 1)
137
+
138
+ q_scaled = query_layer / self.norm_factor
139
+ if attn_mask is not None:
140
+ attention_probs = torch.baddbmm(attn_mask, q_scaled,
141
+ key_layer.transpose(-2, -1))
142
+ else:
143
+ attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1))
144
+ attention_probs = attention_probs.softmax(dim=-1)
145
+
146
+ value_layer = value_layer.view(
147
+ sq, b * self.num_attention_heads_per_partition,
148
+ self.hidden_size_per_attention_head).transpose(0, 1)
149
+
150
+ # matmul: [b * np, sq, hn]
151
+ context_layer = torch.bmm(attention_probs, value_layer)
152
+
153
+ # change view [b, np, sq, hn]
154
+ context_layer = context_layer.view(
155
+ b, self.num_attention_heads_per_partition, sq,
156
+ self.hidden_size_per_attention_head)
157
+
158
+ # [b, np, sq, hn] --> [sq, b, np, hn]
159
+ context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
160
+
161
+ # [sq, b, np, hn] --> [sq, b, hp]
162
+ new_context_layer_shape = context_layer.size()[:-2] + \
163
+ (self.hidden_size_per_partition,)
164
+ context_layer = context_layer.view(*new_context_layer_shape)
165
+
166
+ output, _ = self.out_proj(context_layer)
167
+
168
+ return output
169
+
170
+
171
+class QwenVLMLP(nn.Module):
172
+ """MLP for the visual component of the Qwen model."""
173
+
174
+ def __init__(
175
+ self,
176
+ hidden_size: int,
177
+ intermediate_size: int,
178
+ quant_config: Optional[QuantizationConfig] = None,
179
+ ):
180
+ super().__init__()
181
+ self.c_fc = ColumnParallelLinear(hidden_size,
182
+ intermediate_size,
183
+ bias=True,
184
+ quant_config=quant_config)
185
+ self.act_fn = get_act_fn("gelu")
186
+ self.c_proj = RowParallelLinear(
187
+ intermediate_size,
188
+ hidden_size,
189
+ bias=True,
190
+ quant_config=quant_config,
191
+ )
192
+
193
+ def forward(self, x):
194
+ x, _ = self.c_fc(x)
195
+ x = self.act_fn(x)
196
+ x, _ = self.c_proj(x)
197
+ return x
198
+
199
+
200
+class VisualAttentionBlock(nn.Module):
201
+
202
+ def __init__(
203
+ self,
204
+ d_model: int,
205
+ n_head: int,
206
+ mlp_ratio: float = 4.0,
207
+ norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
208
+ quant_config: Optional[QuantizationConfig] = None,
209
+ ):
210
+ super().__init__()
211
+
212
+ self.ln_1 = norm_layer(d_model)
213
+ self.ln_2 = norm_layer(d_model)
214
+ mlp_width = int(d_model * mlp_ratio)
215
+ self.attn = VisualAttention(d_model, n_head)
216
+ self.mlp = QwenVLMLP(
217
+ hidden_size=d_model,
218
+ intermediate_size=mlp_width,
219
+ quant_config=quant_config,
220
+ )
221
+
222
+ def attention(
223
+ self,
224
+ x: torch.Tensor,
225
+ attn_mask: Optional[torch.Tensor] = None,
226
+ ) -> torch.Tensor:
227
+ attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None
228
+ return self.attn(x, attn_mask=attn_mask)
229
+
230
+ def forward(
231
+ self,
232
+ x: torch.Tensor,
233
+ attn_mask: Optional[torch.Tensor] = None,
234
+ ) -> torch.Tensor:
235
+ x = x + self.attention(self.ln_1(x), attn_mask=attn_mask)
236
+ x = x + self.mlp(self.ln_2(x))
237
+ return x
238
+
239
+
240
+class TransformerBlock(nn.Module):
241
+
242
+ def __init__(
243
+ self,
244
+ width: int,
245
+ layers: int,
246
+ heads: int,
247
+ mlp_ratio: float = 4.0,
248
+ norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
249
+ quant_config: Optional[QuantizationConfig] = None,
250
+ ):
251
+ super().__init__()
252
+ self.width = width
253
+ self.layers = layers
254
+
255
+ self.resblocks = nn.ModuleList([
256
+ VisualAttentionBlock(width,
257
+ heads,
258
+ mlp_ratio,
259
+ norm_layer=norm_layer,
260
+ quant_config=quant_config)
261
+ for _ in range(layers)
262
+ ])
263
+
264
+ def get_cast_dtype(self) -> torch.dtype:
265
+ return self.resblocks[0].mlp.c_fc.weight.dtype
266
+
267
+ def get_cast_device(self) -> torch.device:
268
+ return self.resblocks[0].mlp.c_fc.weight.device
269
+
270
+ def forward(self,
271
+ x: torch.Tensor,
272
+ attn_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
273
+ for r in self.resblocks:
274
+ x = r(x, attn_mask=attn_mask)
275
+ return x
276
+
277
+
278
+class VisionTransformer(nn.Module):
279
+
280
+ def __init__(self,
281
+ image_size: int,
282
+ patch_size: int,
283
+ width: int,
284
+ layers: int,
285
+ heads: int,
286
+ mlp_ratio: float,
287
+ n_queries: int = 256,
288
+ output_dim: int = 512,
289
+ image_start_id: int = 151857,
290
+ quant_config: Optional[QuantizationConfig] = None,
291
+ **kwargs):
292
+ super().__init__()
293
+ image_height, image_width = self.image_size = (image_size, image_size)
294
+ patch_height, patch_width = self.patch_size = (patch_size, patch_size)
295
+ self.grid_size = (image_height // patch_height,
296
+ image_width // patch_width)
297
+ self.output_dim = output_dim
298
+ self.conv1 = nn.Conv2d(in_channels=3,
299
+ out_channels=width,
300
+ kernel_size=patch_size,
301
+ stride=patch_size,
302
+ bias=False)
303
+
304
+ # class embeddings and positional embeddings
305
+ scale = width**-0.5
306
+ self.positional_embedding = nn.Parameter(scale *
307
+ torch.randn(256, width))
308
+
309
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
310
+
311
+ self.ln_pre = norm_layer(width)
312
+ self.transformer = TransformerBlock(width,
313
+ layers,
314
+ heads,
315
+ mlp_ratio,
316
+ norm_layer=norm_layer,
317
+ quant_config=quant_config)
318
+
319
+ self.attn_pool = Resampler2(
320
+ grid_size=int(math.sqrt(n_queries)),
321
+ embed_dim=output_dim,
322
+ num_heads=output_dim // 128,
323
+ kv_dim=width,
324
+ norm_layer=norm_layer,
325
+ adaptive=False,
326
+ do_post_projection=False,
327
+ ).to(
328
+ device=self.positional_embedding.device,
329
+ dtype=self.positional_embedding.dtype,
330
+ )
331
+
332
+ self.ln_post = norm_layer(output_dim)
333
+ self.proj = nn.Parameter(
334
+ (output_dim**-0.5) * torch.randn(output_dim, output_dim))
335
+
336
+ self.image_start_id = image_start_id
337
+ self.image_end_id = image_start_id + 1
338
+ self.image_pad_id = image_start_id + 2
339
+
340
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
341
+ x = x.to(
342
+ dtype=self.transformer.get_cast_dtype(),
343
+ device=self.transformer.get_cast_device(),
344
+ )
345
+
346
+ # to patches
347
+ x = self.conv1(x) # shape = [*, width, grid, grid]
348
+ x = x.reshape(x.shape[0], x.shape[1],
349
+ -1) # shape = [*, width, grid ** 2]
350
+ x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
351
+
352
+ x = x + get_abs_pos(self.positional_embedding, int(math.sqrt(
353
+ x.size(1))))
354
+
355
+ x = self.ln_pre(x)
356
+
357
+ x = x.permute(1, 0, 2) # NLD -> LND
358
+ x = self.transformer(x)
359
+ x = x.permute(1, 0, 2) # LND -> NLD
360
+
361
+ x = self.attn_pool(x)
362
+ x = self.ln_post(x)
363
+ x = x @ self.proj
364
+
365
+ return x
366
+
367
+
368
+class QwenVLModel(QWenModel):
369
+
370
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
371
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
372
+
373
+ config = vllm_config.model_config.hf_config
374
+ quant_config = vllm_config.quant_config
375
+
376
+ self.visual = VisionTransformer(**config.visual,
377
+ quant_config=quant_config)
378
+
379
+
380
+@lru_cache(maxsize=1)
381
+def _get_tokenizer_without_image_pad(
382
+ tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
383
+ """
384
+ The logic of adding image pad tokens should only be applied in
385
+ :class:`QwenVLProcessor`, so they are patched out here.
386
+
387
+ The definition of the wrapped tokenizer can be found here:
388
+ https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
389
+ """
390
+ new_tokenizer = copy.deepcopy(tokenizer)
391
+
392
+ class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
393
+
394
+ def tokenize(
395
+ self,
396
+ text: str,
397
+ allowed_special: Union[AbstractSet[str], str] = "all",
398
+ disallowed_special: Union[Collection[str], str] = (),
399
+ **kwargs,
400
+ ) -> list[Union[bytes, str]]:
401
+ text = unicodedata.normalize("NFC", text)
402
+
403
+ return [
404
+ self.decoder[t] for t in self.tokenizer.encode(
405
+ text,
406
+ allowed_special=allowed_special,
407
+ disallowed_special=disallowed_special,
408
+ )
409
+ ]
410
+
411
+ def _decode(
412
+ self,
413
+ token_ids: Union[int, List[int]],
414
+ skip_special_tokens: bool = False,
415
+ errors: Optional[str] = None,
416
+ **kwargs,
417
+ ) -> str:
418
+ if isinstance(token_ids, int):
419
+ token_ids = [token_ids]
420
+
421
+ return self.tokenizer.decode(
422
+ token_ids,
423
+ errors=errors or self.errors,
424
+ )
425
+
426
+ TokenizerWithoutImagePad.__name__ = \
427
+ f"{tokenizer.__class__.__name__}WithoutImagePad"
428
+
429
+ new_tokenizer.__class__ = TokenizerWithoutImagePad
430
+ return new_tokenizer
431
+
432
+
433
+class QwenVLProcessor:
434
+ """
435
+ This model doesn't define its own HF processor,
436
+ so we implement our own one here.
437
+
438
+ We call the wrapped tokenizer to automatically insert image pad tokens:
439
+ https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py#L245
440
+
441
+ The image processor is defined here:
442
+ https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
443
+ """
444
+
445
+ def __init__(
446
+ self,
447
+ config: PretrainedConfig,
448
+ tokenizer: PreTrainedTokenizer,
449
+ ) -> None:
450
+ super().__init__()
451
+
452
+ self.config = config
453
+ self.tokenizer = tokenizer
454
+
455
+ vision_config = config.visual
456
+ image_size = vision_config["image_size"]
457
+
458
+ self.image_transform = transforms.Compose([
459
+ transforms.Resize(
460
+ (image_size, image_size),
461
+ interpolation=InterpolationMode.BICUBIC,
462
+ ),
463
+ transforms.ToTensor(),
464
+ transforms.Normalize(
465
+ mean=(0.48145466, 0.4578275, 0.40821073),
466
+ std=(0.26862954, 0.26130258, 0.27577711),
467
+ ),
468
+ ])
469
+
470
+ @property
471
+ def image_start_tag(self) -> str:
472
+ return self.tokenizer.image_start_tag # type: ignore
473
+
474
+ @property
475
+ def image_end_tag(self) -> str:
476
+ return self.tokenizer.image_end_tag # type: ignore
477
+
478
+ @property
479
+ def image_pad_tag(self) -> str:
480
+ return self.tokenizer.image_pad_tag # type: ignore
481
+
482
+ def __call__(
483
+ self,
484
+ text: Optional[Union[TextInput, list[TextInput]]] = None,
485
+ images: Optional[Union[ImageInput, list[ImageInput]]] = None,
486
+ return_tensors: Optional[Union[str, TensorType]] = None,
487
+ ) -> BatchFeature:
488
+ if text is None:
489
+ text = []
490
+ if not isinstance(text, list):
491
+ text = [text]
492
+ if images is None:
493
+ images = []
494
+ if not isinstance(images, list):
495
+ images = [images]
496
+
497
+ text_inputs = self.tokenizer(text)
498
+
499
+ if len(images) == 0:
500
+ image_inputs = {}
501
+ else:
502
+ pixel_values = [self.image_transform(image) for image in images]
503
+ image_inputs = {"pixel_values": torch.stack(pixel_values)}
504
+
505
+ return BatchFeature(
506
+ {
507
+ **text_inputs,
508
+ **image_inputs,
509
+ },
510
+ tensor_type=return_tensors,
511
+ )
512
+
513
+
514
+class QwenVLProcessingInfo(BaseProcessingInfo):
515
+
516
+ def get_tokenizer(self) -> PreTrainedTokenizer:
517
+ tokenizer = self.ctx.tokenizer
518
+ assert isinstance(tokenizer, PreTrainedTokenizer)
519
+
520
+ return _get_tokenizer_without_image_pad(tokenizer)
521
+
522
+ def get_hf_processor(self) -> QwenVLProcessor:
523
+ tokenizer = self.ctx.tokenizer
524
+ assert isinstance(tokenizer, PreTrainedTokenizer)
525
+
526
+ return QwenVLProcessor(self.get_hf_config(), tokenizer)
527
+
528
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
529
+ return {"image": None}
530
+
531
+ def get_mm_max_tokens_per_item(
532
+ self,
533
+ seq_len: int,
534
+ mm_counts: Mapping[str, int],
535
+ ) -> Mapping[str, int]:
536
+ return {"image": self.get_num_image_tokens()}
537
+
538
+ def get_num_image_tokens(self) -> int:
539
+ hf_config = self.get_hf_config()
540
+ vision_config = hf_config.visual
541
+
542
+ image_size = vision_config["image_size"]
543
+ patch_size = vision_config["patch_size"]
544
+ grid_length = image_size // patch_size // 2
545
+ return grid_length * grid_length
546
+
547
+
548
+class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):
549
+
550
+ def get_dummy_processor_inputs(
551
+ self,
552
+ seq_len: int,
553
+ mm_counts: Mapping[str, int],
554
+ ) -> ProcessorInputs:
555
+ hf_config = self.info.get_hf_config()
556
+ vision_config = hf_config.visual
557
+
558
+ processor = self.info.get_hf_processor()
559
+ img_start = processor.image_start_tag
560
+ img_end = processor.image_end_tag
561
+
562
+ target_width = target_height = vision_config["image_size"]
563
+ num_images = mm_counts.get("image", 0)
564
+
565
+ mm_data = {
566
+ "image":
567
+ self._get_dummy_images(width=target_width,
568
+ height=target_height,
569
+ num_images=num_images)
570
+ }
571
+
572
+ return ProcessorInputs(
573
+ prompt_text="".join(f"Picture {i}: {img_start}{img_end}\n"
574
+ for i in range(1, num_images + 1)),
575
+ mm_data=mm_data,
576
+ )
577
+
578
+
579
+class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
580
+
581
+ def _call_hf_processor(
582
+ self,
583
+ prompt: str,
584
+ mm_data: Mapping[str, object],
585
+ mm_kwargs: Mapping[str, object],
586
+ ) -> BatchFeature:
587
+ # Drops anything between <img>/</img> tags; encoding with the tokenizer
588
+ # will automatically add the image pads for the context.
589
+ prompt, num_matched_images = re.subn(
590
+ r"(Picture \d*: <img>).*?(<\/img>\n)",
591
+ r"\1\2",
592
+ prompt,
593
+ )
594
+
595
+ image_data = mm_data.get("images")
596
+ if image_data is not None:
597
+ assert isinstance(image_data, list)
598
+
599
+ num_images = len(image_data)
600
+ assert num_matched_images == num_images
601
+
602
+ return super()._call_hf_processor(
603
+ prompt=prompt,
604
+ mm_data=mm_data,
605
+ mm_kwargs=mm_kwargs,
606
+ )
607
+
608
+ def _get_mm_fields_config(
609
+ self,
610
+ hf_inputs: BatchFeature,
611
+ hf_processor_mm_kwargs: Mapping[str, object],
612
+ ) -> Mapping[str, MultiModalFieldConfig]:
613
+ return dict(
614
+ pixel_values=MultiModalFieldConfig.batched("image"),
615
+ image_embeds=MultiModalFieldConfig.batched("image"),
616
+ )
617
+
618
+ def _get_prompt_replacements(
619
+ self,
620
+ mm_items: MultiModalDataItems,
621
+ hf_processor_mm_kwargs: Mapping[str, object],
622
+ out_mm_kwargs: MultiModalKwargs,
623
+ ) -> list[PromptReplacement]:
624
+ tokenizer = self.info.get_tokenizer()
625
+ special_tokens: dict[str,
626
+ int] = tokenizer.special_tokens # type: ignore
627
+
628
+ processor = self.info.get_hf_processor()
629
+ img_start_id = special_tokens[processor.image_start_tag]
630
+ img_end_id = special_tokens[processor.image_end_tag]
631
+ img_pad_id = special_tokens[processor.image_pad_tag]
632
+
633
+ num_image_tokens = self.info.get_num_image_tokens()
634
+ image_tokens = [img_pad_id] * num_image_tokens
635
+
636
+ return [
637
+ PromptReplacement(
638
+ modality="image",
639
+ target=[img_start_id, img_end_id],
640
+ replacement=PromptReplacementDetails(
641
+ full=[img_start_id] + image_tokens + [img_end_id],
642
+ features=image_tokens,
643
+ ),
644
+ )
645
+ ]
646
+
647
+
648
+@MULTIMODAL_REGISTRY.register_processor(QwenVLMultiModalProcessor,
649
+ info=QwenVLProcessingInfo,
650
+ dummy_inputs=QwenVLDummyInputsBuilder)
651
+class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,
652
+ SupportsMultiModal):
653
+ packed_modules_mapping = {
654
+ "c_attn": ["c_attn"],
655
+ "gate_up_proj": [
656
+ "w2",
657
+ "w1",
658
+ ],
659
+ }
660
+ # LoRA specific attributes
661
+ supported_lora_modules = [
662
+ "c_attn",
663
+ "gate_up_proj",
664
+ "c_proj",
665
+ # visual module
666
+ "out_proj",
667
+ "in_proj",
668
+ "c_fc",
669
+ # resampler
670
+ "kv_proj",
671
+ ]
672
+
673
+ embedding_modules = {}
674
+ embedding_padding_modules = []
675
+
676
+ def get_mm_mapping(self) -> MultiModelKeys:
677
+ """
678
+ Get the module prefix in multimodal models
679
+ """
680
+ return MultiModelKeys.from_string_field(
681
+ language_model="transformer.h",
682
+ connector="transformer.visual.attn_pool",
683
+ tower_model="transformer.visual.transformer")
684
+
685
+ def __init__(
686
+ self,
687
+ *,
688
+ vllm_config: VllmConfig,
689
+ prefix: str = "",
690
+ transformer_type: type[QwenVLModel] = QwenVLModel,
691
+ ) -> None:
692
+ super().__init__(
693
+ vllm_config=vllm_config,
694
+ prefix=prefix,
695
+ transformer_type=transformer_type,
696
+ )
697
+
698
+ self.transformer: QwenVLModel
699
+
700
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
701
+ h = w = self.config.visual["image_size"]
702
+ expected_dims = (3, h, w)
703
+ actual_dims = tuple(data.shape[1:])
704
+
705
+ if actual_dims != expected_dims:
706
+ expected_expr = ("batch_size", *map(str, expected_dims))
707
+ raise ValueError(
708
+ f"The expected shape of pixel values is {expected_expr}. "
709
+ f"You supplied {tuple(data.shape)}.")
710
+
711
+ return data
712
+
713
+ def _parse_and_validate_image_input(
714
+ self, **kwargs: object) -> Optional[QwenImageInputs]:
715
+ pixel_values = kwargs.pop("pixel_values", None)
716
+ image_embeds = kwargs.pop("image_embeds", None)
717
+
718
+ if pixel_values is not None:
719
+ if not isinstance(pixel_values, torch.Tensor):
720
+ raise ValueError("Incorrect type of pixel values. "
721
+ f"Got type: {type(pixel_values)}")
722
+
723
+ return QwenImagePixelInputs(
724
+ type="pixel_values",
725
+ data=self._validate_pixel_values(
726
+ flatten_bn(pixel_values, concat=True)),
727
+ )
728
+
729
+ if image_embeds is not None:
730
+ if not isinstance(image_embeds, torch.Tensor):
731
+ raise ValueError("Incorrect type of image embeddings. "
732
+ f"Got type: {type(image_embeds)}")
733
+
734
+ return QwenImageEmbeddingInputs(
735
+ type="image_embeds",
736
+ data=flatten_bn(image_embeds),
737
+ )
738
+
739
+ return None
740
+
741
+ def _process_image_input(self,
742
+ image_input: QwenImageInputs) -> torch.Tensor:
743
+ if image_input["type"] == "image_embeds":
744
+ return image_input["data"]
745
+
746
+ return self.transformer.visual(image_input["data"])
747
+
748
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
749
+ image_input = self._parse_and_validate_image_input(**kwargs)
750
+ if image_input is None:
751
+ return None
752
+
753
+ vision_embeddings = self._process_image_input(image_input)
754
+ return vision_embeddings
755
+
756
+ def get_input_embeddings(
757
+ self,
758
+ input_ids: torch.Tensor,
759
+ multimodal_embeddings: Optional[NestedTensors] = None,
760
+ ) -> torch.Tensor:
761
+ inputs_embeds = self.transformer.get_input_embeddings(input_ids)
762
+
763
+ if multimodal_embeddings is not None:
764
+ inputs_embeds = merge_multimodal_embeddings(
765
+ input_ids, inputs_embeds, multimodal_embeddings,
766
+ self.transformer.visual.image_pad_id)
767
+
768
+ return inputs_embeds
769
+
770
+ def forward(
771
+ self,
772
+ input_ids: torch.Tensor,
773
+ positions: torch.Tensor,
774
+ kv_caches: List[torch.Tensor],
775
+ attn_metadata: AttentionMetadata,
776
+ intermediate_tensors: Optional[IntermediateTensors] = None,
777
+ inputs_embeds: Optional[torch.Tensor] = None,
778
+ **kwargs: object,
779
+ ) -> Union[torch.Tensor, IntermediateTensors]:
780
+ if intermediate_tensors is not None:
781
+ inputs_embeds = None
782
+
783
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
784
+ # condition is for v0 compatibility.
785
+ elif inputs_embeds is None:
786
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
787
+ inputs_embeds = self.get_input_embeddings(input_ids,
788
+ vision_embeddings)
789
+ input_ids = None
790
+
791
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
792
+ attn_metadata, intermediate_tensors,
793
+ inputs_embeds)
794
+ return hidden_states
vllm/model_executor/models/registry.py
+4
-5
@@ -39,7 +39,7 @@ _TEXT_GENERATION_MODELS = {
39
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
40
"BambaForCausalLM": ("bamba", "BambaForCausalLM"),
41
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
42
- # ChatGLMModel supports multimodal
42
+ "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
43
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
44
"Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
45
"DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
@@ -90,7 +90,7 @@ _TEXT_GENERATION_MODELS = {
90
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
91
"Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
92
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
93
- # QWenLMHeadModel supports multimodal
93
+ "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
94
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
95
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
96
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
@@ -156,10 +156,9 @@ _MULTIMODAL_MODELS = {
156
"AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
157
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
158
"ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
159
- "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
160
- "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
159
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
160
"FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
161
+ "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
162
"H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
163
"InternVLChatModel": ("internvl", "InternVLChatModel"),
164
"Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
@@ -175,7 +174,7 @@ _MULTIMODAL_MODELS = {
174
"PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
175
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
176
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
178
- "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
177
+ "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), # noqa: E501
178
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
179
"Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"), # noqa: E501
180
"Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501