@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1bc3b5e71b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[VLM] Separate text-only and vision variants of the same model architecture (#13157)

Cyrus Leung committed Feb 13, 2025 at 22:19 UTC 1bc3b5e71b26d82e655f09d4d1733b7706ea3ff6
15 files changed +1729 -1643
docs/source/models/supported_models.md
+8 -9
@@ -699,10 +699,10 @@ See [this page](#generative-models) for more information on how to use generativ
699 *
700 * ✅︎
701 * ✅︎
702 -- * `DeepseekVLV2ForCausalLM`
702 +- * `DeepseekVLV2ForCausalLM`<sup>^</sup>
703 * DeepSeek-VL2
704 * T + I<sup>+</sup>
705 - * `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc. (see note)
705 + * `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc.
706 *
707 * ✅︎
708 * ✅︎
@@ -713,10 +713,10 @@ See [this page](#generative-models) for more information on how to use generativ
713 *
714 * ✅︎
715 * ✅︎
716 -- * `ChatGLMModel`
716 +- * `GLM4VForCausalLM`<sup>^</sup>
717 * GLM-4V
718 * T + I
719 - * `THUDM/glm-4v-9b` etc.
719 + * `THUDM/glm-4v-9b`, `THUDM/cogagent-9b-20241220` etc.
720 * ✅︎
721 * ✅︎
722 * ✅︎
@@ -825,7 +825,7 @@ See [this page](#generative-models) for more information on how to use generativ
825 *
826 * ✅︎
827 * ✅︎
828 -- * `QWenLMHeadModel`
828 +- * `QwenVLForConditionalGeneration`<sup>^</sup>
829 * Qwen-VL
830 * T + I<sup>E+</sup>
831 * `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc.
@@ -862,13 +862,12 @@ See [this page](#generative-models) for more information on how to use generativ
862 * ✅︎
863 :::
864
865 +<sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
866 +&nbsp;&nbsp;&nbsp;&nbsp;• For example, to use DeepSeek-VL2 series models:
867 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;`--hf-overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'`
868 <sup>E</sup> Pre-computed embeddings can be inputted for this modality.
869 <sup>+</sup> Multiple items can be inputted per text prompt for this modality.
870
868 -:::{note}
869 -To use DeepSeek-VL2 series models, you have to pass `--hf_overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'` when running vLLM.
870 -:::
871 -
871 :::{note}
872 H2O-VL series models will be available in V1 once we support backends other than FlashAttention.
873 :::
examples/offline_inference/vision_language.py
+3
@@ -105,7 +105,9 @@ def run_glm4v(question: str, modality: str):
105 max_num_seqs=2,
106 trust_remote_code=True,
107 enforce_eager=True,
108 + hf_overrides={"architectures": ["GLM4VForCausalLM"]},
109 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
110 +
111 prompt = f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
112 {question}<|assistant|>"
113
@@ -495,6 +497,7 @@ def run_qwen_vl(question: str, modality: str):
497 trust_remote_code=True,
498 max_model_len=1024,
499 max_num_seqs=2,
500 + hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
501 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
502 )
503
examples/offline_inference/vision_language_multi_image.py
+3 -2
@@ -77,7 +77,7 @@ def load_deepseek_vl2(question: str, image_urls: List[str]):
77 )
78
79
80 -def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
80 +def load_h2ovl(question: str, image_urls: List[str]) -> ModelRequestData:
81 model_name = "h2oai/h2ovl-mississippi-2b"
82
83 llm = LLM(
@@ -302,6 +302,7 @@ def load_qwen_vl_chat(question: str,
302 trust_remote_code=True,
303 max_model_len=1024,
304 max_num_seqs=2,
305 + hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
306 limit_mm_per_prompt={"image": len(image_urls)},
307 )
308 placeholders = "".join(f"Picture {i}: <img></img>\n"
@@ -452,7 +453,7 @@ def load_qwen2_5_vl(question, image_urls: List[str]) -> ModelRequestData:
453 model_example_map = {
454 "aria": load_aria,
455 "deepseek_vl_v2": load_deepseek_vl2,
455 - "h2ovl_chat": load_h2onvl,
456 + "h2ovl_chat": load_h2ovl,
457 "idefics3": load_idefics3,
458 "internvl_chat": load_internvl,
459 "mllama": load_mllama,
tests/distributed/test_pipeline_parallel.py
+87 -84
@@ -6,6 +6,7 @@ WARNING: This test runs in both single-node (4 GPUs) and multi-node
6 all workers in a node other than the head node, which can cause the test
7 to fail.
8 """
9 +import json
10 import os
11 from dataclasses import dataclass
12 from typing import List, Literal, NamedTuple, Optional
@@ -15,6 +16,7 @@ import pytest
16 from vllm.config import TaskOption
17 from vllm.logger import init_logger
18
19 +from ..models.registry import HF_EXAMPLE_MODELS
20 from ..utils import compare_two_settings, fork_new_process_for_each_test
21
22 logger = init_logger("test_pipeline_parallel")
@@ -31,10 +33,7 @@ class ParallelSetup(NamedTuple):
33
34 class PPTestOptions(NamedTuple):
35 multi_node_only: bool
34 - trust_remote_code: bool
35 - tokenizer_mode: Optional[str]
36 load_format: Optional[str] = None
37 - hf_overrides: Optional[str] = None
37
38
39 @dataclass
@@ -64,10 +63,7 @@ class PPTestSettings:
63 pp_base: int = 2,
64 multi_node_only: bool = False,
65 task: TaskOption = "auto",
67 - trust_remote_code: bool = False,
68 - tokenizer_mode: Optional[str] = None,
66 load_format: Optional[str] = None,
70 - hf_overrides: Optional[str] = None,
67 ):
68 return PPTestSettings(
69 parallel_setups=[
@@ -97,10 +93,7 @@ class PPTestSettings:
93 vllm_major_versions=["0", "0", "1"],
94 task=task,
95 test_options=PPTestOptions(multi_node_only=multi_node_only,
100 - trust_remote_code=trust_remote_code,
101 - tokenizer_mode=tokenizer_mode,
102 - load_format=load_format,
103 - hf_overrides=hf_overrides),
96 + load_format=load_format),
97 )
98
99 @staticmethod
@@ -110,10 +103,7 @@ class PPTestSettings:
103 pp_base: int = 2,
104 task: TaskOption = "auto",
105 multi_node_only: bool = False,
113 - trust_remote_code: bool = False,
114 - tokenizer_mode: Optional[str] = None,
106 load_format: Optional[str] = None,
116 - hf_overrides: Optional[str] = None,
107 ):
108 return PPTestSettings(
109 parallel_setups=[
@@ -126,19 +116,16 @@ class PPTestSettings:
116 vllm_major_versions=["0"],
117 task=task,
118 test_options=PPTestOptions(multi_node_only=multi_node_only,
129 - trust_remote_code=trust_remote_code,
130 - tokenizer_mode=tokenizer_mode,
131 - load_format=load_format,
132 - hf_overrides=hf_overrides),
119 + load_format=load_format),
120 )
121
135 - def iter_params(self, model_name: str):
122 + def iter_params(self, model_id: str):
123 opts = self.test_options
124
125 for parallel_setup in self.parallel_setups:
126 for backend, vllm_major_version in zip(self.distributed_backends,
127 self.vllm_major_versions):
141 - yield (model_name, parallel_setup, backend, vllm_major_version,
128 + yield (model_id, parallel_setup, backend, vllm_major_version,
129 self.task, opts)
130
131
@@ -150,16 +137,16 @@ TEXT_GENERATION_MODELS = {
137 # [Decoder-only]
138 # Uses Llama
139 # "BAAI/AquilaChat-7B": PPTestSettings.fast(),
153 - "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(tp_base=8, trust_remote_code=True), # noqa: E501
154 - "baichuan-inc/Baichuan-7B": PPTestSettings.fast(trust_remote_code=True),
155 - "baichuan-inc/Baichuan2-13B-Chat": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
140 + "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(load_format="dummy"), # noqa: E501
141 + "baichuan-inc/Baichuan-7B": PPTestSettings.fast(),
142 + "baichuan-inc/Baichuan2-13B-Chat": PPTestSettings.fast(),
143 "bigscience/bloomz-1b1": PPTestSettings.fast(),
157 - "THUDM/chatglm3-6b": PPTestSettings.fast(trust_remote_code=True),
158 - "CohereForAI/c4ai-command-r-v01": PPTestSettings.fast(tp_base=2, trust_remote_code=True), # noqa: E501
159 - "databricks/dbrx-instruct": PPTestSettings.fast(tp_base=8),
160 - "Deci/DeciLM-7B-instruct": PPTestSettings.fast(trust_remote_code=True),
144 + "THUDM/chatglm3-6b": PPTestSettings.fast(),
145 + "CohereForAI/c4ai-command-r-v01": PPTestSettings.fast(load_format="dummy"),
146 + "databricks/dbrx-instruct": PPTestSettings.fast(load_format="dummy"),
147 + "Deci/DeciLM-7B-instruct": PPTestSettings.fast(),
148 "deepseek-ai/deepseek-llm-7b-chat": PPTestSettings.fast(),
162 - "deepseek-ai/DeepSeek-V2-Lite-Chat": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
149 + "deepseek-ai/DeepSeek-V2-Lite-Chat": PPTestSettings.fast(),
150 "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct": PPTestSettings.fast(),
151 "tiiuae/falcon-7b": PPTestSettings.fast(),
152 "google/gemma-2b": PPTestSettings.fast(),
@@ -172,36 +159,36 @@ TEXT_GENERATION_MODELS = {
159 "ibm/PowerMoE-3b": PPTestSettings.fast(),
160 # Uses Llama
161 # "internlm/internlm-chat-7b": PPTestSettings.fast(),
175 - "internlm/internlm2-chat-7b": PPTestSettings.fast(trust_remote_code=True),
162 + "internlm/internlm2-chat-7b": PPTestSettings.fast(),
163 "inceptionai/jais-13b-chat": PPTestSettings.fast(),
164 "ai21labs/Jamba-tiny-dev": PPTestSettings.fast(),
165 "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
179 - "openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(trust_remote_code=True),
180 - "openbmb/MiniCPM3-4B": PPTestSettings.fast(trust_remote_code=True),
166 + "openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(),
167 + "openbmb/MiniCPM3-4B": PPTestSettings.fast(),
168 # Uses Llama
169 # "mistralai/Mistral-7B-Instruct-v0.1": PPTestSettings.fast(),
170 "state-spaces/mamba-130m-hf": PPTestSettings.fast(),
184 - "mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(tp_base=4),
171 + "mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(load_format="dummy"), # noqa: E501
172 "mosaicml/mpt-7b": PPTestSettings.fast(),
173 "nvidia/Minitron-8B-Base": PPTestSettings.fast(),
174 "allenai/OLMo-1B-hf": PPTestSettings.fast(),
175 "shanearora/OLMo-7B-1124-hf": PPTestSettings.fast(),
176 "allenai/OLMoE-1B-7B-0924-Instruct": PPTestSettings.fast(),
177 "facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
191 - "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
178 + "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(),
179 "adept/persimmon-8b-chat": PPTestSettings.fast(),
180 "microsoft/phi-2": PPTestSettings.fast(),
194 - "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
195 - "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True, load_format="dummy", hf_overrides='{"num_hidden_layers": 4, "hidden_size": 512, "intermediate_size": 800, "num_attention_heads": 4, "num_key_value_heads": 1}'), # noqa: E501
196 - "Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
181 + "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(),
182 + "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(multi_node_only=True, load_format="dummy"), # noqa: E501
183 + "Qwen/Qwen-7B-Chat": PPTestSettings.fast(),
184 "Qwen/Qwen2-7B-Instruct": PPTestSettings.fast(),
185 "Qwen/Qwen1.5-MoE-A2.7B-Chat": PPTestSettings.fast(),
186 "stabilityai/stablelm-3b-4e1t": PPTestSettings.fast(),
187 "bigcode/starcoder2-3b": PPTestSettings.fast(),
201 - "upstage/solar-pro-preview-instruct": PPTestSettings.fast(tp_base=2),
188 + "upstage/solar-pro-preview-instruct": PPTestSettings.fast(load_format="dummy"), # noqa: E501
189 # FIXME: Cannot load tokenizer in latest transformers version.
190 # Need to use tokenizer from `meta-llama/Llama-2-7b-chat-hf`
204 - # "xverse/XVERSE-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
191 + # "xverse/XVERSE-7B-Chat": PPTestSettings.fast(),
192 # [Encoder-only]
193 # TODO: Implement PP
194 # "facebook/bart-base": PPTestSettings.fast(),
@@ -211,7 +198,7 @@ EMBEDDING_MODELS = { # type: ignore[var-annotated]
198 # [Text-only]
199 "intfloat/e5-mistral-7b-instruct": PPTestSettings.fast(),
200 "BAAI/bge-multilingual-gemma2": PPTestSettings.fast(),
214 - "Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(tp_base=4, trust_remote_code=True), # noqa: E501
201 + "Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(load_format="dummy"),
202 }
203
204 MULTIMODAL_MODELS = {
@@ -219,20 +206,20 @@ MULTIMODAL_MODELS = {
206 "Salesforce/blip2-opt-2.7b": PPTestSettings.fast(),
207 "facebook/chameleon-7b": PPTestSettings.fast(),
208 "adept/fuyu-8b": PPTestSettings.fast(),
222 - "THUDM/glm-4v-9b": PPTestSettings.fast(trust_remote_code=True),
223 - "OpenGVLab/InternVL2-1B": PPTestSettings.fast(trust_remote_code=True),
209 + "THUDM/glm-4v-9b": PPTestSettings.fast(),
210 + "OpenGVLab/InternVL2-1B": PPTestSettings.fast(),
211 "llava-hf/llava-1.5-7b-hf": PPTestSettings.fast(),
212 "llava-hf/llava-v1.6-mistral-7b-hf": PPTestSettings.fast(),
213 "llava-hf/LLaVA-NeXT-Video-7B-hf": PPTestSettings.fast(),
214 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf": PPTestSettings.fast(),
228 - "openbmb/MiniCPM-Llama3-V-2_5": PPTestSettings.fast(trust_remote_code=True),
229 - "allenai/Molmo-7B-D-0924": PPTestSettings.fast(trust_remote_code=True),
230 - "microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
231 - "mistralai/Pixtral-12B-2409": PPTestSettings.fast(tp_base=2, tokenizer_mode="mistral"), # noqa: E501
232 - "Qwen/Qwen-VL-Chat": PPTestSettings.fast(trust_remote_code=True),
215 + "openbmb/MiniCPM-Llama3-V-2_5": PPTestSettings.fast(),
216 + "allenai/Molmo-7B-D-0924": PPTestSettings.fast(),
217 + "microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(),
218 + "mistralai/Pixtral-12B-2409": PPTestSettings.fast(load_format="dummy"),
219 + "Qwen/Qwen-VL-Chat": PPTestSettings.fast(),
220 "Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
221 "Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
235 - "fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
222 + "fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(),
223 # [Encoder-decoder]
224 # TODO: Implement PP
225 # "meta-llama/Llama-3.2-11B-Vision-Instruct": PPTestSettings.fast(),
@@ -258,7 +245,7 @@ TEST_MODELS = [
245
246
247 def _compare_tp(
261 - model_name: str,
248 + model_id: str,
249 parallel_setup: ParallelSetup,
250 distributed_backend: str,
251 vllm_major_version: str,
@@ -267,6 +254,7 @@ def _compare_tp(
254 num_gpus_available: int,
255 *,
256 method: Literal["generate", "encode"],
257 + is_multimodal: bool,
258 ):
259 (
260 tp_size,
@@ -274,13 +262,32 @@ def _compare_tp(
262 eager_mode,
263 chunked_prefill,
264 ) = parallel_setup
277 - (
278 - multi_node_only,
279 - trust_remote_code,
280 - tokenizer_mode,
281 - load_format,
282 - hf_overrides,
283 - ) = test_options
265 +
266 + multi_node_only, load_format = test_options
267 +
268 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
269 + model_info.check_transformers_version(on_fail="skip")
270 +
271 + trust_remote_code = model_info.trust_remote_code
272 + tokenizer_mode = model_info.tokenizer_mode
273 + hf_overrides = model_info.hf_overrides
274 +
275 + if load_format == "dummy":
276 + # Avoid OOM
277 + text_overrides = {
278 + "num_layers": 1,
279 + "num_hidden_layers": 1,
280 + "num_experts": 2,
281 + "num_experts_per_tok": 2,
282 + "num_local_experts": 2,
283 + }
284 +
285 + if is_multimodal:
286 + hf_overrides.update({"text_config": text_overrides})
287 + else:
288 + hf_overrides.update(text_overrides)
289 + else:
290 + model_info.check_available_online(on_fail="skip")
291
292 if num_gpus_available < tp_size * pp_size:
293 pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
@@ -312,7 +319,7 @@ def _compare_tp(
319 if load_format:
320 common_args.extend(["--load-format", load_format])
321 if hf_overrides:
315 - common_args.extend(["--hf-overrides", hf_overrides])
322 + common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
323
324 specific_case = tp_size == 2 and pp_size == 2 and chunked_prefill
325 if distributed_backend == "ray" and (vllm_major_version == "1"
@@ -355,11 +362,7 @@ def _compare_tp(
362 ]
363
364 try:
358 - compare_two_settings(model_name,
359 - pp_args,
360 - tp_args,
361 - pp_env,
362 - method=method)
365 + compare_two_settings(model_id, pp_args, tp_args, pp_env, method=method)
366 except Exception:
367 if pp_env is None:
368 raise
@@ -369,17 +372,16 @@ def _compare_tp(
372
373
374 @pytest.mark.parametrize(
372 - ("model_name", "parallel_setup", "distributed_backend",
373 - "vllm_major_version", "task", "test_options"),
375 + ("model_id", "parallel_setup", "distributed_backend", "vllm_major_version",
376 + "task", "test_options"),
377 [
375 - params for model_name, settings in TEXT_GENERATION_MODELS.items()
376 - for params in settings.iter_params(model_name)
377 - if model_name in TEST_MODELS
378 + params for model_id, settings in TEXT_GENERATION_MODELS.items()
379 + for params in settings.iter_params(model_id) if model_id in TEST_MODELS
380 ],
381 )
382 @fork_new_process_for_each_test
383 def test_tp_language_generation(
382 - model_name: str,
384 + model_id: str,
385 parallel_setup: ParallelSetup,
386 distributed_backend: str,
387 vllm_major_version: str,
@@ -387,28 +389,28 @@ def test_tp_language_generation(
389 test_options: PPTestOptions,
390 num_gpus_available,
391 ):
390 - _compare_tp(model_name,
392 + _compare_tp(model_id,
393 parallel_setup,
394 distributed_backend,
395 vllm_major_version,
396 task,
397 test_options,
398 num_gpus_available,
397 - method="generate")
399 + method="generate",
400 + is_multimodal=False)
401
402
403 @pytest.mark.parametrize(
401 - ("model_name", "parallel_setup", "distributed_backend",
402 - "vllm_major_version", "task", "test_options"),
404 + ("model_id", "parallel_setup", "distributed_backend", "vllm_major_version",
405 + "task", "test_options"),
406 [
404 - params for model_name, settings in EMBEDDING_MODELS.items()
405 - for params in settings.iter_params(model_name)
406 - if model_name in TEST_MODELS
407 + params for model_id, settings in EMBEDDING_MODELS.items()
408 + for params in settings.iter_params(model_id) if model_id in TEST_MODELS
409 ],
410 )
411 @fork_new_process_for_each_test
412 def test_tp_language_embedding(
411 - model_name: str,
413 + model_id: str,
414 parallel_setup: ParallelSetup,
415 distributed_backend: str,
416 vllm_major_version: str,
@@ -416,28 +418,28 @@ def test_tp_language_embedding(
418 test_options: PPTestOptions,
419 num_gpus_available,
420 ):
419 - _compare_tp(model_name,
421 + _compare_tp(model_id,
422 parallel_setup,
423 distributed_backend,
424 vllm_major_version,
425 task,
426 test_options,
427 num_gpus_available,
426 - method="encode")
428 + method="encode",
429 + is_multimodal=False)
430
431
432 @pytest.mark.parametrize(
430 - ("model_name", "parallel_setup", "distributed_backend",
431 - "vllm_major_version", "task", "test_options"),
433 + ("model_id", "parallel_setup", "distributed_backend", "vllm_major_version",
434 + "task", "test_options"),
435 [
433 - params for model_name, settings in MULTIMODAL_MODELS.items()
434 - for params in settings.iter_params(model_name)
435 - if model_name in TEST_MODELS
436 + params for model_id, settings in MULTIMODAL_MODELS.items()
437 + for params in settings.iter_params(model_id) if model_id in TEST_MODELS
438 ],
439 )
440 @fork_new_process_for_each_test
441 def test_tp_multimodal_generation(
440 - model_name: str,
442 + model_id: str,
443 parallel_setup: ParallelSetup,
444 distributed_backend: str,
445 vllm_major_version: str,
@@ -445,11 +447,12 @@ def test_tp_multimodal_generation(
447 test_options: PPTestOptions,
448 num_gpus_available,
449 ):
448 - _compare_tp(model_name,
450 + _compare_tp(model_id,
451 parallel_setup,
452 distributed_backend,
453 vllm_major_version,
454 task,
455 test_options,
456 num_gpus_available,
455 - method="generate")
457 + method="generate",
458 + is_multimodal=True)
tests/models/decoder_only/vision_language/test_models.py
+3 -8
@@ -155,10 +155,7 @@ VLM_TEST_SETTINGS = {
155 auto_cls=AutoModelForVision2Seq,
156 vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
157 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
158 - marks=[pytest.mark.skipif(
159 - TRANSFORMERS_VERSION < "4.49.0",
160 - reason="HF model requires transformers>=4.49.0",
161 - ), pytest.mark.core_model, pytest.mark.cpu_model],
158 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
159 ),
160 #### Extended model tests
161 "aria": VLMTestInfo(
@@ -215,7 +212,6 @@ VLM_TEST_SETTINGS = {
212 "cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
213 }),
214 multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
218 - vllm_runner_kwargs={"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]}}, # noqa: E501
215 patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
216 postprocess_inputs=model_utils.cast_dtype_post_processor("images"),
217 hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
@@ -240,7 +236,7 @@ VLM_TEST_SETTINGS = {
236 num_logprobs=10,
237 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
238 ),
243 - "glm4": VLMTestInfo(
239 + "glm4v": VLMTestInfo(
240 models=["THUDM/glm-4v-9b"],
241 test_type=VLMTestType.IMAGE,
242 prompt_formatter=identity,
@@ -351,7 +347,6 @@ VLM_TEST_SETTINGS = {
347 postprocess_inputs=model_utils.cast_dtype_post_processor(
348 "pixel_values"
349 ),
354 - vllm_runner_kwargs={"hf_overrides": {"architectures": ["MantisForConditionalGeneration"]}}, # noqa: E501
350 get_stop_token_ids=lambda tok: [128009],
351 auto_cls=AutoModelForVision2Seq,
352 vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
@@ -437,7 +432,7 @@ VLM_TEST_SETTINGS = {
432 auto_cls=AutoModelForVision2Seq,
433 marks=[large_gpu_mark(min_gb=48)],
434 ),
440 - "qwen": VLMTestInfo(
435 + "qwen_vl": VLMTestInfo(
436 models=["Qwen/Qwen-VL"],
437 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
438 prompt_formatter=identity,
tests/models/decoder_only/vision_language/vlm_utils/core.py
+39 -23
@@ -4,12 +4,14 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
4
5 import torch
6 from PIL.Image import Image
7 -from transformers import AutoTokenizer, BatchEncoding, PreTrainedTokenizerBase
7 +from transformers import BatchEncoding
8 from transformers.models.auto.auto_factory import _BaseAutoModelClass
9
10 from vllm.config import TaskOption
11 +from vllm.transformers_utils.tokenizer import AnyTokenizer
12
13 from .....conftest import HfRunner, VllmRunner
14 +from ....registry import HF_EXAMPLE_MODELS
15 from .types import RunnerOutput
16
17
@@ -31,10 +33,8 @@ def run_test(
33 use_tokenizer_eos: bool,
34 postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],
35 comparator: Callable[..., None],
34 - get_stop_token_ids: Optional[Callable[[PreTrainedTokenizerBase],
35 - List[int]]],
36 + get_stop_token_ids: Optional[Callable[[AnyTokenizer], list[int]]],
37 stop_str: Optional[List[str]],
37 - tokenizer_mode: str,
38 limit_mm_per_prompt: Dict[str, int],
39 vllm_runner_kwargs: Optional[Dict[str, Any]],
40 hf_model_kwargs: Optional[Dict[str, Any]],
@@ -48,7 +48,10 @@ def run_test(
48 """Modality agnostic test test executor for comparing HF/vLLM outputs."""
49 # In the case of embeddings, vLLM takes separate input tensors
50 vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs
51 - tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
51 +
52 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
53 + model_info.check_available_online(on_fail="skip")
54 + model_info.check_transformers_version(on_fail="skip")
55
56 vllm_outputs_per_mm = []
57 hf_outputs_per_mm = []
@@ -57,17 +60,19 @@ def run_test(
60 # vLLM needs a fresh new process without cuda initialization.
61 # if we run HF first, the cuda initialization will be done and it
62 # will hurt multiprocessing backend with fork method (the default method).
60 - vllm_kwargs: Dict[str, Any] = {}
61 - if get_stop_token_ids is not None:
62 - vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
63 - if stop_str:
64 - vllm_kwargs["stop"] = stop_str
63
66 - if vllm_runner_kwargs is None:
67 - vllm_runner_kwargs = {}
64 + vllm_runner_kwargs_: Dict[str, Any] = {}
65 + if model_info.tokenizer:
66 + vllm_runner_kwargs_["tokenizer"] = model_info.tokenizer
67 + if model_info.tokenizer_mode:
68 + vllm_runner_kwargs_["tokenizer_mode"] = model_info.tokenizer_mode
69 + if model_info.hf_overrides:
70 + vllm_runner_kwargs_["hf_overrides"] = model_info.hf_overrides
71 +
72 + if vllm_runner_kwargs:
73 + vllm_runner_kwargs_.update(vllm_runner_kwargs)
74
75 with vllm_runner(model,
70 - tokenizer_mode=tokenizer_mode,
76 max_model_len=max_model_len,
77 max_num_seqs=max_num_seqs,
78 dtype=dtype,
@@ -76,7 +81,15 @@ def run_test(
81 distributed_executor_backend=distributed_executor_backend,
82 enforce_eager=enforce_eager,
83 task=task,
79 - **vllm_runner_kwargs) as vllm_model:
84 + **vllm_runner_kwargs_) as vllm_model:
85 + tokenizer = vllm_model.model.get_tokenizer()
86 +
87 + vllm_kwargs: Dict[str, Any] = {}
88 + if get_stop_token_ids is not None:
89 + vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
90 + if stop_str:
91 + vllm_kwargs["stop"] = stop_str
92 +
93 for prompts, media in vllm_inputs:
94 vllm_kwargs[runner_mm_key] = media
95 vllm_output = vllm_model.generate_greedy_logprobs(
@@ -93,16 +106,19 @@ def run_test(
106 if patch_hf_runner is not None:
107 hf_model = patch_hf_runner(hf_model)
108
96 - # Some models need to explicitly pass the eos_token_id off the tokenizer or
97 - # processor for a good comparison; currently assume processor/tokenizer
98 - # agree on the EOS, and pull it off the tokenizer if requested.
99 - hf_kwargs = {}
100 - if use_tokenizer_eos:
101 - hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
102 - if stop_str:
103 - hf_kwargs["stop_strings"] = stop_str
104 -
109 with hf_model, torch.no_grad():
110 + tokenizer = hf_model.tokenizer
111 +
112 + # Some models need to explicitly pass the eos_token_id off the tokenizer
113 + # or processor for a good comparison;
114 + # currently assume processor/tokenizer agree on the EOS, and pull it off
115 + # the tokenizer if requested.
116 + hf_kwargs = {}
117 + if use_tokenizer_eos:
118 + hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
119 + if stop_str:
120 + hf_kwargs["stop_strings"] = stop_str
121 +
122 for prompts, media in inputs:
123 hf_kwargs[runner_mm_key] = media
124 hf_output = hf_model.generate_greedy_logprobs_limit(
tests/models/decoder_only/vision_language/vlm_utils/types.py
+3 -7
@@ -8,12 +8,12 @@ from typing import (Any, Callable, Dict, Iterable, List, NamedTuple, Optional,
8 import torch
9 from PIL.Image import Image
10 from pytest import MarkDecorator
11 -from transformers import (AutoModelForCausalLM, BatchEncoding,
12 - PreTrainedTokenizerBase)
11 +from transformers import AutoModelForCausalLM, BatchEncoding
12 from transformers.models.auto.auto_factory import _BaseAutoModelClass
13
14 from vllm.config import TaskOption
15 from vllm.sequence import SampleLogprobs
16 +from vllm.transformers_utils.tokenizer import AnyTokenizer
17 from vllm.utils import identity
18
19 from .....conftest import IMAGE_ASSETS, HfRunner, ImageAsset, _ImageAssets
@@ -100,8 +100,7 @@ class VLMTestInfo(NamedTuple):
100 vllm_runner_kwargs: Optional[Dict[str, Any]] = None
101
102 # Optional callable which gets a list of token IDs from the model tokenizer
103 - get_stop_token_ids: Optional[Callable[[PreTrainedTokenizerBase],
104 - List[int]]] = None
103 + get_stop_token_ids: Optional[Callable[[AnyTokenizer], list[int]]] = None
104 # Optional list of strings to stop generation, useful when stop tokens are
105 # not special tokens in the tokenizer
106 stop_str: Optional[List[str]] = None
@@ -156,8 +155,6 @@ class VLMTestInfo(NamedTuple):
155
156 marks: Optional[List[MarkDecorator]] = None
157
159 - tokenizer_mode: str = "auto"
160 -
158 def get_non_parametrized_runner_kwargs(self):
159 """Returns a dictionary of expandable kwargs for items that are used
160 in all test types, which are NOT used when creating the parametrized
@@ -180,7 +177,6 @@ class VLMTestInfo(NamedTuple):
177 "hf_model_kwargs": self.hf_model_kwargs,
178 "stop_str": self.stop_str,
179 "patch_hf_runner": self.patch_hf_runner,
183 - "tokenizer_mode": self.tokenizer_mode
180 }
181
182
tests/models/registry.py
+22 -15
@@ -104,7 +104,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
104 trust_remote_code=True),
105 "BambaForCausalLM": _HfExamplesInfo("ibm-ai-platform/Bamba-9B"),
106 "BloomForCausalLM": _HfExamplesInfo("bigscience/bloomz-1b1"),
107 - # ChatGLMModel supports multimodal
107 + "ChatGLMModel": _HfExamplesInfo("THUDM/chatglm3-6b",
108 + trust_remote_code=True),
109 "CohereForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r-v01",
110 trust_remote_code=True),
111 "Cohere2ForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r7b-12-2024", # noqa: E501
@@ -138,7 +139,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
139 "InternLM3ForCausalLM": _HfExamplesInfo("internlm/internlm3-8b-instruct",
140 trust_remote_code=True),
141 "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
141 - "JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini"),
142 + "JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini",
143 + extras={"tiny": "ai21labs/Jamba-tiny-dev"}), # noqa: E501
144 "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Meta-Llama-3-8B"),
145 "LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
146 is_available_online=False),
@@ -167,7 +169,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
169 trust_remote_code=True),
170 "PhiMoEForCausalLM": _HfExamplesInfo("microsoft/Phi-3.5-MoE-instruct",
171 trust_remote_code=True),
170 - # QWenLMHeadModel supports multimodal
172 + "QWenLMHeadModel": _HfExamplesInfo("Qwen/Qwen-7B-Chat",
173 + trust_remote_code=True),
174 "Qwen2ForCausalLM": _HfExamplesInfo("Qwen/Qwen2-7B-Instruct"),
175 "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
176 "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b",
@@ -232,18 +235,19 @@ _MULTIMODAL_EXAMPLE_MODELS = {
235 "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
236 "Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b"), # noqa: E501
237 "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501
235 - "ChatGLMModel": _HfExamplesInfo("THUDM/glm-4v-9b",
236 - extras={"text_only": "THUDM/chatglm3-6b"},
237 - trust_remote_code=True),
238 - "ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
239 - is_available_online=False),
238 "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny", # noqa: E501
239 hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]}), # noqa: E501
240 "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
243 - "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
241 + "GLM4VForCausalLM": _HfExamplesInfo("THUDM/glm-4v-9b",
242 + trust_remote_code=True,
243 + hf_overrides={"architectures": ["GLM4VForCausalLM"]}), # noqa: E501
244 + "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m",
245 + extras={"2b": "h2oai/h2ovl-mississippi-2b"}), # noqa: E501
246 "InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
247 + extras={"2B": "OpenGVLab/InternVL2-2B"}, # noqa: E501
248 trust_remote_code=True),
246 - "Idefics3ForConditionalGeneration": _HfExamplesInfo("HuggingFaceM4/Idefics3-8B-Llama3"), # noqa: E501
249 + "Idefics3ForConditionalGeneration": _HfExamplesInfo("HuggingFaceM4/Idefics3-8B-Llama3", # noqa: E501
250 + {"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"}), # noqa: E501
251 "LlavaForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-1.5-7b-hf",
252 extras={"mistral": "mistral-community/pixtral-12b"}), # noqa: E501
253 "LlavaNextForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-v1.6-mistral-7b-hf"), # noqa: E501
@@ -253,21 +257,24 @@ _MULTIMODAL_EXAMPLE_MODELS = {
257 hf_overrides={"architectures": ["MantisForConditionalGeneration"]}), # noqa: E501
258 "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6",
259 trust_remote_code=True),
256 - "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-V-2_6",
260 + "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
261 + extras={"2.6": "openbmb/MiniCPM-V-2_6"}, # noqa: E501
262 trust_remote_code=True),
263 "MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
264 extras={"olmo": "allenai/Molmo-7B-O-0924"}, # noqa: E501
265 trust_remote_code=True),
266 "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B",
267 trust_remote_code=True),
263 - "PaliGemmaForConditionalGeneration": _HfExamplesInfo("google/paligemma-3b-pt-224"), # noqa: E501
268 + "PaliGemmaForConditionalGeneration": _HfExamplesInfo("google/paligemma-3b-mix-224", # noqa: E501
269 + extras={"v2": "google/paligemma2-3b-ft-docci-448"}), # noqa: E501
270 "Phi3VForCausalLM": _HfExamplesInfo("microsoft/Phi-3-vision-128k-instruct",
271 trust_remote_code=True),
272 "PixtralForConditionalGeneration": _HfExamplesInfo("mistralai/Pixtral-12B-2409", # noqa: E501
273 tokenizer_mode="mistral"),
268 - "QWenLMHeadModel": _HfExamplesInfo("Qwen/Qwen-VL-Chat",
269 - extras={"text_only": "Qwen/Qwen-7B-Chat"}, # noqa: E501
270 - trust_remote_code=True),
274 + "QwenVLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen-VL",
275 + extras={"chat": "Qwen/Qwen-VL-Chat"}, # noqa: E501
276 + trust_remote_code=True,
277 + hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]}), # noqa: E501
278 "Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
279 "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
280 "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
tests/models/test_initialization.py
+1 -2
@@ -18,8 +18,7 @@ def test_can_initialize(model_arch):
18
19 # Avoid OOM
20 def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
21 - if hf_config.model_type == "deepseek_vl_v2":
22 - hf_config.update({"architectures": ["DeepseekVLV2ForCausalLM"]})
21 + hf_config.update(model_info.hf_overrides)
22
23 if hasattr(hf_config, "text_config"):
24 text_config: PretrainedConfig = hf_config.text_config
vllm/model_executor/models/chatglm.py
+58 -362
@@ -1,20 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 -
2 # Adapted from
4 -# https://github.com/THUDM/CogAgent
5 -"""Inference-only CogAgent model compatible with THUDM weights."""
6 -from argparse import Namespace
7 -from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
8 - Union)
3 +# https://github.com/THUDM/ChatGLM2-6B
4 +"""Inference-only ChatGLM model compatible with THUDM weights."""
5 +from typing import Iterable, List, Optional, Set, Tuple, Union
6
7 import torch
8 from torch import nn
9 from torch.nn import LayerNorm
13 -from torchvision import transforms
14 -from torchvision.transforms import InterpolationMode
15 -from transformers import PreTrainedTokenizer, TensorType
16 -from transformers.image_utils import ImageInput
17 -from transformers.tokenization_utils_base import TextInput
10
11 from vllm.attention import Attention, AttentionMetadata
12 from vllm.config import CacheConfig, VllmConfig
@@ -31,204 +23,14 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 ParallelLMHead, VocabParallelEmbedding)
25 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34 -from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
35 -from vllm.model_executor.models.module_mapping import MultiModelKeys
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
37 -from vllm.multimodal import MULTIMODAL_REGISTRY
38 -from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
39 -from vllm.multimodal.parse import MultiModalDataItems
40 -from vllm.multimodal.processing import (BaseMultiModalProcessor,
41 - BaseProcessingInfo, BatchFeature,
42 - MultiModalFieldConfig,
43 - PromptReplacement)
44 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
27 from vllm.sequence import IntermediateTensors
28 from vllm.transformers_utils.configs import ChatGLMConfig
29
48 -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
30 +from .interfaces import SupportsLoRA, SupportsPP
31 from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
32 make_empty_intermediate_tensors_factory, make_layers,
51 - maybe_prefix, merge_multimodal_embeddings)
52 -
53 -
54 -class GLMImagePixelInputs(TypedDict):
55 - pixel_values: torch.Tensor
56 - """Shape: `(batch_size, num_channels, height, width)`"""
57 -
58 -
59 -class GLM4VProcessor:
60 - """
61 - This model doesn't define its own HF processor,
62 - so we implement our own one here.
63 -
64 - """
65 -
66 - def __init__(
67 - self,
68 - config: ChatGLMConfig,
69 - tokenizer: PreTrainedTokenizer,
70 - ) -> None:
71 - super().__init__()
72 -
73 - self.config = config
74 - self.tokenizer = tokenizer
75 -
76 - if vision_config := getattr(config, "vision_config", None):
77 - image_size = vision_config["image_size"]
78 -
79 - self.image_transform = transforms.Compose([
80 - transforms.Resize(
81 - (image_size, image_size),
82 - interpolation=InterpolationMode.BICUBIC,
83 - ),
84 - transforms.ToTensor(),
85 - transforms.Normalize(
86 - mean=(0.48145466, 0.4578275, 0.40821073),
87 - std=(0.26862954, 0.26130258, 0.27577711),
88 - ),
89 - ])
90 - else:
91 - self.image_transform = None
92 -
93 - def __call__(
94 - self,
95 - text: Optional[Union[TextInput, list[TextInput]]] = None,
96 - images: Optional[Union[ImageInput, list[ImageInput]]] = None,
97 - return_tensors: Optional[Union[str, TensorType]] = None,
98 - ) -> BatchFeature:
99 - if text is None:
100 - text = []
101 - if not isinstance(text, list):
102 - text = [text]
103 - if images is None:
104 - images = []
105 - if not isinstance(images, list):
106 - images = [images]
107 - text_inputs = self.tokenizer(text)
108 - if len(images) == 0:
109 - image_inputs = {}
110 - else:
111 - if self.image_transform is None:
112 - raise ValueError("This model does not support image inputs")
113 -
114 - pixel_values = [self.image_transform(image) for image in images]
115 - image_inputs = {"pixel_values": torch.stack(pixel_values)}
116 -
117 - return BatchFeature(
118 - {
119 - **text_inputs,
120 - **image_inputs,
121 - },
122 - tensor_type=return_tensors,
123 - )
124 -
125 -
126 -class GLM4VProcessingInfo(BaseProcessingInfo):
127 -
128 - def get_tokenizer(self):
129 - tokenizer = self.ctx.tokenizer
130 - assert isinstance(tokenizer, PreTrainedTokenizer)
131 - return tokenizer
132 -
133 - def get_hf_config(self):
134 - return self.ctx.get_hf_config(ChatGLMConfig)
135 -
136 - def get_hf_processor(self) -> GLM4VProcessor:
137 - return GLM4VProcessor(
138 - self.get_hf_config(),
139 - self.get_tokenizer(),
140 - )
141 -
142 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
143 - return {"image": 1}
144 -
145 - def get_mm_max_tokens_per_item(
146 - self,
147 - seq_len: int,
148 - mm_counts: Mapping[str, int],
149 - ) -> Mapping[str, int]:
150 - return {"image": self.get_num_image_feature_tokens()}
151 -
152 - def get_num_image_tokens(self) -> int:
153 - hf_config = self.get_hf_config()
154 - if not (vision_config := getattr(hf_config, "vision_config", None)):
155 - return 0
156 -
157 - image_size = vision_config["image_size"]
158 - patch_size = vision_config["patch_size"]
159 - grid_length = image_size // patch_size // 2
160 - return grid_length * grid_length
161 -
162 - def get_num_image_feature_tokens(self) -> int:
163 - # EVA2CLIPModel has embeddings for boi and eoi tokens as well
164 - return self.get_num_image_tokens() + 2
165 -
166 -
167 -class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
168 -
169 - def get_dummy_processor_inputs(
170 - self,
171 - seq_len: int,
172 - mm_counts: Mapping[str, int],
173 - ) -> ProcessorInputs:
174 - hf_config = self.info.get_hf_config()
175 - if not (vision_config := getattr(hf_config, "vision_config", None)):
176 - return ProcessorInputs(prompt_text="", mm_data={})
177 -
178 - target_width = target_height = vision_config["image_size"]
179 - num_images = mm_counts.get("image", 0)
180 -
181 - mm_data = {
182 - "image":
183 - self._get_dummy_images(width=target_width,
184 - height=target_height,
185 - num_images=num_images)
186 - }
187 -
188 - base_text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
189 -
190 - return ProcessorInputs(
191 - prompt_text=base_text * num_images,
192 - mm_data=mm_data,
193 - )
194 -
195 -
196 -class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
197 -
198 - def _get_mm_fields_config(
199 - self,
200 - hf_inputs: BatchFeature,
201 - hf_processor_mm_kwargs: Mapping[str, object],
202 - ) -> Mapping[str, MultiModalFieldConfig]:
203 - return dict(pixel_values=MultiModalFieldConfig.batched("image"))
204 -
205 - def _get_prompt_replacements(
206 - self,
207 - mm_items: MultiModalDataItems,
208 - hf_processor_mm_kwargs: Mapping[str, object],
209 - out_mm_kwargs: MultiModalKwargs,
210 - ) -> list[PromptReplacement]:
211 - hf_config = self.info.get_hf_config()
212 - if not hasattr(hf_config, "vision_config"):
213 - return []
214 -
215 - boi_token_id = hf_config.boi_token_id
216 - image_token_id = hf_config.pad_token_id
217 - eoi_token_id = hf_config.eoi_token_id
218 -
219 - def get_replacement(item_idx: int):
220 - num_image_tokens = self.info.get_num_image_tokens()
221 - image_tokens = [image_token_id] * num_image_tokens
222 -
223 - return [boi_token_id] + image_tokens + [eoi_token_id]
224 -
225 - return [
226 - PromptReplacement(
227 - modality="image",
228 - target=[boi_token_id, image_token_id, eoi_token_id],
229 - replacement=get_replacement,
230 - ),
231 - ]
33 + maybe_prefix)
34
35
36 class GLMAttention(nn.Module):
@@ -489,7 +291,7 @@ class GLMTransformer(nn.Module):
291 position_ids: torch.Tensor,
292 kv_caches: List[torch.Tensor],
293 attn_metadata: AttentionMetadata,
492 - ) -> torch.Tensor:
294 + ) -> Union[torch.Tensor, IntermediateTensors]:
295 for i in range(self.start_layer, self.end_layer):
296 layer = self.layers[i]
297 hidden_states = layer(
@@ -498,8 +300,12 @@ class GLMTransformer(nn.Module):
300 kv_cache=kv_caches[i - self.start_layer],
301 attn_metadata=attn_metadata,
302 )
303 +
304 + if not get_pp_group().is_last_rank:
305 + return IntermediateTensors({"hidden_states": hidden_states})
306 +
307 # Final layer norm.
502 - if get_pp_group().is_last_rank and self.post_layer_norm:
308 + if self.post_layer_norm:
309 hidden_states = self.final_layernorm(hidden_states)
310
311 return hidden_states
@@ -534,61 +340,11 @@ class ChatGLMModel(nn.Module):
340 quant_config=quant_config,
341 prefix=f"{prefix}.output_layer")
342
537 - vision_config_flag = getattr(config, 'vision_config', None)
538 - if vision_config_flag is not None:
539 - self.vision_config = Namespace(**config.vision_config)
540 - self.vision = EVA2CLIPModel(self.config,
541 - quant_config,
542 - prefix=f"{prefix}.vision")
543 - else:
544 - self.vision = None
545 -
343 self.make_empty_intermediate_tensors = (
344 self.encoder.make_empty_intermediate_tensors)
345
549 - def _parse_and_validate_image_input(
550 - self, **kwargs: object) -> GLMImagePixelInputs:
551 -
552 - pixel_values = kwargs.pop("pixel_values", None)
553 - if pixel_values is not None and self.vision is not None:
554 - if isinstance(pixel_values, torch.Tensor):
555 - if pixel_values.ndim > 2:
556 - pixel_values = torch.concat(list(pixel_values))
557 - elif isinstance(pixel_values, list):
558 - return torch.concat(pixel_values)
559 - else:
560 - raise TypeError("""pixel_values must be a torch.Tensor
561 - or a list of torch.Tensor
562 - """)
563 - return GLMImagePixelInputs(pixel_values=pixel_values)
564 -
565 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
566 - image_input = self._parse_and_validate_image_input(**kwargs)
567 - if image_input["pixel_values"] is None:
568 - return None
569 - pixel_values = image_input["pixel_values"].to(
570 - dtype=self.config.torch_dtype)
571 - vision_embeddings = self.vision(pixel_values)
572 - return vision_embeddings
573 -
574 - def get_input_embeddings(
575 - self,
576 - input_ids: torch.Tensor,
577 - multimodal_embeddings: Optional[NestedTensors] = None,
578 - ) -> torch.Tensor:
579 - inputs_embeds = self.embedding(input_ids)
580 - if multimodal_embeddings is not None:
581 - inputs_embeds = merge_multimodal_embeddings(
582 - input_ids=input_ids,
583 - inputs_embeds=inputs_embeds,
584 - multimodal_embeddings=multimodal_embeddings,
585 - placeholder_token_id=[
586 - self.config.boi_token_id,
587 - self.config.pad_token_id,
588 - self.config.eoi_token_id,
589 - ],
590 - )
591 - return inputs_embeds
346 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
347 + return self.embedding(input_ids)
348
349 def forward(
350 self,
@@ -599,26 +355,24 @@ class ChatGLMModel(nn.Module):
355 intermediate_tensors: Optional[IntermediateTensors] = None,
356 inputs_embeds: Optional[torch.Tensor] = None,
357 **kwargs: object,
602 - ) -> torch.Tensor:
358 + ) -> Union[torch.Tensor, IntermediateTensors]:
359 + if get_pp_group().is_first_rank:
360 + if inputs_embeds is not None:
361 + hidden_states = inputs_embeds
362 + else:
363 + hidden_states = self.get_input_embeddings(input_ids)
364 + else:
365 + assert intermediate_tensors is not None
366 + hidden_states = intermediate_tensors["hidden_states"]
367
604 - # NOTE: In v1, inputs_embeds is always generated at model runner, this
605 - # condition is for v0 compatibility.
606 - if intermediate_tensors is not None:
607 - inputs_embeds = intermediate_tensors["hidden_states"]
608 - elif inputs_embeds is None:
609 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)
610 - inputs_embeds = self.get_input_embeddings(input_ids,
611 - vision_embeddings)
368 # Run encoder.
369 hidden_states = self.encoder(
614 - hidden_states=inputs_embeds,
370 + hidden_states=hidden_states,
371 position_ids=positions,
372 kv_caches=kv_caches,
373 attn_metadata=attn_metadata,
374 )
375
620 - if not get_pp_group().is_last_rank:
621 - return IntermediateTensors({"hidden_states": hidden_states})
376 return hidden_states
377
378 def load_weights(self, weights: Iterable[Tuple[str,
@@ -660,12 +414,18 @@ class ChatGLMModel(nn.Module):
414 return loaded_params
415
416
663 -class ChatGLMBaseModel(nn.Module, SupportsLoRA, SupportsPP):
417 +class ChatGLMBaseModel(nn.Module):
418
419 hf_to_vllm_mapper = WeightsMapper(
420 orig_to_new_substr={".word_embeddings": ""}, )
421
668 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
422 + def __init__(
423 + self,
424 + *,
425 + vllm_config: VllmConfig,
426 + prefix: str = "",
427 + transformer_type: type[ChatGLMModel] = ChatGLMModel,
428 + ) -> None:
429 super().__init__()
430 config = vllm_config.model_config.hf_config
431 quant_config = vllm_config.quant_config
@@ -678,27 +438,17 @@ class ChatGLMBaseModel(nn.Module, SupportsLoRA, SupportsPP):
438 self.quant_config = quant_config
439 self.max_position_embeddings = getattr(config, "max_sequence_length",
440 8192)
681 - self.transformer = ChatGLMModel(vllm_config=vllm_config,
682 - prefix=maybe_prefix(
683 - prefix, "transformer"))
441 + self.transformer = transformer_type(vllm_config=vllm_config,
442 + prefix=maybe_prefix(
443 + prefix, "transformer"))
444 if self.config.tie_word_embeddings:
445 self.transformer.output_layer.weight = (
446 self.transformer.embedding.weight)
447 self.lm_head = self.transformer.output_layer
448 self.logits_processor = LogitsProcessor(config.padded_vocab_size)
449 self.sampler = get_sampler()
690 -
691 - def forward(self,
692 - input_ids: torch.Tensor,
693 - positions: torch.Tensor,
694 - kv_caches: List[torch.Tensor],
695 - attn_metadata: AttentionMetadata,
696 - intermediate_tensors: Optional[IntermediateTensors] = None,
697 - **kwargs) -> torch.Tensor:
698 - hidden_states = self.transformer(input_ids, positions, kv_caches,
699 - attn_metadata, intermediate_tensors,
700 - **kwargs)
701 - return hidden_states
450 + self.make_empty_intermediate_tensors = (
451 + self.transformer.make_empty_intermediate_tensors)
452
453 def compute_logits(
454 self,
@@ -722,7 +472,7 @@ class ChatGLMBaseModel(nn.Module, SupportsLoRA, SupportsPP):
472 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
473
474
725 -class ChatGLM(ChatGLMBaseModel):
475 +class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP):
476 packed_modules_mapping = {
477 "query_key_value": ["query_key_value"],
478 "dense_h_to_4h": ["dense_h_to_4h"]
@@ -738,82 +488,28 @@ class ChatGLM(ChatGLMBaseModel):
488 embedding_modules = {}
489 embedding_padding_modules = []
490
491 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
492 + config = vllm_config.model_config.hf_config
493 + if hasattr(config, "vision_config"):
494 + hf_overrides = {"architectures": ["GLM4VForCausalLM"]}
495 + raise RuntimeError(
496 + "The configuration of this model indicates that it supports "
497 + "vision inputs, but you instantiated the text-only version "
498 + "of this model. Please use the vision model by setting "
499 + f"`--hf-overrides {hf_overrides!r}`")
500
742 -class ChatGLMV(ChatGLMBaseModel, SupportsMultiModal):
743 -
744 - packed_modules_mapping = {
745 - "query_key_value": ["query_key_value"],
746 - "dense_h_to_4h": ["dense_h_to_4h"],
747 - "merged_proj": ["gate_proj", "dense_h_to_4h"]
748 - }
749 - # LoRA specific attributes
750 - supported_lora_modules = [
751 - "query_key_value",
752 - "dense",
753 - "dense_h_to_4h",
754 - "dense_4h_to_h",
755 - # vision
756 - "fc1",
757 - "fc2",
758 - "merged_proj",
759 - "linear_proj"
760 - ]
761 -
762 - embedding_modules = {}
763 - embedding_padding_modules = []
764 -
765 - def get_mm_mapping(self) -> MultiModelKeys:
766 - """
767 - Get the module prefix in multimodal models
768 - """
769 - return MultiModelKeys.from_string_field(
770 - language_model="transformer.encoder",
771 - connector="transformer.vision.linear_proj",
772 - tower_model="transformer.vision.transformer")
773 -
774 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
775 - return self.transformer.get_multimodal_embeddings(**kwargs)
501 + super().__init__(vllm_config=vllm_config, prefix=prefix)
502
777 - def get_input_embeddings(
503 + def forward(
504 self,
505 input_ids: torch.Tensor,
780 - multimodal_embeddings: Optional[NestedTensors] = None,
781 - ) -> torch.Tensor:
782 - return self.transformer.get_input_embeddings(input_ids,
783 - multimodal_embeddings)
784 -
785 -
786 -@MULTIMODAL_REGISTRY.register_processor(GLM4VMultiModalProcessor,
787 - info=GLM4VProcessingInfo,
788 - dummy_inputs=GLM4VDummyInputsBuilder)
789 -class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
790 - SupportsMultiModal):
791 - # Ensure that the LoRA support check passes when the class is not
792 - # initialized, but set all these attributes to empty.
793 - # These will be updated when an instance class is selected
794 - packed_modules_mapping = {}
795 - supported_lora_modules = []
796 - embedding_modules = {}
797 - embedding_padding_modules = []
798 -
799 - def __new__(
800 - cls,
801 - vllm_config: VllmConfig,
802 - prefix: str = "",
803 - ) -> None:
804 - config = vllm_config.model_config.hf_config
805 -
806 - # Initialize VL
807 - if hasattr(config, "vision_config"): # noqa: SIM108
808 - instance_cls = ChatGLMV
809 - # Initialize LLM
810 - else:
811 - instance_cls = ChatGLM
812 -
813 - # quant_config references base class members,
814 - # so update values before init is called
815 - cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
816 - cls.supported_lora_modules += instance_cls.supported_lora_modules
817 - cls.embedding_modules.update(instance_cls.embedding_modules)
818 - cls.embedding_padding_modules += instance_cls.embedding_padding_modules
819 - return instance_cls(vllm_config=vllm_config, prefix=prefix)
506 + positions: torch.Tensor,
507 + kv_caches: List[torch.Tensor],
508 + attn_metadata: AttentionMetadata,
509 + intermediate_tensors: Optional[IntermediateTensors] = None,
510 + inputs_embeds: Optional[torch.Tensor] = None,
511 + ) -> Union[torch.Tensor, IntermediateTensors]:
512 + hidden_states = self.transformer(input_ids, positions, kv_caches,
513 + attn_metadata, intermediate_tensors,
514 + inputs_embeds)
515 + return hidden_states
vllm/model_executor/models/glm4_vision_encoder.py deleted
-312
@@ -1,312 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -# Adapted from
4 -# https://github.com/THUDM/GLM-4
5 -"""Inference-only GLM-4v model visual encoder compatible with THUDM weights."""
6 -from argparse import Namespace
7 -from typing import Optional
8 -
9 -import torch
10 -from torch import nn
11 -from torch.nn import LayerNorm
12 -
13 -from vllm.attention.layer import MultiHeadAttention
14 -from vllm.distributed import get_tensor_model_parallel_world_size
15 -from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
16 -from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 - MergedColumnParallelLinear,
18 - QKVParallelLinear,
19 - ReplicatedLinear,
20 - RowParallelLinear)
21 -from vllm.model_executor.layers.quantization.base_config import (
22 - QuantizationConfig)
23 -
24 -
25 -class PatchEmbedding(nn.Module):
26 -
27 - def __init__(self, config):
28 - super().__init__()
29 - self.proj = nn.Conv2d(config.in_channels,
30 - config.hidden_size,
31 - kernel_size=config.patch_size,
32 - stride=config.patch_size)
33 - self.cls_embedding = nn.Parameter(torch.zeros(1, config.hidden_size))
34 - self.position_embedding = nn.Embedding(config.num_positions,
35 - config.hidden_size)
36 -
37 - def forward(self, images: torch.Tensor) -> torch.Tensor:
38 - """
39 - Parameters:
40 - images : torch.Tensor
41 - Input image tensor with shape (B, C, H, W)
42 -
43 - Returns:
44 - torch.Tensor
45 - Transformed tensor with shape (B, L, D)
46 - """
47 - images = images.to(device=self.proj.weight.device,
48 - dtype=self.proj.weight.dtype)
49 - x = self.proj(images)
50 - x = x.flatten(2).transpose(1, 2)
51 - cls_token = self.cls_embedding.expand(x.shape[0], -1, -1)
52 - x = torch.cat((cls_token, x), dim=1)
53 - x += self.position_embedding.weight.unsqueeze(0)
54 - return x
55 -
56 -
57 -class Attention(nn.Module):
58 -
59 - def __init__(
60 - self,
61 - config,
62 - quant_config: Optional[QuantizationConfig] = None,
63 - prefix: str = '',
64 - ):
65 - super().__init__()
66 - self.hidden_size = config.hidden_size
67 - self.tp_size = get_tensor_model_parallel_world_size()
68 - self.num_heads_per_rank = config.num_heads // self.tp_size
69 - self.head_dim = config.hidden_size // config.num_heads
70 - self.scale = self.head_dim**-0.5
71 -
72 - self.query_key_value = QKVParallelLinear(
73 - config.hidden_size,
74 - self.head_dim,
75 - config.num_heads,
76 - quant_config=quant_config,
77 - prefix=f"{prefix}.query_key_value",
78 - )
79 - self.dense = RowParallelLinear(
80 - config.hidden_size,
81 - config.hidden_size,
82 - quant_config=quant_config,
83 - prefix=f"{prefix}.dense",
84 - )
85 -
86 - self.attn = MultiHeadAttention(self.num_heads_per_rank, self.head_dim,
87 - self.scale)
88 - self.output_dropout = torch.nn.Dropout(config.dropout_prob)
89 -
90 - def forward(self, x: torch.Tensor) -> torch.Tensor:
91 - qkv, _ = self.query_key_value(x) # B, L, 3 * H * D
92 - q, k, v = qkv.chunk(3, dim=-1)
93 -
94 - out = self.attn(q, k, v)
95 - output, _ = self.dense(out)
96 - output = self.output_dropout(output)
97 - return output
98 -
99 -
100 -class MLP(nn.Module):
101 -
102 - def __init__(
103 - self,
104 - config,
105 - quant_config: Optional[QuantizationConfig] = None,
106 - prefix: str = '',
107 - ):
108 - super().__init__()
109 - self.config = config
110 - self.activation_fn = get_act_fn(config.hidden_act)
111 - self.fc1 = ColumnParallelLinear(
112 - config.hidden_size,
113 - config.intermediate_size,
114 - quant_config=quant_config,
115 - prefix=f"{prefix}.fc1",
116 - )
117 - self.fc2 = RowParallelLinear(
118 - config.intermediate_size,
119 - config.hidden_size,
120 - quant_config=quant_config,
121 - prefix=f"{prefix}.fc2",
122 - )
123 -
124 - def forward(self, x: torch.Tensor) -> torch.Tensor:
125 - x, _ = self.fc1(x)
126 - x = self.activation_fn(x)
127 - x, _ = self.fc2(x)
128 - return x
129 -
130 -
131 -class TransformerLayer(nn.Module):
132 -
133 - def __init__(
134 - self,
135 - config,
136 - quant_config: Optional[QuantizationConfig] = None,
137 - prefix: str = '',
138 - ):
139 - super().__init__()
140 - self.input_layernorm = LayerNorm(config.hidden_size,
141 - eps=config.layer_norm_eps)
142 - self.attention = Attention(config,
143 - quant_config=quant_config,
144 - prefix=f"{prefix}.attention")
145 - self.mlp = MLP(config,
146 - quant_config=quant_config,
147 - prefix=f"{prefix}.mlp")
148 - self.post_attention_layernorm = LayerNorm(config.hidden_size,
149 - eps=config.layer_norm_eps)
150 -
151 - def forward(self, hidden_states):
152 - attention_input = hidden_states
153 - attention_output = self.input_layernorm(
154 - self.attention(attention_input))
155 - hidden_states = attention_input + attention_output
156 - mlp_input = hidden_states
157 - mlp_output = self.post_attention_layernorm(self.mlp(mlp_input))
158 - output = mlp_input + mlp_output
159 - return output
160 -
161 -
162 -class Transformer(nn.Module):
163 -
164 - def __init__(
165 - self,
166 - config,
167 - quant_config: Optional[QuantizationConfig] = None,
168 - prefix: str = '',
169 - ):
170 - super().__init__()
171 - self.layers = nn.ModuleList([
172 - TransformerLayer(config,
173 - quant_config=quant_config,
174 - prefix=f"{prefix}.layers.{layer_idx}")
175 - for layer_idx in range(config.num_hidden_layers)
176 - ])
177 -
178 - def forward(self, hidden_states):
179 - for layer_module in self.layers:
180 - hidden_states = layer_module(hidden_states)
181 - return hidden_states
182 -
183 -
184 -class GLU(nn.Module):
185 -
186 - def __init__(
187 - self,
188 - config,
189 - in_features,
190 - quant_config: Optional[QuantizationConfig] = None,
191 - prefix: str = '',
192 - ):
193 - """
194 - The original implementation is the same as:
195 - ```python
196 - self.dense_h_to_4h = ColumnParallelLinear(
197 - config.hidden_size,
198 - config.ffn_hidden_size,
199 - bias=False,
200 - quant_config=quant_config
201 - )
202 -
203 - self.gate_proj = ColumnParallelLinear(
204 - config.hidden_size,
205 - config.ffn_hidden_size,
206 - bias=False,
207 - quant_config=quant_config
208 - )
209 - ```
210 - ```
211 - gate_proj_output, _ = self.gate_proj(x)
212 - dense_h_to_4h_output, _ = self.dense_h_to_4h(x)
213 - x = torch.cat([gate_proj_output, dense_h_to_4h_output], dim=-1)
214 - ```
215 -
216 - We merge two ColumnParallelLinear into one MergedColumnParallelLinear:
217 - ```
218 - self.merged_proj = MergedColumnParallelLinear(
219 - config.hidden_size,
220 - [config.ffn_hidden_size] * 2,
221 - bias=False,
222 - quant_config=quant_config
223 - )
224 - ```
225 - ```
226 - x, _ = self.merged_proj(x)
227 - ```
228 - """
229 - super().__init__()
230 - self.linear_proj = ReplicatedLinear(in_features,
231 - config.hidden_size,
232 - bias=False,
233 - quant_config=quant_config,
234 - prefix=f"{prefix}.linear_proj")
235 - self.norm1 = nn.LayerNorm(config.hidden_size)
236 - self.act1 = nn.GELU()
237 - self.act2 = SiluAndMul()
238 -
239 - self.merged_proj = MergedColumnParallelLinear(
240 - config.hidden_size, [config.ffn_hidden_size] * 2,
241 - bias=False,
242 - quant_config=quant_config,
243 - prefix=f"{prefix}.merged_proj")
244 -
245 - self.dense_4h_to_h = RowParallelLinear(
246 - config.ffn_hidden_size,
247 - config.hidden_size,
248 - bias=False,
249 - quant_config=quant_config,
250 - prefix=f"{prefix}.dense_4h_to_h")
251 -
252 - def forward(self, x):
253 - x, _ = self.linear_proj(x)
254 - x = self.act1(self.norm1(x))
255 - x, _ = self.merged_proj(x)
256 - x = self.act2(x)
257 - x, _ = self.dense_4h_to_h(x)
258 - return x
259 -
260 -
261 -class EVA2CLIPModel(nn.Module):
262 -
263 - def __init__(
264 - self,
265 - config,
266 - quant_config: Optional[QuantizationConfig] = None,
267 - prefix: str = '',
268 - ):
269 - super().__init__()
270 - vision_config = Namespace(**config.vision_config)
271 - self.patch_embedding = PatchEmbedding(vision_config)
272 - self.transformer = Transformer(vision_config,
273 - quant_config=quant_config,
274 - prefix=f"{prefix}.transformer")
275 - self.linear_proj = GLU(config,
276 - in_features=config.hidden_size,
277 - quant_config=quant_config,
278 - prefix=f"{prefix}.linear_proj")
279 - self.conv = nn.Conv2d(in_channels=vision_config.hidden_size,
280 - out_channels=config.hidden_size,
281 - kernel_size=2,
282 - stride=2)
283 - self.boi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
284 - self.eoi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
285 - self.scaling_factor = vision_config.scaling_factor
286 -
287 - def forward(self, images: torch.Tensor) -> torch.Tensor:
288 - """
289 - Parameters:
290 - images : torch.Tensor
291 - Input image tensor with shape (B, C, H, W)
292 -
293 - Returns:
294 - torch.Tensor
295 - Transformed tensor with shape (B, L, D)
296 - """
297 - x = self.patch_embedding(images)
298 - x = self.transformer(x)
299 - x = x[:, 1:]
300 -
301 - b, s, h = x.shape
302 - grid_size = int(s**0.5)
303 - x = x.view(b, grid_size, grid_size, h).permute(0, 3, 1, 2)
304 - x = self.conv(x)
305 -
306 - x = x.flatten(2).transpose(1, 2)
307 - x = self.linear_proj(x)
308 - boi = self.boi.expand(x.shape[0], -1, -1)
309 - eoi = self.eoi.expand(x.shape[0], -1, -1)
310 - x = torch.cat((boi, x, eoi), dim=1)
311 - x = x / self.scaling_factor
312 - return x
vllm/model_executor/models/glm4v.py new
+662
@@ -0,0 +1,662 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +# Adapted from
4 +# https://github.com/THUDM/CogAgent
5 +"""Inference-only CogAgent model compatible with THUDM weights."""
6 +from argparse import Namespace
7 +from typing import List, Literal, Mapping, Optional, TypedDict, Union
8 +
9 +import torch
10 +from torch import nn
11 +from torch.nn import LayerNorm
12 +from torchvision import transforms
13 +from torchvision.transforms import InterpolationMode
14 +from transformers import PreTrainedTokenizer, TensorType
15 +from transformers.image_utils import ImageInput
16 +from transformers.tokenization_utils_base import TextInput
17 +
18 +from vllm.attention import AttentionMetadata
19 +from vllm.attention.layer import MultiHeadAttention
20 +from vllm.config import VllmConfig
21 +from vllm.distributed import get_tensor_model_parallel_world_size
22 +from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
23 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24 + MergedColumnParallelLinear,
25 + QKVParallelLinear,
26 + ReplicatedLinear,
27 + RowParallelLinear)
28 +from vllm.model_executor.layers.quantization import QuantizationConfig
29 +from vllm.model_executor.models.module_mapping import MultiModelKeys
30 +from vllm.multimodal import MULTIMODAL_REGISTRY
31 +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
32 +from vllm.multimodal.parse import MultiModalDataItems
33 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 + BaseProcessingInfo, BatchFeature,
35 + MultiModalFieldConfig,
36 + PromptReplacement)
37 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38 +from vllm.sequence import IntermediateTensors
39 +from vllm.transformers_utils.configs import ChatGLMConfig
40 +
41 +from .chatglm import ChatGLMBaseModel, ChatGLMModel
42 +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
43 +from .utils import flatten_bn, merge_multimodal_embeddings
44 +
45 +
46 +class GLMVImagePixelInputs(TypedDict):
47 + type: Literal["pixel_values"]
48 + data: torch.Tensor
49 + """Shape: `(batch_size, num_channels, height, width)`"""
50 +
51 +
52 +class EVA2CLIPPatchEmbedding(nn.Module):
53 +
54 + def __init__(self, config):
55 + super().__init__()
56 + self.proj = nn.Conv2d(config.in_channels,
57 + config.hidden_size,
58 + kernel_size=config.patch_size,
59 + stride=config.patch_size)
60 + self.cls_embedding = nn.Parameter(torch.zeros(1, config.hidden_size))
61 + self.position_embedding = nn.Embedding(config.num_positions,
62 + config.hidden_size)
63 +
64 + def forward(self, images: torch.Tensor) -> torch.Tensor:
65 + """
66 + Parameters:
67 + images : torch.Tensor
68 + Input image tensor with shape (B, C, H, W)
69 +
70 + Returns:
71 + torch.Tensor
72 + Transformed tensor with shape (B, L, D)
73 + """
74 + images = images.to(device=self.proj.weight.device,
75 + dtype=self.proj.weight.dtype)
76 + x = self.proj(images)
77 + x = x.flatten(2).transpose(1, 2)
78 + cls_token = self.cls_embedding.expand(x.shape[0], -1, -1)
79 + x = torch.cat((cls_token, x), dim=1)
80 + x += self.position_embedding.weight.unsqueeze(0)
81 + return x
82 +
83 +
84 +class EVA2CLIPAttention(nn.Module):
85 +
86 + def __init__(
87 + self,
88 + config,
89 + quant_config: Optional[QuantizationConfig] = None,
90 + prefix: str = '',
91 + ):
92 + super().__init__()
93 + self.hidden_size = config.hidden_size
94 + self.tp_size = get_tensor_model_parallel_world_size()
95 + self.num_heads_per_rank = config.num_heads // self.tp_size
96 + self.head_dim = config.hidden_size // config.num_heads
97 + self.scale = self.head_dim**-0.5
98 +
99 + self.query_key_value = QKVParallelLinear(
100 + config.hidden_size,
101 + self.head_dim,
102 + config.num_heads,
103 + quant_config=quant_config,
104 + prefix=f"{prefix}.query_key_value",
105 + )
106 + self.dense = RowParallelLinear(
107 + config.hidden_size,
108 + config.hidden_size,
109 + quant_config=quant_config,
110 + prefix=f"{prefix}.dense",
111 + )
112 +
113 + self.attn = MultiHeadAttention(self.num_heads_per_rank, self.head_dim,
114 + self.scale)
115 + self.output_dropout = torch.nn.Dropout(config.dropout_prob)
116 +
117 + def forward(self, x: torch.Tensor) -> torch.Tensor:
118 + qkv, _ = self.query_key_value(x) # B, L, 3 * H * D
119 + q, k, v = qkv.chunk(3, dim=-1)
120 +
121 + out = self.attn(q, k, v)
122 + output, _ = self.dense(out)
123 + output = self.output_dropout(output)
124 + return output
125 +
126 +
127 +class EVA2CLIPMLP(nn.Module):
128 +
129 + def __init__(
130 + self,
131 + config,
132 + quant_config: Optional[QuantizationConfig] = None,
133 + prefix: str = '',
134 + ):
135 + super().__init__()
136 + self.config = config
137 + self.activation_fn = get_act_fn(config.hidden_act)
138 + self.fc1 = ColumnParallelLinear(
139 + config.hidden_size,
140 + config.intermediate_size,
141 + quant_config=quant_config,
142 + prefix=f"{prefix}.fc1",
143 + )
144 + self.fc2 = RowParallelLinear(
145 + config.intermediate_size,
146 + config.hidden_size,
147 + quant_config=quant_config,
148 + prefix=f"{prefix}.fc2",
149 + )
150 +
151 + def forward(self, x: torch.Tensor) -> torch.Tensor:
152 + x, _ = self.fc1(x)
153 + x = self.activation_fn(x)
154 + x, _ = self.fc2(x)
155 + return x
156 +
157 +
158 +class EVA2CLIPTransformerLayer(nn.Module):
159 +
160 + def __init__(
161 + self,
162 + config,
163 + quant_config: Optional[QuantizationConfig] = None,
164 + prefix: str = '',
165 + ):
166 + super().__init__()
167 + self.input_layernorm = LayerNorm(config.hidden_size,
168 + eps=config.layer_norm_eps)
169 + self.attention = EVA2CLIPAttention(config,
170 + quant_config=quant_config,
171 + prefix=f"{prefix}.attention")
172 + self.mlp = EVA2CLIPMLP(config,
173 + quant_config=quant_config,
174 + prefix=f"{prefix}.mlp")
175 + self.post_attention_layernorm = LayerNorm(config.hidden_size,
176 + eps=config.layer_norm_eps)
177 +
178 + def forward(self, hidden_states):
179 + attention_input = hidden_states
180 + attention_output = self.input_layernorm(
181 + self.attention(attention_input))
182 + hidden_states = attention_input + attention_output
183 + mlp_input = hidden_states
184 + mlp_output = self.post_attention_layernorm(self.mlp(mlp_input))
185 + output = mlp_input + mlp_output
186 + return output
187 +
188 +
189 +class EVA2CLIPTransformer(nn.Module):
190 +
191 + def __init__(
192 + self,
193 + config,
194 + quant_config: Optional[QuantizationConfig] = None,
195 + prefix: str = '',
196 + ):
197 + super().__init__()
198 + self.layers = nn.ModuleList([
199 + EVA2CLIPTransformerLayer(config,
200 + quant_config=quant_config,
201 + prefix=f"{prefix}.layers.{layer_idx}")
202 + for layer_idx in range(config.num_hidden_layers)
203 + ])
204 +
205 + def forward(self, hidden_states):
206 + for layer_module in self.layers:
207 + hidden_states = layer_module(hidden_states)
208 + return hidden_states
209 +
210 +
211 +class EVA2CLIPGLU(nn.Module):
212 +
213 + def __init__(
214 + self,
215 + config,
216 + in_features,
217 + quant_config: Optional[QuantizationConfig] = None,
218 + prefix: str = '',
219 + ):
220 + """
221 + The original implementation is the same as:
222 + ```python
223 + self.dense_h_to_4h = ColumnParallelLinear(
224 + config.hidden_size,
225 + config.ffn_hidden_size,
226 + bias=False,
227 + quant_config=quant_config
228 + )
229 +
230 + self.gate_proj = ColumnParallelLinear(
231 + config.hidden_size,
232 + config.ffn_hidden_size,
233 + bias=False,
234 + quant_config=quant_config
235 + )
236 + ```
237 + ```
238 + gate_proj_output, _ = self.gate_proj(x)
239 + dense_h_to_4h_output, _ = self.dense_h_to_4h(x)
240 + x = torch.cat([gate_proj_output, dense_h_to_4h_output], dim=-1)
241 + ```
242 +
243 + We merge two ColumnParallelLinear into one MergedColumnParallelLinear:
244 + ```
245 + self.merged_proj = MergedColumnParallelLinear(
246 + config.hidden_size,
247 + [config.ffn_hidden_size] * 2,
248 + bias=False,
249 + quant_config=quant_config
250 + )
251 + ```
252 + ```
253 + x, _ = self.merged_proj(x)
254 + ```
255 + """
256 + super().__init__()
257 + self.linear_proj = ReplicatedLinear(in_features,
258 + config.hidden_size,
259 + bias=False,
260 + quant_config=quant_config,
261 + prefix=f"{prefix}.linear_proj")
262 + self.norm1 = nn.LayerNorm(config.hidden_size)
263 + self.act1 = nn.GELU()
264 + self.act2 = SiluAndMul()
265 +
266 + self.merged_proj = MergedColumnParallelLinear(
267 + config.hidden_size, [config.ffn_hidden_size] * 2,
268 + bias=False,
269 + quant_config=quant_config,
270 + prefix=f"{prefix}.merged_proj")
271 +
272 + self.dense_4h_to_h = RowParallelLinear(
273 + config.ffn_hidden_size,
274 + config.hidden_size,
275 + bias=False,
276 + quant_config=quant_config,
277 + prefix=f"{prefix}.dense_4h_to_h")
278 +
279 + def forward(self, x):
280 + x, _ = self.linear_proj(x)
281 + x = self.act1(self.norm1(x))
282 + x, _ = self.merged_proj(x)
283 + x = self.act2(x)
284 + x, _ = self.dense_4h_to_h(x)
285 + return x
286 +
287 +
288 +class EVA2CLIPModel(nn.Module):
289 +
290 + def __init__(
291 + self,
292 + config,
293 + quant_config: Optional[QuantizationConfig] = None,
294 + prefix: str = '',
295 + ):
296 + super().__init__()
297 + vision_config = Namespace(**config.vision_config)
298 + self.patch_embedding = EVA2CLIPPatchEmbedding(vision_config)
299 + self.transformer = EVA2CLIPTransformer(vision_config,
300 + quant_config=quant_config,
301 + prefix=f"{prefix}.transformer")
302 + self.linear_proj = EVA2CLIPGLU(config,
303 + in_features=config.hidden_size,
304 + quant_config=quant_config,
305 + prefix=f"{prefix}.linear_proj")
306 + self.conv = nn.Conv2d(in_channels=vision_config.hidden_size,
307 + out_channels=config.hidden_size,
308 + kernel_size=2,
309 + stride=2)
310 + self.boi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
311 + self.eoi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
312 + self.scaling_factor = vision_config.scaling_factor
313 +
314 + def forward(self, images: torch.Tensor) -> torch.Tensor:
315 + """
316 + Parameters:
317 + images : torch.Tensor
318 + Input image tensor with shape (B, C, H, W)
319 +
320 + Returns:
321 + torch.Tensor
322 + Transformed tensor with shape (B, L, D)
323 + """
324 + x = self.patch_embedding(images)
325 + x = self.transformer(x)
326 + x = x[:, 1:]
327 +
328 + b, s, h = x.shape
329 + grid_size = int(s**0.5)
330 + x = x.view(b, grid_size, grid_size, h).permute(0, 3, 1, 2)
331 + x = self.conv(x)
332 +
333 + x = x.flatten(2).transpose(1, 2)
334 + x = self.linear_proj(x)
335 + boi = self.boi.expand(x.shape[0], -1, -1)
336 + eoi = self.eoi.expand(x.shape[0], -1, -1)
337 + x = torch.cat((boi, x, eoi), dim=1)
338 + x = x / self.scaling_factor
339 + return x
340 +
341 +
342 +class GLM4VModel(ChatGLMModel):
343 +
344 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
345 + super().__init__(vllm_config=vllm_config, prefix=prefix)
346 +
347 + quant_config = vllm_config.quant_config
348 +
349 + self.vision = EVA2CLIPModel(self.config,
350 + quant_config,
351 + prefix=f"{prefix}.vision")
352 +
353 +
354 +class GLM4VProcessor:
355 + """
356 + This model doesn't define its own HF processor,
357 + so we implement our own one here.
358 + """
359 +
360 + def __init__(
361 + self,
362 + config: ChatGLMConfig,
363 + tokenizer: PreTrainedTokenizer,
364 + ) -> None:
365 + super().__init__()
366 +
367 + self.config = config
368 + self.tokenizer = tokenizer
369 +
370 + vision_config = config.vision_config
371 + image_size = vision_config["image_size"]
372 +
373 + self.image_transform = transforms.Compose([
374 + transforms.Resize(
375 + (image_size, image_size),
376 + interpolation=InterpolationMode.BICUBIC,
377 + ),
378 + transforms.ToTensor(),
379 + transforms.Normalize(
380 + mean=(0.48145466, 0.4578275, 0.40821073),
381 + std=(0.26862954, 0.26130258, 0.27577711),
382 + ),
383 + ])
384 +
385 + def __call__(
386 + self,
387 + text: Optional[Union[TextInput, list[TextInput]]] = None,
388 + images: Optional[Union[ImageInput, list[ImageInput]]] = None,
389 + return_tensors: Optional[Union[str, TensorType]] = None,
390 + ) -> BatchFeature:
391 + if text is None:
392 + text = []
393 + if not isinstance(text, list):
394 + text = [text]
395 + if images is None:
396 + images = []
397 + if not isinstance(images, list):
398 + images = [images]
399 +
400 + text_inputs = self.tokenizer(text)
401 +
402 + if len(images) == 0:
403 + image_inputs = {}
404 + else:
405 + pixel_values = [self.image_transform(image) for image in images]
406 + image_inputs = {"pixel_values": torch.stack(pixel_values)}
407 +
408 + return BatchFeature(
409 + {
410 + **text_inputs,
411 + **image_inputs,
412 + },
413 + tensor_type=return_tensors,
414 + )
415 +
416 +
417 +class GLM4VProcessingInfo(BaseProcessingInfo):
418 +
419 + def get_tokenizer(self):
420 + tokenizer = self.ctx.tokenizer
421 + assert isinstance(tokenizer, PreTrainedTokenizer)
422 + return tokenizer
423 +
424 + def get_hf_config(self):
425 + return self.ctx.get_hf_config(ChatGLMConfig)
426 +
427 + def get_hf_processor(self) -> GLM4VProcessor:
428 + return GLM4VProcessor(
429 + self.get_hf_config(),
430 + self.get_tokenizer(),
431 + )
432 +
433 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
434 + return {"image": 1}
435 +
436 + def get_mm_max_tokens_per_item(
437 + self,
438 + seq_len: int,
439 + mm_counts: Mapping[str, int],
440 + ) -> Mapping[str, int]:
441 + return {"image": self.get_num_image_feature_tokens()}
442 +
443 + def get_num_image_tokens(self) -> int:
444 + hf_config = self.get_hf_config()
445 + vision_config = hf_config.vision_config
446 +
447 + image_size = vision_config["image_size"]
448 + patch_size = vision_config["patch_size"]
449 + grid_length = image_size // patch_size // 2
450 + return grid_length * grid_length
451 +
452 + def get_num_image_feature_tokens(self) -> int:
453 + # EVA2CLIPModel has embeddings for boi and eoi tokens as well
454 + return self.get_num_image_tokens() + 2
455 +
456 +
457 +class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
458 +
459 + def get_dummy_processor_inputs(
460 + self,
461 + seq_len: int,
462 + mm_counts: Mapping[str, int],
463 + ) -> ProcessorInputs:
464 + hf_config = self.info.get_hf_config()
465 + vision_config = hf_config.vision_config
466 +
467 + target_width = target_height = vision_config["image_size"]
468 + num_images = mm_counts.get("image", 0)
469 +
470 + mm_data = {
471 + "image":
472 + self._get_dummy_images(width=target_width,
473 + height=target_height,
474 + num_images=num_images)
475 + }
476 +
477 + base_text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
478 +
479 + return ProcessorInputs(
480 + prompt_text=base_text * num_images,
481 + mm_data=mm_data,
482 + )
483 +
484 +
485 +class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
486 +
487 + def _get_mm_fields_config(
488 + self,
489 + hf_inputs: BatchFeature,
490 + hf_processor_mm_kwargs: Mapping[str, object],
491 + ) -> Mapping[str, MultiModalFieldConfig]:
492 + return dict(pixel_values=MultiModalFieldConfig.batched("image"))
493 +
494 + def _get_prompt_replacements(
495 + self,
496 + mm_items: MultiModalDataItems,
497 + hf_processor_mm_kwargs: Mapping[str, object],
498 + out_mm_kwargs: MultiModalKwargs,
499 + ) -> list[PromptReplacement]:
500 + hf_config = self.info.get_hf_config()
501 +
502 + boi_token_id = hf_config.boi_token_id
503 + image_token_id = hf_config.pad_token_id
504 + eoi_token_id = hf_config.eoi_token_id
505 +
506 + def get_replacement(item_idx: int):
507 + num_image_tokens = self.info.get_num_image_tokens()
508 + image_tokens = [image_token_id] * num_image_tokens
509 +
510 + return [boi_token_id] + image_tokens + [eoi_token_id]
511 +
512 + return [
513 + PromptReplacement(
514 + modality="image",
515 + target=[boi_token_id, image_token_id, eoi_token_id],
516 + replacement=get_replacement,
517 + ),
518 + ]
519 +
520 +
521 +@MULTIMODAL_REGISTRY.register_processor(GLM4VMultiModalProcessor,
522 + info=GLM4VProcessingInfo,
523 + dummy_inputs=GLM4VDummyInputsBuilder)
524 +class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
525 + SupportsMultiModal):
526 +
527 + packed_modules_mapping = {
528 + "query_key_value": ["query_key_value"],
529 + "dense_h_to_4h": ["dense_h_to_4h"],
530 + "merged_proj": ["gate_proj", "dense_h_to_4h"]
531 + }
532 + # LoRA specific attributes
533 + supported_lora_modules = [
534 + "query_key_value",
535 + "dense",
536 + "dense_h_to_4h",
537 + "dense_4h_to_h",
538 + # vision
539 + "fc1",
540 + "fc2",
541 + "merged_proj",
542 + "linear_proj"
543 + ]
544 +
545 + embedding_modules = {}
546 + embedding_padding_modules = []
547 +
548 + def get_mm_mapping(self) -> MultiModelKeys:
549 + """
550 + Get the module prefix in multimodal models
551 + """
552 + return MultiModelKeys.from_string_field(
553 + language_model="transformer.encoder",
554 + connector="transformer.vision.linear_proj",
555 + tower_model="transformer.vision.transformer")
556 +
557 + def __init__(
558 + self,
559 + *,
560 + vllm_config: VllmConfig,
561 + prefix: str = "",
562 + transformer_type: type[GLM4VModel] = GLM4VModel,
563 + ) -> None:
564 + super().__init__(
565 + vllm_config=vllm_config,
566 + prefix=prefix,
567 + transformer_type=transformer_type,
568 + )
569 +
570 + self.transformer: GLM4VModel
571 +
572 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
573 + h = w = self.config.vision_config["image_size"]
574 + expected_dims = (3, h, w)
575 + actual_dims = tuple(data.shape[1:])
576 +
577 + if actual_dims != expected_dims:
578 + expected_expr = ("batch_size", *map(str, expected_dims))
579 + raise ValueError(
580 + f"The expected shape of pixel values is {expected_expr}. "
581 + f"You supplied {tuple(data.shape)}.")
582 +
583 + return data
584 +
585 + def _parse_and_validate_image_input(
586 + self, **kwargs: object) -> Optional[GLMVImagePixelInputs]:
587 + pixel_values = kwargs.pop("pixel_values", None)
588 +
589 + if pixel_values is not None:
590 + if not isinstance(pixel_values, torch.Tensor):
591 + raise ValueError("Incorrect type of pixel values. "
592 + f"Got type: {type(pixel_values)}")
593 +
594 + return GLMVImagePixelInputs(
595 + type="pixel_values",
596 + data=self._validate_pixel_values(
597 + flatten_bn(pixel_values, concat=True)),
598 + )
599 +
600 + return None
601 +
602 + def _process_image_input(
603 + self, image_input: GLMVImagePixelInputs) -> torch.Tensor:
604 + pixel_values = image_input["data"].to(dtype=self.config.torch_dtype)
605 +
606 + return self.transformer.vision(pixel_values)
607 +
608 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
609 + image_input = self._parse_and_validate_image_input(**kwargs)
610 + if image_input is None:
611 + return None
612 +
613 + vision_embeddings = self._process_image_input(image_input)
614 + return vision_embeddings
615 +
616 + def get_input_embeddings(
617 + self,
618 + input_ids: torch.Tensor,
619 + multimodal_embeddings: Optional[NestedTensors] = None,
620 + ) -> torch.Tensor:
621 + inputs_embeds = self.transformer.get_input_embeddings(input_ids)
622 +
623 + if multimodal_embeddings is not None:
624 + inputs_embeds = merge_multimodal_embeddings(
625 + input_ids=input_ids,
626 + inputs_embeds=inputs_embeds,
627 + multimodal_embeddings=multimodal_embeddings,
628 + placeholder_token_id=[
629 + self.config.boi_token_id,
630 + self.config.pad_token_id,
631 + self.config.eoi_token_id,
632 + ],
633 + )
634 +
635 + return inputs_embeds
636 +
637 + def forward(
638 + self,
639 + input_ids: torch.Tensor,
640 + positions: torch.Tensor,
641 + kv_caches: List[torch.Tensor],
642 + attn_metadata: AttentionMetadata,
643 + intermediate_tensors: Optional[IntermediateTensors] = None,
644 + inputs_embeds: Optional[torch.Tensor] = None,
645 + **kwargs: object,
646 + ) -> Union[torch.Tensor, IntermediateTensors]:
647 + if intermediate_tensors is not None:
648 + inputs_embeds = None
649 +
650 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
651 + # condition is for v0 compatibility.
652 + elif inputs_embeds is None:
653 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
654 + inputs_embeds = self.get_input_embeddings(input_ids,
655 + vision_embeddings)
656 + input_ids = None
657 +
658 + hidden_states = self.transformer(input_ids, positions, kv_caches,
659 + attn_metadata, intermediate_tensors,
660 + inputs_embeds)
661 +
662 + return hidden_states
vllm/model_executor/models/qwen.py
+42 -814
@@ -6,381 +6,35 @@
6 # LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
7 """Inference-only QWen model compatible with HuggingFace weights."""
8
9 -import copy
10 -import math
11 -import re
12 -import unicodedata
13 -from functools import lru_cache, partial
14 -from typing import (AbstractSet, Any, Callable, Collection, Dict, Iterable,
15 - List, Literal, Mapping, Optional, Set, Tuple, TypedDict,
16 - Union)
9 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
10
11 import torch
12 from torch import nn
20 -from torchvision import transforms
21 -from torchvision.transforms import InterpolationMode
22 -from transformers import (BatchFeature, PretrainedConfig, PreTrainedTokenizer,
23 - TensorType)
24 -from transformers.image_utils import ImageInput
25 -from transformers.tokenization_utils_base import TextInput
13 +from transformers import PretrainedConfig
14
15 from vllm.attention import Attention, AttentionMetadata
16 from vllm.compilation.decorators import support_torch_compile
17 from vllm.config import CacheConfig, VllmConfig
18 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
31 -from vllm.logger import init_logger
32 -from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
19 +from vllm.model_executor.layers.activation import SiluAndMul
20 from vllm.model_executor.layers.layernorm import RMSNorm
34 -from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35 - MergedColumnParallelLinear,
21 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
22 QKVParallelLinear,
37 - ReplicatedLinear,
23 RowParallelLinear)
24 from vllm.model_executor.layers.logits_processor import LogitsProcessor
25 from vllm.model_executor.layers.quantization import QuantizationConfig
41 -from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
26 from vllm.model_executor.layers.rotary_embedding import get_rope
27 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
28 from vllm.model_executor.layers.vocab_parallel_embedding import (
29 ParallelLMHead, VocabParallelEmbedding)
30 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47 -from vllm.model_executor.models.module_mapping import MultiModelKeys
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 -from vllm.multimodal import MULTIMODAL_REGISTRY
50 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
51 - NestedTensors)
52 -from vllm.multimodal.parse import MultiModalDataItems
53 -from vllm.multimodal.processing import (BaseMultiModalProcessor,
54 - BaseProcessingInfo, PromptReplacement,
55 - PromptReplacementDetails)
56 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32 from vllm.sequence import IntermediateTensors
33
59 -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
60 -from .utils import (flatten_bn, is_pp_missing_parameter,
34 +from .interfaces import SupportsLoRA, SupportsPP
35 +from .utils import (is_pp_missing_parameter,
36 make_empty_intermediate_tensors_factory, make_layers,
62 - maybe_prefix, merge_multimodal_embeddings)
63 -
64 -logger = init_logger(__name__)
65 -
66 -
67 -class QwenImagePixelInputs(TypedDict):
68 - type: Literal["pixel_values"]
69 - data: torch.Tensor
70 - """
71 - Shape: `(batch_size * num_images, 3, image_size, image_size)`
72 -
73 - Note that image_size is the value in the vision config to which we resize
74 - the image to in the normalization transform. Currently multi-image support
75 - can only be leveraged by passing image embeddings directly.
76 - """
77 -
78 -
79 -class QwenImageEmbeddingInputs(TypedDict):
80 - type: Literal["image_embeds"]
81 - data: torch.Tensor
82 - """Shape: `(batch_size * num_images, 256, hidden_size)`
83 -
84 - `hidden_size` must match the hidden size of the language model backbone
85 - and is stored in the visual config of the model if we have one.
86 - """
87 -
88 -
89 -QwenImageInputs = Union[QwenImagePixelInputs, QwenImageEmbeddingInputs]
90 -
91 -
92 -class VisualAttention(nn.Module):
93 - """self-attention layer class.
94 - Self-attention layer takes input with size [s, b, h]
95 - and returns output of the same size.
96 - """
97 -
98 - def __init__(
99 - self,
100 - embed_dim: int,
101 - num_heads: int,
102 - bias: bool = True,
103 - kdim: Optional[int] = None,
104 - vdim: Optional[int] = None,
105 - ):
106 - super().__init__()
107 - self.embed_dim = embed_dim
108 - self.kdim = kdim if kdim is not None else embed_dim
109 - self.vdim = vdim if vdim is not None else embed_dim
110 - self._qkv_same_embed_dim = self.kdim == embed_dim \
111 - and self.vdim == embed_dim
112 -
113 - self.num_heads = num_heads
114 -
115 - # Per attention head and per partition values.
116 - assert embed_dim % num_heads == 0
117 - self.hidden_size_per_attention_head = embed_dim // num_heads
118 - self.num_attention_heads_per_partition = num_heads
119 - self.hidden_size_per_partition = embed_dim
120 -
121 - # Strided linear layer.
122 - assert self._qkv_same_embed_dim, \
123 - 'Visual Attention implementation only supports self-attention'
124 - self.in_proj = ReplicatedLinear(embed_dim, 3 * embed_dim)
125 - self.out_proj = ReplicatedLinear(embed_dim, embed_dim)
126 - self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
127 -
128 - def forward(
129 - self,
130 - x: torch.Tensor,
131 - attn_mask: Optional[torch.Tensor] = None,
132 - ) -> torch.Tensor:
133 - # query/key/value: [sq, b, h]
134 - sq, b, _ = x.size()
135 - mixed_x_layer, _ = self.in_proj(x)
136 -
137 - # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
138 - new_tensor_shape = mixed_x_layer.size()[:-1] + \
139 - (self.num_attention_heads_per_partition,
140 - 3 * self.hidden_size_per_attention_head)
141 - mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
142 -
143 - # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
144 - query_layer, key_layer, value_layer = mixed_x_layer.split(
145 - self.hidden_size_per_attention_head, dim=-1)
146 -
147 - # [sq, b, np, hn] -> [sq, b * np, hn]
148 - query_layer = query_layer.view(
149 - sq, b * self.num_attention_heads_per_partition,
150 - self.hidden_size_per_attention_head).transpose(0, 1)
151 - # [sk, b, np, hn] -> [sk, b * np, hn]
152 - key_layer = key_layer.view(
153 - sq, b * self.num_attention_heads_per_partition,
154 - self.hidden_size_per_attention_head).transpose(0, 1)
155 -
156 - q_scaled = query_layer / self.norm_factor
157 - if attn_mask is not None:
158 - attention_probs = torch.baddbmm(attn_mask, q_scaled,
159 - key_layer.transpose(-2, -1))
160 - else:
161 - attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1))
162 - attention_probs = attention_probs.softmax(dim=-1)
163 -
164 - value_layer = value_layer.view(
165 - sq, b * self.num_attention_heads_per_partition,
166 - self.hidden_size_per_attention_head).transpose(0, 1)
167 -
168 - # matmul: [b * np, sq, hn]
169 - context_layer = torch.bmm(attention_probs, value_layer)
170 -
171 - # change view [b, np, sq, hn]
172 - context_layer = context_layer.view(
173 - b, self.num_attention_heads_per_partition, sq,
174 - self.hidden_size_per_attention_head)
175 -
176 - # [b, np, sq, hn] --> [sq, b, np, hn]
177 - context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
178 -
179 - # [sq, b, np, hn] --> [sq, b, hp]
180 - new_context_layer_shape = context_layer.size()[:-2] + \
181 - (self.hidden_size_per_partition,)
182 - context_layer = context_layer.view(*new_context_layer_shape)
183 -
184 - output, _ = self.out_proj(context_layer)
185 -
186 - return output
187 -
188 -
189 -class QwenVMLP(nn.Module):
190 - """MLP for the visual component of the Qwen model."""
191 -
192 - def __init__(
193 - self,
194 - hidden_size: int,
195 - intermediate_size: int,
196 - quant_config: Optional[QuantizationConfig] = None,
197 - ):
198 - super().__init__()
199 - self.c_fc = ColumnParallelLinear(hidden_size,
200 - intermediate_size,
201 - bias=True,
202 - quant_config=quant_config)
203 - self.act_fn = get_act_fn("gelu")
204 - self.c_proj = RowParallelLinear(
205 - intermediate_size,
206 - hidden_size,
207 - bias=True,
208 - quant_config=quant_config,
209 - )
210 -
211 - def forward(self, x):
212 - x, _ = self.c_fc(x)
213 - x = self.act_fn(x)
214 - x, _ = self.c_proj(x)
215 - return x
216 -
217 -
218 -class VisualAttentionBlock(nn.Module):
219 -
220 - def __init__(
221 - self,
222 - d_model: int,
223 - n_head: int,
224 - mlp_ratio: float = 4.0,
225 - norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
226 - quant_config: Optional[QuantizationConfig] = None,
227 - ):
228 - super().__init__()
229 -
230 - self.ln_1 = norm_layer(d_model)
231 - self.ln_2 = norm_layer(d_model)
232 - mlp_width = int(d_model * mlp_ratio)
233 - self.attn = VisualAttention(d_model, n_head)
234 - self.mlp = QwenVMLP(
235 - hidden_size=d_model,
236 - intermediate_size=mlp_width,
237 - quant_config=quant_config,
238 - )
239 -
240 - def attention(
241 - self,
242 - x: torch.Tensor,
243 - attn_mask: Optional[torch.Tensor] = None,
244 - ) -> torch.Tensor:
245 - attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None
246 - return self.attn(x, attn_mask=attn_mask)
247 -
248 - def forward(
249 - self,
250 - x: torch.Tensor,
251 - attn_mask: Optional[torch.Tensor] = None,
252 - ) -> torch.Tensor:
253 - x = x + self.attention(self.ln_1(x), attn_mask=attn_mask)
254 - x = x + self.mlp(self.ln_2(x))
255 - return x
256 -
257 -
258 -class TransformerBlock(nn.Module):
259 -
260 - def __init__(
261 - self,
262 - width: int,
263 - layers: int,
264 - heads: int,
265 - mlp_ratio: float = 4.0,
266 - norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
267 - quant_config: Optional[QuantizationConfig] = None,
268 - ):
269 - super().__init__()
270 - self.width = width
271 - self.layers = layers
272 -
273 - self.resblocks = nn.ModuleList([
274 - VisualAttentionBlock(width,
275 - heads,
276 - mlp_ratio,
277 - norm_layer=norm_layer,
278 - quant_config=quant_config)
279 - for _ in range(layers)
280 - ])
281 -
282 - def get_cast_dtype(self) -> torch.dtype:
283 - return self.resblocks[0].mlp.c_fc.weight.dtype
284 -
285 - def get_cast_device(self) -> torch.device:
286 - return self.resblocks[0].mlp.c_fc.weight.device
287 -
288 - def forward(self,
289 - x: torch.Tensor,
290 - attn_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
291 - for r in self.resblocks:
292 - x = r(x, attn_mask=attn_mask)
293 - return x
294 -
295 -
296 -class VisionTransformer(nn.Module):
297 -
298 - def __init__(self,
299 - image_size: int,
300 - patch_size: int,
301 - width: int,
302 - layers: int,
303 - heads: int,
304 - mlp_ratio: float,
305 - n_queries: int = 256,
306 - output_dim: int = 512,
307 - image_start_id: int = 151857,
308 - quant_config: Optional[QuantizationConfig] = None,
309 - **kwargs):
310 - super().__init__()
311 - image_height, image_width = self.image_size = (image_size, image_size)
312 - patch_height, patch_width = self.patch_size = (patch_size, patch_size)
313 - self.grid_size = (image_height // patch_height,
314 - image_width // patch_width)
315 - self.output_dim = output_dim
316 - self.conv1 = nn.Conv2d(in_channels=3,
317 - out_channels=width,
318 - kernel_size=patch_size,
319 - stride=patch_size,
320 - bias=False)
321 -
322 - # class embeddings and positional embeddings
323 - scale = width**-0.5
324 - self.positional_embedding = nn.Parameter(scale *
325 - torch.randn(256, width))
326 -
327 - norm_layer = partial(nn.LayerNorm, eps=1e-6)
328 -
329 - self.ln_pre = norm_layer(width)
330 - self.transformer = TransformerBlock(width,
331 - layers,
332 - heads,
333 - mlp_ratio,
334 - norm_layer=norm_layer,
335 - quant_config=quant_config)
336 -
337 - self.attn_pool = Resampler2(
338 - grid_size=int(math.sqrt(n_queries)),
339 - embed_dim=output_dim,
340 - num_heads=output_dim // 128,
341 - kv_dim=width,
342 - norm_layer=norm_layer,
343 - adaptive=False,
344 - do_post_projection=False,
345 - ).to(
346 - device=self.positional_embedding.device,
347 - dtype=self.positional_embedding.dtype,
348 - )
349 -
350 - self.ln_post = norm_layer(output_dim)
351 - self.proj = nn.Parameter(
352 - (output_dim**-0.5) * torch.randn(output_dim, output_dim))
353 -
354 - self.image_start_id = image_start_id
355 - self.image_end_id = image_start_id + 1
356 - self.image_pad_id = image_start_id + 2
357 -
358 - def forward(self, x: torch.Tensor) -> torch.Tensor:
359 - x = x.to(
360 - dtype=self.transformer.get_cast_dtype(),
361 - device=self.transformer.get_cast_device(),
362 - )
363 -
364 - # to patches
365 - x = self.conv1(x) # shape = [*, width, grid, grid]
366 - x = x.reshape(x.shape[0], x.shape[1],
367 - -1) # shape = [*, width, grid ** 2]
368 - x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
369 -
370 - x = x + get_abs_pos(self.positional_embedding, int(math.sqrt(
371 - x.size(1))))
372 -
373 - x = self.ln_pre(x)
374 -
375 - x = x.permute(1, 0, 2) # NLD -> LND
376 - x = self.transformer(x)
377 - x = x.permute(1, 0, 2) # LND -> NLD
378 -
379 - x = self.attn_pool(x)
380 - x = self.ln_post(x)
381 - x = x @ self.proj
382 -
383 - return x
37 + maybe_prefix)
38
39
40 class QWenMLP(nn.Module):
@@ -564,12 +218,6 @@ class QWenModel(nn.Module):
218 make_empty_intermediate_tensors_factory(
219 ["hidden_states", "residual"], config.hidden_size))
220
567 - if (vision_config := getattr(config, "visual", None)):
568 - self.visual = VisionTransformer(**vision_config,
569 - quant_config=quant_config)
570 - else:
571 - self.visual = None
572 -
221 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
222 return self.wte(input_ids)
223
@@ -592,6 +240,7 @@ class QWenModel(nn.Module):
240 assert intermediate_tensors is not None
241 hidden_states = intermediate_tensors["hidden_states"]
242 residual = intermediate_tensors["residual"]
243 +
244 for i in range(self.start_layer, self.end_layer):
245 layer = self.h[i]
246 hidden_states, residual = layer(
@@ -610,302 +259,25 @@ class QWenModel(nn.Module):
259 return hidden_states
260
261
613 -@lru_cache(maxsize=1)
614 -def _get_tokenizer_without_image_pad(
615 - tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
616 - """
617 - The logic of adding image pad tokens should only be applied in
618 - :class:`QWenVLProcessor`, so they are patched out here.
619 -
620 - The definition of the wrapped tokenizer can be found here:
621 - https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
622 - """
623 - new_tokenizer = copy.deepcopy(tokenizer)
624 -
625 - class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
626 -
627 - def tokenize(
628 - self,
629 - text: str,
630 - allowed_special: Union[AbstractSet[str], str] = "all",
631 - disallowed_special: Union[Collection[str], str] = (),
632 - **kwargs,
633 - ) -> list[Union[bytes, str]]:
634 - text = unicodedata.normalize("NFC", text)
635 -
636 - return [
637 - self.decoder[t] for t in self.tokenizer.encode(
638 - text,
639 - allowed_special=allowed_special,
640 - disallowed_special=disallowed_special,
641 - )
642 - ]
643 -
644 - def _decode(
645 - self,
646 - token_ids: Union[int, List[int]],
647 - skip_special_tokens: bool = False,
648 - errors: Optional[str] = None,
649 - **kwargs,
650 - ) -> str:
651 - if isinstance(token_ids, int):
652 - token_ids = [token_ids]
653 -
654 - return self.tokenizer.decode(
655 - token_ids,
656 - errors=errors or self.errors,
657 - )
658 -
659 - TokenizerWithoutImagePad.__name__ = \
660 - f"{tokenizer.__class__.__name__}WithoutImagePad"
661 -
662 - new_tokenizer.__class__ = TokenizerWithoutImagePad
663 - return new_tokenizer
664 -
665 -
666 -class QWenVLProcessor:
667 - """
668 - This model doesn't define its own HF processor,
669 - so we implement our own one here.
670 -
671 - We call the wrapped tokenizer to automatically insert image pad tokens:
672 - https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py#L245
673 -
674 - The image processor is defined here:
675 - https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
676 - """
262 +class QWenBaseModel(nn.Module):
263
264 def __init__(
265 self,
680 - config: PretrainedConfig,
681 - tokenizer: PreTrainedTokenizer,
266 + *,
267 + vllm_config: VllmConfig,
268 + prefix: str = "",
269 + transformer_type: type[QWenModel] = QWenModel,
270 ) -> None:
271 super().__init__()
684 -
685 - self.config = config
686 - self.tokenizer = tokenizer
687 -
688 - if vision_config := getattr(self.config, "visual", None):
689 - image_size = vision_config["image_size"]
690 -
691 - self.image_transform = transforms.Compose([
692 - transforms.Resize(
693 - (image_size, image_size),
694 - interpolation=InterpolationMode.BICUBIC,
695 - ),
696 - transforms.ToTensor(),
697 - transforms.Normalize(
698 - mean=(0.48145466, 0.4578275, 0.40821073),
699 - std=(0.26862954, 0.26130258, 0.27577711),
700 - ),
701 - ])
702 - else:
703 - self.image_transform = None
704 -
705 - @property
706 - def image_start_tag(self) -> str:
707 - return self.tokenizer.image_start_tag # type: ignore
708 -
709 - @property
710 - def image_end_tag(self) -> str:
711 - return self.tokenizer.image_end_tag # type: ignore
712 -
713 - @property
714 - def image_pad_tag(self) -> str:
715 - return self.tokenizer.image_pad_tag # type: ignore
716 -
717 - def __call__(
718 - self,
719 - text: Optional[Union[TextInput, list[TextInput]]] = None,
720 - images: Optional[Union[ImageInput, list[ImageInput]]] = None,
721 - return_tensors: Optional[Union[str, TensorType]] = None,
722 - ) -> BatchFeature:
723 - if text is None:
724 - text = []
725 - if not isinstance(text, list):
726 - text = [text]
727 - if images is None:
728 - images = []
729 - if not isinstance(images, list):
730 - images = [images]
731 -
732 - text_inputs = self.tokenizer(text)
733 -
734 - if len(images) == 0:
735 - image_inputs = {}
736 - else:
737 - if self.image_transform is None:
738 - raise ValueError("This model does not support image inputs")
739 -
740 - pixel_values = [self.image_transform(image) for image in images]
741 - image_inputs = {"pixel_values": torch.stack(pixel_values)}
742 -
743 - return BatchFeature(
744 - {
745 - **text_inputs,
746 - **image_inputs,
747 - },
748 - tensor_type=return_tensors,
749 - )
750 -
751 -
752 -class QWenVLProcessingInfo(BaseProcessingInfo):
753 -
754 - def get_tokenizer(self) -> PreTrainedTokenizer:
755 - tokenizer = self.ctx.tokenizer
756 - assert isinstance(tokenizer, PreTrainedTokenizer)
757 -
758 - return _get_tokenizer_without_image_pad(tokenizer)
759 -
760 - def get_hf_processor(self) -> QWenVLProcessor:
761 - tokenizer = self.ctx.tokenizer
762 - assert isinstance(tokenizer, PreTrainedTokenizer)
763 -
764 - return QWenVLProcessor(self.get_hf_config(), tokenizer)
765 -
766 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
767 - return {"image": None}
768 -
769 - def get_mm_max_tokens_per_item(
770 - self,
771 - seq_len: int,
772 - mm_counts: Mapping[str, int],
773 - ) -> Mapping[str, int]:
774 - return {"image": self.get_num_image_tokens()}
775 -
776 - def get_num_image_tokens(self) -> int:
777 - hf_config = self.get_hf_config()
778 - if not (vision_config := getattr(hf_config, "visual", None)):
779 - return 0
780 -
781 - image_size = vision_config["image_size"]
782 - patch_size = vision_config["patch_size"]
783 - grid_length = image_size // patch_size // 2
784 - return grid_length * grid_length
785 -
786 -
787 -class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
788 -
789 - def get_dummy_processor_inputs(
790 - self,
791 - seq_len: int,
792 - mm_counts: Mapping[str, int],
793 - ) -> ProcessorInputs:
794 - hf_config = self.info.get_hf_config()
795 - if not (vision_config := getattr(hf_config, "visual", None)):
796 - return ProcessorInputs(prompt_text="", mm_data={})
797 -
798 - processor = self.info.get_hf_processor()
799 - img_start = processor.image_start_tag
800 - img_end = processor.image_end_tag
801 -
802 - target_width = target_height = vision_config["image_size"]
803 - num_images = mm_counts.get("image", 0)
804 -
805 - mm_data = {
806 - "image":
807 - self._get_dummy_images(width=target_width,
808 - height=target_height,
809 - num_images=num_images)
810 - }
811 -
812 - return ProcessorInputs(
813 - prompt_text="".join(f"Picture {i}: {img_start}{img_end}\n"
814 - for i in range(1, num_images + 1)),
815 - mm_data=mm_data,
816 - )
817 -
818 -
819 -class QWenVLMultiModalProcessor(BaseMultiModalProcessor[QWenVLProcessingInfo]):
820 -
821 - def _call_hf_processor(
822 - self,
823 - prompt: str,
824 - mm_data: Mapping[str, object],
825 - mm_kwargs: Mapping[str, object],
826 - ) -> BatchFeature:
827 - # Drops anything between <img>/</img> tags; encoding with the tokenizer
828 - # will automatically add the image pads for the context.
829 - prompt, num_matched_images = re.subn(
830 - r"(Picture \d*: <img>).*?(<\/img>\n)",
831 - r"\1\2",
832 - prompt,
833 - )
834 -
835 - image_data = mm_data.get("images")
836 - if image_data is not None:
837 - assert isinstance(image_data, list)
838 -
839 - num_images = len(image_data)
840 - if num_matched_images != num_images:
841 - logger.warning(
842 - "Number of matched image placeholders %s doesn't match "
843 - "the number of expected images %s; check your placeholder "
844 - "formatting.", num_matched_images, num_images)
845 -
846 - return super()._call_hf_processor(
847 - prompt=prompt,
848 - mm_data=mm_data,
849 - mm_kwargs=mm_kwargs,
850 - )
851 -
852 - def _get_mm_fields_config(
853 - self,
854 - hf_inputs: BatchFeature,
855 - hf_processor_mm_kwargs: Mapping[str, object],
856 - ) -> Mapping[str, MultiModalFieldConfig]:
857 - return dict(
858 - pixel_values=MultiModalFieldConfig.batched("image"),
859 - image_embeds=MultiModalFieldConfig.batched("image"),
860 - )
861 -
862 - def _get_prompt_replacements(
863 - self,
864 - mm_items: MultiModalDataItems,
865 - hf_processor_mm_kwargs: Mapping[str, object],
866 - out_mm_kwargs: MultiModalKwargs,
867 - ) -> list[PromptReplacement]:
868 - hf_config = self.info.get_hf_config()
869 - if not hasattr(hf_config, "visual"):
870 - return []
871 -
872 - tokenizer = self.info.get_tokenizer()
873 - special_tokens: dict[str,
874 - int] = tokenizer.special_tokens # type: ignore
875 -
876 - processor = self.info.get_hf_processor()
877 - img_start_id = special_tokens[processor.image_start_tag]
878 - img_end_id = special_tokens[processor.image_end_tag]
879 - img_pad_id = special_tokens[processor.image_pad_tag]
880 -
881 - num_image_tokens = self.info.get_num_image_tokens()
882 - image_tokens = [img_pad_id] * num_image_tokens
883 -
884 - return [
885 - PromptReplacement(
886 - modality="image",
887 - target=[img_start_id, img_end_id],
888 - replacement=PromptReplacementDetails(
889 - full=[img_start_id] + image_tokens + [img_end_id],
890 - features=image_tokens,
891 - ),
892 - )
893 - ]
894 -
895 -
896 -class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
897 -
898 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
899 - super().__init__()
272 config = vllm_config.model_config.hf_config
273 quant_config = vllm_config.quant_config
274 multimodal_config = vllm_config.model_config.multimodal_config
275 self.config = config
276 self.multimodal_config = multimodal_config
277 self.quant_config = quant_config
906 - self.transformer = QWenModel(vllm_config=vllm_config,
907 - prefix=maybe_prefix(
908 - prefix, "transformer"))
278 + self.transformer = transformer_type(vllm_config=vllm_config,
279 + prefix=maybe_prefix(
280 + prefix, "transformer"))
281 self.lm_head = ParallelLMHead(config.vocab_size,
282 config.hidden_size,
283 quant_config=quant_config)
@@ -916,104 +288,6 @@ class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
288 self.make_empty_intermediate_tensors = (
289 self.transformer.make_empty_intermediate_tensors)
290
919 - def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
920 - h = w = self.config.visual["image_size"]
921 - expected_dims = (3, h, w)
922 - actual_dims = tuple(data.shape[1:])
923 -
924 - if actual_dims != expected_dims:
925 - expected_expr = ("batch_size", *map(str, expected_dims))
926 - raise ValueError(
927 - f"The expected shape of pixel values is {expected_expr}. "
928 - f"You supplied {tuple(data.shape)}.")
929 -
930 - return data
931 -
932 - def _parse_and_validate_image_input(
933 - self, **kwargs: object) -> Optional[QwenImageInputs]:
934 - pixel_values = kwargs.pop("pixel_values", None)
935 - image_embeds = kwargs.pop("image_embeds", None)
936 -
937 - if pixel_values is not None:
938 - if not isinstance(pixel_values, torch.Tensor):
939 - raise ValueError("Incorrect type of pixel values. "
940 - f"Got type: {type(pixel_values)}")
941 -
942 - return QwenImagePixelInputs(
943 - type="pixel_values",
944 - data=self._validate_pixel_values(
945 - flatten_bn(pixel_values, concat=True)),
946 - )
947 -
948 - if image_embeds is not None:
949 - if not isinstance(image_embeds, torch.Tensor):
950 - raise ValueError("Incorrect type of image embeddings. "
951 - f"Got type: {type(image_embeds)}")
952 -
953 - return QwenImageEmbeddingInputs(
954 - type="image_embeds",
955 - data=flatten_bn(image_embeds),
956 - )
957 -
958 - return None
959 -
960 - def _process_image_input(self,
961 - image_input: QwenImageInputs) -> torch.Tensor:
962 - if image_input["type"] == "image_embeds":
963 - return image_input["data"]
964 -
965 - assert self.transformer.visual is not None
966 - return self.transformer.visual(image_input["data"])
967 -
968 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
969 - image_input = self._parse_and_validate_image_input(**kwargs)
970 - if image_input is None:
971 - return None
972 -
973 - vision_embeddings = self._process_image_input(image_input)
974 - return vision_embeddings
975 -
976 - def get_input_embeddings(
977 - self,
978 - input_ids: torch.Tensor,
979 - multimodal_embeddings: Optional[NestedTensors] = None,
980 - ) -> torch.Tensor:
981 - inputs_embeds = self.transformer.get_input_embeddings(input_ids)
982 -
983 - if multimodal_embeddings is not None:
984 - assert self.transformer.visual is not None
985 - inputs_embeds = merge_multimodal_embeddings(
986 - input_ids, inputs_embeds, multimodal_embeddings,
987 - self.transformer.visual.image_pad_id)
988 -
989 - return inputs_embeds
990 -
991 - def forward(
992 - self,
993 - input_ids: torch.Tensor,
994 - positions: torch.Tensor,
995 - kv_caches: List[torch.Tensor],
996 - attn_metadata: AttentionMetadata,
997 - intermediate_tensors: Optional[IntermediateTensors] = None,
998 - inputs_embeds: Optional[torch.Tensor] = None,
999 - **kwargs: object,
1000 - ) -> Union[torch.Tensor, IntermediateTensors]:
1001 - if intermediate_tensors is not None:
1002 - inputs_embeds = None
1003 -
1004 - # NOTE: In v1, inputs_embeds is always generated at model runner, this
1005 - # condition is for v0 compatibility.
1006 - elif inputs_embeds is None:
1007 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1008 - inputs_embeds = self.get_input_embeddings(input_ids,
1009 - vision_embeddings)
1010 - input_ids = None
1011 -
1012 - hidden_states = self.transformer(input_ids, positions, kv_caches,
1013 - attn_metadata, intermediate_tensors,
1014 - inputs_embeds)
1015 - return hidden_states
1016 -
291 def compute_logits(
292 self,
293 hidden_states: torch.Tensor,
@@ -1072,26 +346,7 @@ class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
346 return loaded_params
347
348
1075 -class QWenLLM(QWenBaseModel):
1076 - packed_modules_mapping = {
1077 - "c_attn": ["c_attn"],
1078 - "gate_up_proj": [
1079 - "w2",
1080 - "w1",
1081 - ],
1082 - }
1083 - # LoRA specific attributes
1084 - supported_lora_modules = [
1085 - "c_attn",
1086 - "gate_up_proj",
1087 - "c_proj",
1088 - ]
1089 -
1090 - embedding_modules = {}
1091 - embedding_padding_modules = []
1092 -
1093 -
1094 -class QWenVL(QWenBaseModel, SupportsMultiModal):
349 +class QWenLMHeadModel(QWenBaseModel, SupportsPP, SupportsLoRA):
350 packed_modules_mapping = {
351 "c_attn": ["c_attn"],
352 "gate_up_proj": [
@@ -1104,62 +359,35 @@ class QWenVL(QWenBaseModel, SupportsMultiModal):
359 "c_attn",
360 "gate_up_proj",
361 "c_proj",
1107 - # visual module
1108 - "out_proj",
1109 - "in_proj",
1110 - "c_fc",
1111 - # resampler
1112 - "kv_proj",
362 ]
363
364 embedding_modules = {}
365 embedding_padding_modules = []
366
1118 - def get_mm_mapping(self) -> MultiModelKeys:
1119 - """
1120 - Get the module prefix in multimodal models
1121 - """
1122 - return MultiModelKeys.from_string_field(
1123 - language_model="transformer.h",
1124 - connector="transformer.visual.attn_pool",
1125 - tower_model="transformer.visual.transformer")
1126 -
1127 -
1128 -@MULTIMODAL_REGISTRY.register_processor(QWenVLMultiModalProcessor,
1129 - info=QWenVLProcessingInfo,
1130 - dummy_inputs=QWenVLDummyInputsBuilder)
1131 -class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1132 - """
1133 - QWenLMHeadModel is not only applicable to LLM but also to VL, which is not
1134 - conducive to the current integration logic of LoRA in vLLM. Therefore, it
1135 - is necessary to separate them.
1136 - """
1137 - # Ensure that the LoRA support check passes when the class is not
1138 - # initialized, but set all these attributes to empty.
1139 - # These will be updated when an instance class is selected
1140 - packed_modules_mapping = {}
1141 - supported_lora_modules = []
1142 - embedding_modules = {}
1143 - embedding_padding_modules = []
1144 -
1145 - def __new__(
1146 - cls,
1147 - vllm_config: VllmConfig,
1148 - prefix: str = "",
1149 - ) -> QWenBaseModel:
367 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
368 config = vllm_config.model_config.hf_config
369 + if hasattr(config, "visual"):
370 + hf_overrides = {
371 + "architectures": ["QwenVLForConditionalGeneration"]
372 + }
373 + raise RuntimeError(
374 + "The configuration of this model indicates that it supports "
375 + "vision inputs, but you instantiated the text-only version "
376 + "of this model. Please use the vision model by setting "
377 + f"`--hf-overrides {hf_overrides!r}`")
378 +
379 + super().__init__(vllm_config=vllm_config, prefix=prefix)
380
1152 - # Initialize VL
1153 - if hasattr(config, "visual"): # noqa: SIM108
1154 - instance_cls = QWenVL
1155 - # Initialize LLM
1156 - else:
1157 - instance_cls = QWenLLM
1158 -
1159 - # quant_config references base class members,
1160 - # so update values before init is called
1161 - cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
1162 - cls.supported_lora_modules += instance_cls.supported_lora_modules
1163 - cls.embedding_modules.update(instance_cls.embedding_modules)
1164 - cls.embedding_padding_modules += instance_cls.embedding_padding_modules
1165 - return instance_cls(vllm_config=vllm_config, prefix=prefix)
381 + def forward(
382 + self,
383 + input_ids: torch.Tensor,
384 + positions: torch.Tensor,
385 + kv_caches: List[torch.Tensor],
386 + attn_metadata: AttentionMetadata,
387 + intermediate_tensors: Optional[IntermediateTensors] = None,
388 + inputs_embeds: Optional[torch.Tensor] = None,
389 + ) -> Union[torch.Tensor, IntermediateTensors]:
390 + hidden_states = self.transformer(input_ids, positions, kv_caches,
391 + attn_metadata, intermediate_tensors,
392 + inputs_embeds)
393 + return hidden_states
vllm/model_executor/models/qwen_vl.py new
+794
@@ -0,0 +1,794 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +# Adapted from
4 +# https://huggingface.co/Qwen/Qwen-VL/blob/main/modeling_qwen.py
5 +# Copyright (c) Alibaba Cloud.
6 +"""Inference-only Qwen-VL model compatible with HuggingFace weights."""
7 +
8 +import copy
9 +import math
10 +import re
11 +import unicodedata
12 +from functools import lru_cache, partial
13 +from typing import (AbstractSet, Callable, Collection, List, Literal, Mapping,
14 + Optional, TypedDict, Union)
15 +
16 +import torch
17 +from torch import nn
18 +from torchvision import transforms
19 +from torchvision.transforms import InterpolationMode
20 +from transformers import (BatchFeature, PretrainedConfig, PreTrainedTokenizer,
21 + TensorType)
22 +from transformers.image_utils import ImageInput
23 +from transformers.tokenization_utils_base import TextInput
24 +
25 +from vllm.attention import AttentionMetadata
26 +from vllm.config import VllmConfig
27 +from vllm.model_executor.layers.activation import get_act_fn
28 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 + ReplicatedLinear,
30 + RowParallelLinear)
31 +from vllm.model_executor.layers.quantization import QuantizationConfig
32 +from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
33 +from vllm.model_executor.models.module_mapping import MultiModelKeys
34 +from vllm.multimodal import MULTIMODAL_REGISTRY
35 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
36 + NestedTensors)
37 +from vllm.multimodal.parse import MultiModalDataItems
38 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 + BaseProcessingInfo, PromptReplacement,
40 + PromptReplacementDetails)
41 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
42 +from vllm.sequence import IntermediateTensors
43 +
44 +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
45 +from .qwen import QWenBaseModel, QWenModel
46 +from .utils import flatten_bn, merge_multimodal_embeddings
47 +
48 +
49 +class QwenImagePixelInputs(TypedDict):
50 + type: Literal["pixel_values"]
51 + data: torch.Tensor
52 + """
53 + Shape: `(batch_size * num_images, 3, image_size, image_size)`
54 +
55 + Note that image_size is the value in the vision config to which we resize
56 + the image to in the normalization transform. Currently multi-image support
57 + can only be leveraged by passing image embeddings directly.
58 + """
59 +
60 +
61 +class QwenImageEmbeddingInputs(TypedDict):
62 + type: Literal["image_embeds"]
63 + data: torch.Tensor
64 + """Shape: `(batch_size * num_images, 256, hidden_size)`
65 +
66 + `hidden_size` must match the hidden size of the language model backbone
67 + and is stored in the visual config of the model if we have one.
68 + """
69 +
70 +
71 +QwenImageInputs = Union[QwenImagePixelInputs, QwenImageEmbeddingInputs]
72 +
73 +
74 +class VisualAttention(nn.Module):
75 + """self-attention layer class.
76 + Self-attention layer takes input with size [s, b, h]
77 + and returns output of the same size.
78 + """
79 +
80 + def __init__(
81 + self,
82 + embed_dim: int,
83 + num_heads: int,
84 + bias: bool = True,
85 + kdim: Optional[int] = None,
86 + vdim: Optional[int] = None,
87 + ):
88 + super().__init__()
89 + self.embed_dim = embed_dim
90 + self.kdim = kdim if kdim is not None else embed_dim
91 + self.vdim = vdim if vdim is not None else embed_dim
92 + self._qkv_same_embed_dim = self.kdim == embed_dim \
93 + and self.vdim == embed_dim
94 +
95 + self.num_heads = num_heads
96 +
97 + # Per attention head and per partition values.
98 + assert embed_dim % num_heads == 0
99 + self.hidden_size_per_attention_head = embed_dim // num_heads
100 + self.num_attention_heads_per_partition = num_heads
101 + self.hidden_size_per_partition = embed_dim
102 +
103 + # Strided linear layer.
104 + assert self._qkv_same_embed_dim, \
105 + 'Visual Attention implementation only supports self-attention'
106 + self.in_proj = ReplicatedLinear(embed_dim, 3 * embed_dim)
107 + self.out_proj = ReplicatedLinear(embed_dim, embed_dim)
108 + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
109 +
110 + def forward(
111 + self,
112 + x: torch.Tensor,
113 + attn_mask: Optional[torch.Tensor] = None,
114 + ) -> torch.Tensor:
115 + # query/key/value: [sq, b, h]
116 + sq, b, _ = x.size()
117 + mixed_x_layer, _ = self.in_proj(x)
118 +
119 + # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
120 + new_tensor_shape = mixed_x_layer.size()[:-1] + \
121 + (self.num_attention_heads_per_partition,
122 + 3 * self.hidden_size_per_attention_head)
123 + mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
124 +
125 + # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
126 + query_layer, key_layer, value_layer = mixed_x_layer.split(
127 + self.hidden_size_per_attention_head, dim=-1)
128 +
129 + # [sq, b, np, hn] -> [sq, b * np, hn]
130 + query_layer = query_layer.view(
131 + sq, b * self.num_attention_heads_per_partition,
132 + self.hidden_size_per_attention_head).transpose(0, 1)
133 + # [sk, b, np, hn] -> [sk, b * np, hn]
134 + key_layer = key_layer.view(
135 + sq, b * self.num_attention_heads_per_partition,
136 + self.hidden_size_per_attention_head).transpose(0, 1)
137 +
138 + q_scaled = query_layer / self.norm_factor
139 + if attn_mask is not None:
140 + attention_probs = torch.baddbmm(attn_mask, q_scaled,
141 + key_layer.transpose(-2, -1))
142 + else:
143 + attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1))
144 + attention_probs = attention_probs.softmax(dim=-1)
145 +
146 + value_layer = value_layer.view(
147 + sq, b * self.num_attention_heads_per_partition,
148 + self.hidden_size_per_attention_head).transpose(0, 1)
149 +
150 + # matmul: [b * np, sq, hn]
151 + context_layer = torch.bmm(attention_probs, value_layer)
152 +
153 + # change view [b, np, sq, hn]
154 + context_layer = context_layer.view(
155 + b, self.num_attention_heads_per_partition, sq,
156 + self.hidden_size_per_attention_head)
157 +
158 + # [b, np, sq, hn] --> [sq, b, np, hn]
159 + context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
160 +
161 + # [sq, b, np, hn] --> [sq, b, hp]
162 + new_context_layer_shape = context_layer.size()[:-2] + \
163 + (self.hidden_size_per_partition,)
164 + context_layer = context_layer.view(*new_context_layer_shape)
165 +
166 + output, _ = self.out_proj(context_layer)
167 +
168 + return output
169 +
170 +
171 +class QwenVLMLP(nn.Module):
172 + """MLP for the visual component of the Qwen model."""
173 +
174 + def __init__(
175 + self,
176 + hidden_size: int,
177 + intermediate_size: int,
178 + quant_config: Optional[QuantizationConfig] = None,
179 + ):
180 + super().__init__()
181 + self.c_fc = ColumnParallelLinear(hidden_size,
182 + intermediate_size,
183 + bias=True,
184 + quant_config=quant_config)
185 + self.act_fn = get_act_fn("gelu")
186 + self.c_proj = RowParallelLinear(
187 + intermediate_size,
188 + hidden_size,
189 + bias=True,
190 + quant_config=quant_config,
191 + )
192 +
193 + def forward(self, x):
194 + x, _ = self.c_fc(x)
195 + x = self.act_fn(x)
196 + x, _ = self.c_proj(x)
197 + return x
198 +
199 +
200 +class VisualAttentionBlock(nn.Module):
201 +
202 + def __init__(
203 + self,
204 + d_model: int,
205 + n_head: int,
206 + mlp_ratio: float = 4.0,
207 + norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
208 + quant_config: Optional[QuantizationConfig] = None,
209 + ):
210 + super().__init__()
211 +
212 + self.ln_1 = norm_layer(d_model)
213 + self.ln_2 = norm_layer(d_model)
214 + mlp_width = int(d_model * mlp_ratio)
215 + self.attn = VisualAttention(d_model, n_head)
216 + self.mlp = QwenVLMLP(
217 + hidden_size=d_model,
218 + intermediate_size=mlp_width,
219 + quant_config=quant_config,
220 + )
221 +
222 + def attention(
223 + self,
224 + x: torch.Tensor,
225 + attn_mask: Optional[torch.Tensor] = None,
226 + ) -> torch.Tensor:
227 + attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None
228 + return self.attn(x, attn_mask=attn_mask)
229 +
230 + def forward(
231 + self,
232 + x: torch.Tensor,
233 + attn_mask: Optional[torch.Tensor] = None,
234 + ) -> torch.Tensor:
235 + x = x + self.attention(self.ln_1(x), attn_mask=attn_mask)
236 + x = x + self.mlp(self.ln_2(x))
237 + return x
238 +
239 +
240 +class TransformerBlock(nn.Module):
241 +
242 + def __init__(
243 + self,
244 + width: int,
245 + layers: int,
246 + heads: int,
247 + mlp_ratio: float = 4.0,
248 + norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
249 + quant_config: Optional[QuantizationConfig] = None,
250 + ):
251 + super().__init__()
252 + self.width = width
253 + self.layers = layers
254 +
255 + self.resblocks = nn.ModuleList([
256 + VisualAttentionBlock(width,
257 + heads,
258 + mlp_ratio,
259 + norm_layer=norm_layer,
260 + quant_config=quant_config)
261 + for _ in range(layers)
262 + ])
263 +
264 + def get_cast_dtype(self) -> torch.dtype:
265 + return self.resblocks[0].mlp.c_fc.weight.dtype
266 +
267 + def get_cast_device(self) -> torch.device:
268 + return self.resblocks[0].mlp.c_fc.weight.device
269 +
270 + def forward(self,
271 + x: torch.Tensor,
272 + attn_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
273 + for r in self.resblocks:
274 + x = r(x, attn_mask=attn_mask)
275 + return x
276 +
277 +
278 +class VisionTransformer(nn.Module):
279 +
280 + def __init__(self,
281 + image_size: int,
282 + patch_size: int,
283 + width: int,
284 + layers: int,
285 + heads: int,
286 + mlp_ratio: float,
287 + n_queries: int = 256,
288 + output_dim: int = 512,
289 + image_start_id: int = 151857,
290 + quant_config: Optional[QuantizationConfig] = None,
291 + **kwargs):
292 + super().__init__()
293 + image_height, image_width = self.image_size = (image_size, image_size)
294 + patch_height, patch_width = self.patch_size = (patch_size, patch_size)
295 + self.grid_size = (image_height // patch_height,
296 + image_width // patch_width)
297 + self.output_dim = output_dim
298 + self.conv1 = nn.Conv2d(in_channels=3,
299 + out_channels=width,
300 + kernel_size=patch_size,
301 + stride=patch_size,
302 + bias=False)
303 +
304 + # class embeddings and positional embeddings
305 + scale = width**-0.5
306 + self.positional_embedding = nn.Parameter(scale *
307 + torch.randn(256, width))
308 +
309 + norm_layer = partial(nn.LayerNorm, eps=1e-6)
310 +
311 + self.ln_pre = norm_layer(width)
312 + self.transformer = TransformerBlock(width,
313 + layers,
314 + heads,
315 + mlp_ratio,
316 + norm_layer=norm_layer,
317 + quant_config=quant_config)
318 +
319 + self.attn_pool = Resampler2(
320 + grid_size=int(math.sqrt(n_queries)),
321 + embed_dim=output_dim,
322 + num_heads=output_dim // 128,
323 + kv_dim=width,
324 + norm_layer=norm_layer,
325 + adaptive=False,
326 + do_post_projection=False,
327 + ).to(
328 + device=self.positional_embedding.device,
329 + dtype=self.positional_embedding.dtype,
330 + )
331 +
332 + self.ln_post = norm_layer(output_dim)
333 + self.proj = nn.Parameter(
334 + (output_dim**-0.5) * torch.randn(output_dim, output_dim))
335 +
336 + self.image_start_id = image_start_id
337 + self.image_end_id = image_start_id + 1
338 + self.image_pad_id = image_start_id + 2
339 +
340 + def forward(self, x: torch.Tensor) -> torch.Tensor:
341 + x = x.to(
342 + dtype=self.transformer.get_cast_dtype(),
343 + device=self.transformer.get_cast_device(),
344 + )
345 +
346 + # to patches
347 + x = self.conv1(x) # shape = [*, width, grid, grid]
348 + x = x.reshape(x.shape[0], x.shape[1],
349 + -1) # shape = [*, width, grid ** 2]
350 + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
351 +
352 + x = x + get_abs_pos(self.positional_embedding, int(math.sqrt(
353 + x.size(1))))
354 +
355 + x = self.ln_pre(x)
356 +
357 + x = x.permute(1, 0, 2) # NLD -> LND
358 + x = self.transformer(x)
359 + x = x.permute(1, 0, 2) # LND -> NLD
360 +
361 + x = self.attn_pool(x)
362 + x = self.ln_post(x)
363 + x = x @ self.proj
364 +
365 + return x
366 +
367 +
368 +class QwenVLModel(QWenModel):
369 +
370 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
371 + super().__init__(vllm_config=vllm_config, prefix=prefix)
372 +
373 + config = vllm_config.model_config.hf_config
374 + quant_config = vllm_config.quant_config
375 +
376 + self.visual = VisionTransformer(**config.visual,
377 + quant_config=quant_config)
378 +
379 +
380 +@lru_cache(maxsize=1)
381 +def _get_tokenizer_without_image_pad(
382 + tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
383 + """
384 + The logic of adding image pad tokens should only be applied in
385 + :class:`QwenVLProcessor`, so they are patched out here.
386 +
387 + The definition of the wrapped tokenizer can be found here:
388 + https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
389 + """
390 + new_tokenizer = copy.deepcopy(tokenizer)
391 +
392 + class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
393 +
394 + def tokenize(
395 + self,
396 + text: str,
397 + allowed_special: Union[AbstractSet[str], str] = "all",
398 + disallowed_special: Union[Collection[str], str] = (),
399 + **kwargs,
400 + ) -> list[Union[bytes, str]]:
401 + text = unicodedata.normalize("NFC", text)
402 +
403 + return [
404 + self.decoder[t] for t in self.tokenizer.encode(
405 + text,
406 + allowed_special=allowed_special,
407 + disallowed_special=disallowed_special,
408 + )
409 + ]
410 +
411 + def _decode(
412 + self,
413 + token_ids: Union[int, List[int]],
414 + skip_special_tokens: bool = False,
415 + errors: Optional[str] = None,
416 + **kwargs,
417 + ) -> str:
418 + if isinstance(token_ids, int):
419 + token_ids = [token_ids]
420 +
421 + return self.tokenizer.decode(
422 + token_ids,
423 + errors=errors or self.errors,
424 + )
425 +
426 + TokenizerWithoutImagePad.__name__ = \
427 + f"{tokenizer.__class__.__name__}WithoutImagePad"
428 +
429 + new_tokenizer.__class__ = TokenizerWithoutImagePad
430 + return new_tokenizer
431 +
432 +
433 +class QwenVLProcessor:
434 + """
435 + This model doesn't define its own HF processor,
436 + so we implement our own one here.
437 +
438 + We call the wrapped tokenizer to automatically insert image pad tokens:
439 + https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py#L245
440 +
441 + The image processor is defined here:
442 + https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
443 + """
444 +
445 + def __init__(
446 + self,
447 + config: PretrainedConfig,
448 + tokenizer: PreTrainedTokenizer,
449 + ) -> None:
450 + super().__init__()
451 +
452 + self.config = config
453 + self.tokenizer = tokenizer
454 +
455 + vision_config = config.visual
456 + image_size = vision_config["image_size"]
457 +
458 + self.image_transform = transforms.Compose([
459 + transforms.Resize(
460 + (image_size, image_size),
461 + interpolation=InterpolationMode.BICUBIC,
462 + ),
463 + transforms.ToTensor(),
464 + transforms.Normalize(
465 + mean=(0.48145466, 0.4578275, 0.40821073),
466 + std=(0.26862954, 0.26130258, 0.27577711),
467 + ),
468 + ])
469 +
470 + @property
471 + def image_start_tag(self) -> str:
472 + return self.tokenizer.image_start_tag # type: ignore
473 +
474 + @property
475 + def image_end_tag(self) -> str:
476 + return self.tokenizer.image_end_tag # type: ignore
477 +
478 + @property
479 + def image_pad_tag(self) -> str:
480 + return self.tokenizer.image_pad_tag # type: ignore
481 +
482 + def __call__(
483 + self,
484 + text: Optional[Union[TextInput, list[TextInput]]] = None,
485 + images: Optional[Union[ImageInput, list[ImageInput]]] = None,
486 + return_tensors: Optional[Union[str, TensorType]] = None,
487 + ) -> BatchFeature:
488 + if text is None:
489 + text = []
490 + if not isinstance(text, list):
491 + text = [text]
492 + if images is None:
493 + images = []
494 + if not isinstance(images, list):
495 + images = [images]
496 +
497 + text_inputs = self.tokenizer(text)
498 +
499 + if len(images) == 0:
500 + image_inputs = {}
501 + else:
502 + pixel_values = [self.image_transform(image) for image in images]
503 + image_inputs = {"pixel_values": torch.stack(pixel_values)}
504 +
505 + return BatchFeature(
506 + {
507 + **text_inputs,
508 + **image_inputs,
509 + },
510 + tensor_type=return_tensors,
511 + )
512 +
513 +
514 +class QwenVLProcessingInfo(BaseProcessingInfo):
515 +
516 + def get_tokenizer(self) -> PreTrainedTokenizer:
517 + tokenizer = self.ctx.tokenizer
518 + assert isinstance(tokenizer, PreTrainedTokenizer)
519 +
520 + return _get_tokenizer_without_image_pad(tokenizer)
521 +
522 + def get_hf_processor(self) -> QwenVLProcessor:
523 + tokenizer = self.ctx.tokenizer
524 + assert isinstance(tokenizer, PreTrainedTokenizer)
525 +
526 + return QwenVLProcessor(self.get_hf_config(), tokenizer)
527 +
528 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
529 + return {"image": None}
530 +
531 + def get_mm_max_tokens_per_item(
532 + self,
533 + seq_len: int,
534 + mm_counts: Mapping[str, int],
535 + ) -> Mapping[str, int]:
536 + return {"image": self.get_num_image_tokens()}
537 +
538 + def get_num_image_tokens(self) -> int:
539 + hf_config = self.get_hf_config()
540 + vision_config = hf_config.visual
541 +
542 + image_size = vision_config["image_size"]
543 + patch_size = vision_config["patch_size"]
544 + grid_length = image_size // patch_size // 2
545 + return grid_length * grid_length
546 +
547 +
548 +class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):
549 +
550 + def get_dummy_processor_inputs(
551 + self,
552 + seq_len: int,
553 + mm_counts: Mapping[str, int],
554 + ) -> ProcessorInputs:
555 + hf_config = self.info.get_hf_config()
556 + vision_config = hf_config.visual
557 +
558 + processor = self.info.get_hf_processor()
559 + img_start = processor.image_start_tag
560 + img_end = processor.image_end_tag
561 +
562 + target_width = target_height = vision_config["image_size"]
563 + num_images = mm_counts.get("image", 0)
564 +
565 + mm_data = {
566 + "image":
567 + self._get_dummy_images(width=target_width,
568 + height=target_height,
569 + num_images=num_images)
570 + }
571 +
572 + return ProcessorInputs(
573 + prompt_text="".join(f"Picture {i}: {img_start}{img_end}\n"
574 + for i in range(1, num_images + 1)),
575 + mm_data=mm_data,
576 + )
577 +
578 +
579 +class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
580 +
581 + def _call_hf_processor(
582 + self,
583 + prompt: str,
584 + mm_data: Mapping[str, object],
585 + mm_kwargs: Mapping[str, object],
586 + ) -> BatchFeature:
587 + # Drops anything between <img>/</img> tags; encoding with the tokenizer
588 + # will automatically add the image pads for the context.
589 + prompt, num_matched_images = re.subn(
590 + r"(Picture \d*: <img>).*?(<\/img>\n)",
591 + r"\1\2",
592 + prompt,
593 + )
594 +
595 + image_data = mm_data.get("images")
596 + if image_data is not None:
597 + assert isinstance(image_data, list)
598 +
599 + num_images = len(image_data)
600 + assert num_matched_images == num_images
601 +
602 + return super()._call_hf_processor(
603 + prompt=prompt,
604 + mm_data=mm_data,
605 + mm_kwargs=mm_kwargs,
606 + )
607 +
608 + def _get_mm_fields_config(
609 + self,
610 + hf_inputs: BatchFeature,
611 + hf_processor_mm_kwargs: Mapping[str, object],
612 + ) -> Mapping[str, MultiModalFieldConfig]:
613 + return dict(
614 + pixel_values=MultiModalFieldConfig.batched("image"),
615 + image_embeds=MultiModalFieldConfig.batched("image"),
616 + )
617 +
618 + def _get_prompt_replacements(
619 + self,
620 + mm_items: MultiModalDataItems,
621 + hf_processor_mm_kwargs: Mapping[str, object],
622 + out_mm_kwargs: MultiModalKwargs,
623 + ) -> list[PromptReplacement]:
624 + tokenizer = self.info.get_tokenizer()
625 + special_tokens: dict[str,
626 + int] = tokenizer.special_tokens # type: ignore
627 +
628 + processor = self.info.get_hf_processor()
629 + img_start_id = special_tokens[processor.image_start_tag]
630 + img_end_id = special_tokens[processor.image_end_tag]
631 + img_pad_id = special_tokens[processor.image_pad_tag]
632 +
633 + num_image_tokens = self.info.get_num_image_tokens()
634 + image_tokens = [img_pad_id] * num_image_tokens
635 +
636 + return [
637 + PromptReplacement(
638 + modality="image",
639 + target=[img_start_id, img_end_id],
640 + replacement=PromptReplacementDetails(
641 + full=[img_start_id] + image_tokens + [img_end_id],
642 + features=image_tokens,
643 + ),
644 + )
645 + ]
646 +
647 +
648 +@MULTIMODAL_REGISTRY.register_processor(QwenVLMultiModalProcessor,
649 + info=QwenVLProcessingInfo,
650 + dummy_inputs=QwenVLDummyInputsBuilder)
651 +class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,
652 + SupportsMultiModal):
653 + packed_modules_mapping = {
654 + "c_attn": ["c_attn"],
655 + "gate_up_proj": [
656 + "w2",
657 + "w1",
658 + ],
659 + }
660 + # LoRA specific attributes
661 + supported_lora_modules = [
662 + "c_attn",
663 + "gate_up_proj",
664 + "c_proj",
665 + # visual module
666 + "out_proj",
667 + "in_proj",
668 + "c_fc",
669 + # resampler
670 + "kv_proj",
671 + ]
672 +
673 + embedding_modules = {}
674 + embedding_padding_modules = []
675 +
676 + def get_mm_mapping(self) -> MultiModelKeys:
677 + """
678 + Get the module prefix in multimodal models
679 + """
680 + return MultiModelKeys.from_string_field(
681 + language_model="transformer.h",
682 + connector="transformer.visual.attn_pool",
683 + tower_model="transformer.visual.transformer")
684 +
685 + def __init__(
686 + self,
687 + *,
688 + vllm_config: VllmConfig,
689 + prefix: str = "",
690 + transformer_type: type[QwenVLModel] = QwenVLModel,
691 + ) -> None:
692 + super().__init__(
693 + vllm_config=vllm_config,
694 + prefix=prefix,
695 + transformer_type=transformer_type,
696 + )
697 +
698 + self.transformer: QwenVLModel
699 +
700 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
701 + h = w = self.config.visual["image_size"]
702 + expected_dims = (3, h, w)
703 + actual_dims = tuple(data.shape[1:])
704 +
705 + if actual_dims != expected_dims:
706 + expected_expr = ("batch_size", *map(str, expected_dims))
707 + raise ValueError(
708 + f"The expected shape of pixel values is {expected_expr}. "
709 + f"You supplied {tuple(data.shape)}.")
710 +
711 + return data
712 +
713 + def _parse_and_validate_image_input(
714 + self, **kwargs: object) -> Optional[QwenImageInputs]:
715 + pixel_values = kwargs.pop("pixel_values", None)
716 + image_embeds = kwargs.pop("image_embeds", None)
717 +
718 + if pixel_values is not None:
719 + if not isinstance(pixel_values, torch.Tensor):
720 + raise ValueError("Incorrect type of pixel values. "
721 + f"Got type: {type(pixel_values)}")
722 +
723 + return QwenImagePixelInputs(
724 + type="pixel_values",
725 + data=self._validate_pixel_values(
726 + flatten_bn(pixel_values, concat=True)),
727 + )
728 +
729 + if image_embeds is not None:
730 + if not isinstance(image_embeds, torch.Tensor):
731 + raise ValueError("Incorrect type of image embeddings. "
732 + f"Got type: {type(image_embeds)}")
733 +
734 + return QwenImageEmbeddingInputs(
735 + type="image_embeds",
736 + data=flatten_bn(image_embeds),
737 + )
738 +
739 + return None
740 +
741 + def _process_image_input(self,
742 + image_input: QwenImageInputs) -> torch.Tensor:
743 + if image_input["type"] == "image_embeds":
744 + return image_input["data"]
745 +
746 + return self.transformer.visual(image_input["data"])
747 +
748 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
749 + image_input = self._parse_and_validate_image_input(**kwargs)
750 + if image_input is None:
751 + return None
752 +
753 + vision_embeddings = self._process_image_input(image_input)
754 + return vision_embeddings
755 +
756 + def get_input_embeddings(
757 + self,
758 + input_ids: torch.Tensor,
759 + multimodal_embeddings: Optional[NestedTensors] = None,
760 + ) -> torch.Tensor:
761 + inputs_embeds = self.transformer.get_input_embeddings(input_ids)
762 +
763 + if multimodal_embeddings is not None:
764 + inputs_embeds = merge_multimodal_embeddings(
765 + input_ids, inputs_embeds, multimodal_embeddings,
766 + self.transformer.visual.image_pad_id)
767 +
768 + return inputs_embeds
769 +
770 + def forward(
771 + self,
772 + input_ids: torch.Tensor,
773 + positions: torch.Tensor,
774 + kv_caches: List[torch.Tensor],
775 + attn_metadata: AttentionMetadata,
776 + intermediate_tensors: Optional[IntermediateTensors] = None,
777 + inputs_embeds: Optional[torch.Tensor] = None,
778 + **kwargs: object,
779 + ) -> Union[torch.Tensor, IntermediateTensors]:
780 + if intermediate_tensors is not None:
781 + inputs_embeds = None
782 +
783 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
784 + # condition is for v0 compatibility.
785 + elif inputs_embeds is None:
786 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
787 + inputs_embeds = self.get_input_embeddings(input_ids,
788 + vision_embeddings)
789 + input_ids = None
790 +
791 + hidden_states = self.transformer(input_ids, positions, kv_caches,
792 + attn_metadata, intermediate_tensors,
793 + inputs_embeds)
794 + return hidden_states
vllm/model_executor/models/registry.py
+4 -5
@@ -39,7 +39,7 @@ _TEXT_GENERATION_MODELS = {
39 "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"),
40 "BambaForCausalLM": ("bamba", "BambaForCausalLM"),
41 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
42 - # ChatGLMModel supports multimodal
42 + "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
43 "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
44 "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
45 "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
@@ -90,7 +90,7 @@ _TEXT_GENERATION_MODELS = {
90 "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
91 "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
92 "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
93 - # QWenLMHeadModel supports multimodal
93 + "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
94 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
95 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
96 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
@@ -156,10 +156,9 @@ _MULTIMODAL_MODELS = {
156 "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
157 "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
158 "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
159 - "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
160 - "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
159 "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
160 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
161 + "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
162 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
163 "InternVLChatModel": ("internvl", "InternVLChatModel"),
164 "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
@@ -175,7 +174,7 @@ _MULTIMODAL_MODELS = {
174 "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
175 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
176 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
178 - "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
177 + "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), # noqa: E501
178 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
179 "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"), # noqa: E501
180 "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501