A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] Initialize Phi-3-vision support (#4986)
Isotr0py committed
Jun 18, 2024 at 10:34 UTC
daef218b5595a8c744ee143223f4f0544619ea9f
8 files changed
+571
docs/source/models/supported_models.rst
+4
@@ -135,6 +135,10 @@ Alongside each architecture, we include some popular models that use it.
135
- Phi-3-Small
136
- :code:`microsoft/Phi-3-small-8k-instruct`, :code:`microsoft/Phi-3-small-128k-instruct`, etc.
137
-
138
+ * - :code:`Phi3VForCausalLM`
139
+ - Phi-3-Vision
140
+ - :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
141
+ -
142
* - :code:`QWenLMHeadModel`
143
- Qwen
144
- :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
examples/phi3v_example.py
new
+57
@@ -0,0 +1,57 @@
1
+import os
2
+import subprocess
3
+
4
+from PIL import Image
5
+
6
+from vllm import LLM, SamplingParams
7
+from vllm.multimodal.image import ImagePixelData
8
+
9
+
10
+def run_phi3v():
11
+ model_path = "microsoft/Phi-3-vision-128k-instruct"
12
+ llm = LLM(
13
+ model=model_path,
14
+ trust_remote_code=True,
15
+ max_model_len=4096,
16
+ image_input_type="pixel_values",
17
+ image_token_id=32044,
18
+ image_input_shape="1,3,1008,1344",
19
+ image_feature_size=1921,
20
+ disable_image_processor=False,
21
+ )
22
+
23
+ image = Image.open("images/cherry_blossom.jpg")
24
+
25
+ # single-image prompt
26
+ prompt = "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n" # noqa: E501
27
+ prompt = prompt.replace("<|image_1|>", "<|image|>" * 1921 + "<s>")
28
+
29
+ sampling_params = SamplingParams(temperature=0, max_tokens=64)
30
+
31
+ outputs = llm.generate({
32
+ "prompt": prompt,
33
+ "sampling_params": sampling_params,
34
+ "multi_modal_data": ImagePixelData(image),
35
+ })
36
+ for o in outputs:
37
+ generated_text = o.outputs[0].text
38
+ print(generated_text)
39
+
40
+
41
+if __name__ == "__main__":
42
+ s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
43
+ local_directory = "images"
44
+
45
+ # Make sure the local directory exists or create it
46
+ os.makedirs(local_directory, exist_ok=True)
47
+
48
+ # Use AWS CLI to sync the directory, assume anonymous access
49
+ subprocess.check_call([
50
+ "aws",
51
+ "s3",
52
+ "sync",
53
+ s3_bucket_path,
54
+ local_directory,
55
+ "--no-sign-request",
56
+ ])
57
+ run_phi3v()
requirements-test.txt
+1
@@ -14,6 +14,7 @@ peft
14
requests
15
ray
16
sentence-transformers # required for embedding
17
+torchvision # required for the image processor of phi3v
18
19
# Benchmarking
20
aiohttp
tests/conftest.py
+3
@@ -144,6 +144,7 @@ class HfRunner:
144
model_name: str,
145
dtype: str = "half",
146
*,
147
+ model_kwargs: Optional[Dict[str, Any]] = None,
148
is_embedding_model: bool = False,
149
is_vision_model: bool = False,
150
) -> None:
@@ -166,11 +167,13 @@ class HfRunner:
167
else:
168
auto_cls = AutoModelForCausalLM
169
170
+ model_kwargs = model_kwargs if model_kwargs is not None else {}
171
self.model = self.wrap_device(
172
auto_cls.from_pretrained(
173
model_name,
174
torch_dtype=torch_dtype,
175
trust_remote_code=True,
176
+ **model_kwargs,
177
))
178
179
self.tokenizer = AutoTokenizer.from_pretrained(
tests/models/test_phi3v.py
new
+124
@@ -0,0 +1,124 @@
1
+from typing import List, Tuple
2
+
3
+import pytest
4
+from transformers import AutoTokenizer
5
+
6
+from vllm.config import VisionLanguageConfig
7
+from vllm.utils import is_cpu
8
+
9
+from ..conftest import IMAGE_FILES
10
+
11
+pytestmark = pytest.mark.llava
12
+
13
+# The image token is placed before "user" on purpose so that the test can pass
14
+HF_IMAGE_PROMPTS = [
15
+ "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
16
+ "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
17
+]
18
+
19
+assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
20
+
21
+
22
+def iter_phi3v_configs(model_name: str):
23
+ image_hw_to_feature_size = {
24
+ (1008, 1344): 1921,
25
+ }
26
+
27
+ for (h, w), f in image_hw_to_feature_size.items():
28
+ for input_type, input_shape in [
29
+ (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
30
+ ]:
31
+ yield (model_name,
32
+ VisionLanguageConfig(image_input_type=input_type,
33
+ image_feature_size=f,
34
+ image_token_id=32044,
35
+ image_input_shape=input_shape,
36
+ image_processor=model_name,
37
+ image_processor_revision=None))
38
+
39
+
40
+model_and_vl_config = [
41
+ *iter_phi3v_configs("microsoft/Phi-3-vision-128k-instruct"),
42
+]
43
+
44
+
45
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
46
+ vlm_config: VisionLanguageConfig, model_id: str):
47
+ """Sanitize vllm output to be comparable with hf output.
48
+ The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
49
+ x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
50
+ It also reduces `output_str` from "<image><image>bla" to "bla".
51
+ """
52
+ input_ids, output_str = vllm_output
53
+ image_token_id = vlm_config.image_token_id
54
+
55
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
56
+ image_token_str = tokenizer.decode(image_token_id)
57
+
58
+ hf_input_ids = [
59
+ input_id if input_id != image_token_id else 0
60
+ for idx, input_id in enumerate(input_ids)
61
+ ]
62
+ hf_output_str = output_str \
63
+ .replace(image_token_str * vlm_config.image_feature_size, "") \
64
+ .replace("<s>", " ").replace("<|user|>", "") \
65
+ .replace("<|end|>\n<|assistant|>", " ")
66
+
67
+ return hf_input_ids, hf_output_str
68
+
69
+
70
+target_dtype = "half"
71
+if is_cpu():
72
+ target_dtype = "bfloat16"
73
+
74
+
75
+# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
76
+# Since we use _attn_implementation="eager" for hf_runner, here is
77
+# numeric difference for longer context and test can't pass
78
+@pytest.mark.parametrize("model_and_config", model_and_vl_config)
79
+@pytest.mark.parametrize("dtype", [target_dtype])
80
+@pytest.mark.parametrize("max_tokens", [8])
81
+def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
82
+ model_and_config, dtype: str, max_tokens: int) -> None:
83
+ """Inference result should be the same between hf and vllm.
84
+
85
+ All the image fixtures for the test is under tests/images.
86
+ For huggingface runner, we provide the PIL images as input.
87
+ For vllm runner, we provide MultiModalData objects and corresponding
88
+ vision language config as input.
89
+ Note, the text input is also adjusted to abide by vllm contract.
90
+ The text output is sanitized to be able to compare with hf.
91
+ """
92
+ model_id, vlm_config = model_and_config
93
+
94
+ # use eager mode for hf runner, since phi3_v didn't work with flash_attn
95
+ hf_model_kwargs = {"_attn_implementation": "eager"}
96
+ with hf_runner(model_id, dtype=dtype,
97
+ model_kwargs=hf_model_kwargs) as hf_model:
98
+ hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
99
+ max_tokens,
100
+ images=hf_images)
101
+
102
+ vllm_image_prompts = [
103
+ p.replace("<|image_1|>",
104
+ "<|image|>" * vlm_config.image_feature_size + "<s>")
105
+ for p in HF_IMAGE_PROMPTS
106
+ ]
107
+
108
+ with vllm_runner(model_id,
109
+ max_model_len=2048,
110
+ dtype=dtype,
111
+ enforce_eager=True,
112
+ **vlm_config.as_cli_args_dict()) as vllm_model:
113
+ vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
114
+ max_tokens,
115
+ images=vllm_images)
116
+
117
+ for i in range(len(HF_IMAGE_PROMPTS)):
118
+ hf_output_ids, hf_output_str = hf_outputs[i]
119
+ vllm_output_ids, vllm_output_str = vllm_to_hf_output(
120
+ vllm_outputs[i], vlm_config, model_id)
121
+ assert hf_output_str == vllm_output_str, (
122
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
123
+ assert hf_output_ids == vllm_output_ids, (
124
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
vllm/model_executor/models/__init__.py
+1
@@ -49,6 +49,7 @@ _GENERATION_MODELS = {
49
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
50
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
51
"Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
52
+ "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
53
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
54
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
55
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
vllm/model_executor/models/phi3v.py
new
+379
@@ -0,0 +1,379 @@
1
+# coding=utf-8
2
+# Copyright 2024 The vLLM team.
3
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
4
+#
5
+# Licensed under the Apache License, Version 2.0 (the "License");
6
+# you may not use this file except in compliance with the License.
7
+# You may obtain a copy of the License at
8
+#
9
+# http://www.apache.org/licenses/LICENSE-2.0
10
+#
11
+# Unless required by applicable law or agreed to in writing, software
12
+# distributed under the License is distributed on an "AS IS" BASIS,
13
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+# See the License for the specific language governing permissions and
15
+# limitations under the License.
16
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
17
+
18
+import torch
19
+import torch.nn as nn
20
+from transformers import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig
21
+from transformers.utils import logging
22
+
23
+from vllm.attention import AttentionMetadata
24
+from vllm.config import CacheConfig, VisionLanguageConfig
25
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
26
+from vllm.model_executor.layers.quantization.base_config import (
27
+ QuantizationConfig)
28
+from vllm.model_executor.layers.sampler import Sampler
29
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
30
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31
+from vllm.model_executor.models.llama import LlamaModel
32
+from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
33
+from vllm.model_executor.sampling_metadata import SamplingMetadata
34
+from vllm.multimodal import MULTIMODAL_REGISTRY
35
+from vllm.multimodal.image import get_dummy_image_data
36
+from vllm.sequence import SamplerOutput
37
+
38
+logger = logging.get_logger(__name__)
39
+
40
+_KEYS_TO_MODIFY_MAPPING = {
41
+ "model.vision_embed_tokens": "vision_embed_tokens",
42
+}
43
+
44
+CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
45
+ hidden_act="quick_gelu",
46
+ hidden_size=1024,
47
+ image_size=336,
48
+ intermediate_size=4096,
49
+ num_attention_heads=16,
50
+ num_channels=3,
51
+ num_hidden_layers=24,
52
+ patch_size=14,
53
+ projection_dim=768)
54
+
55
+
56
+class Phi3ImageEmbeddingBase(nn.Module):
57
+
58
+ def __init__(self, wte=None) -> None:
59
+ super().__init__()
60
+ self.wte = wte
61
+ self.layer_idx: int
62
+ self.type_feature: str
63
+ self.img_processor: CLIPVisionModel
64
+
65
+ def set_img_features(self, img_features: torch.FloatTensor) -> None:
66
+ self.img_features = img_features
67
+
68
+ def set_img_sizes(self, img_sizes: torch.LongTensor) -> None:
69
+ self.img_sizes = img_sizes
70
+
71
+ def get_img_features(self,
72
+ img_embeds: torch.FloatTensor) -> torch.FloatTensor:
73
+ LAYER_IDX = self.layer_idx
74
+ TYPE_FEATURE = self.type_feature
75
+
76
+ img_processor_output = self.img_processor(img_embeds,
77
+ output_hidden_states=True)
78
+ img_feature = img_processor_output.hidden_states[LAYER_IDX]
79
+
80
+ if TYPE_FEATURE == "patch":
81
+ patch_feature = img_feature[:, 1:]
82
+ return patch_feature
83
+
84
+ if TYPE_FEATURE == "cls_patch":
85
+ return img_feature
86
+
87
+ raise NotImplementedError
88
+
89
+
90
+# adapted from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_embedding_phi3_v.py
91
+class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
92
+ """Phi3 Image embedding with HD transform."""
93
+
94
+ def __init__(self,
95
+ vision_language_config: VisionLanguageConfig,
96
+ config: PretrainedConfig,
97
+ wte=None) -> None:
98
+ super().__init__(wte)
99
+
100
+ self.image_token_id = vision_language_config.image_token_id
101
+ # n_embed or hidden_size
102
+ hidden_size = config.n_embd if hasattr(
103
+ config, 'n_embd') else config.hidden_size
104
+
105
+ clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
106
+ self.img_processor = CLIPVisionModel(clip_config)
107
+ image_dim_out = config.img_processor['image_dim_out']
108
+ self.num_img_tokens = config.img_processor['num_img_tokens']
109
+
110
+ self.image_dim_out = image_dim_out
111
+ self.img_sizes = None
112
+
113
+ # global_gn and sub_gn for hd transform, serves as line separator
114
+ self.use_hd_transform = config.embd_layer.get('use_hd_transform',
115
+ False)
116
+ self.with_learnable_separator = config.embd_layer.get(
117
+ 'with_learnable_separator', False)
118
+ self.hd_transform_order = config.embd_layer.get(
119
+ 'hd_transform_order', 'glb_sub')
120
+ # with_hd_transform and with_learnable_separator should have same value
121
+ assert self.use_hd_transform and self.with_learnable_separator
122
+
123
+ # 1024 * 4, merge spatial to channel dimension
124
+ self.glb_GN = nn.Parameter(torch.empty([1, 1, self.image_dim_out * 4]))
125
+ self.sub_GN = nn.Parameter(
126
+ torch.empty([1, 1, 1, self.image_dim_out * 4]))
127
+
128
+ dim_projection = hidden_size
129
+ depth = 2
130
+ layers = [nn.Linear(image_dim_out * 4, dim_projection)]
131
+ for _ in range(1, depth):
132
+ layers.extend(
133
+ [nn.GELU(),
134
+ nn.Linear(dim_projection, dim_projection)])
135
+ self.img_projection = nn.Sequential(*layers)
136
+
137
+ self.vocab_size = config.vocab_size
138
+ self.img_features = None
139
+
140
+ self.layer_idx = config.img_processor.get('layer_idx', -2)
141
+ self.type_feature = config.img_processor.get('type_feature', 'patch')
142
+
143
+ def forward(self,
144
+ input_ids: torch.LongTensor,
145
+ pixel_values: torch.FloatTensor,
146
+ image_sizes=None) -> torch.FloatTensor:
147
+ """process and merge text embeddings with image embeddings."""
148
+
149
+ img_embeds = pixel_values
150
+ img_sizes = image_sizes
151
+
152
+ if self.img_features is not None:
153
+ img_embeds = self.img_features.clone()
154
+ self.img_features = None
155
+
156
+ if self.img_sizes is not None:
157
+ img_sizes = self.img_sizes
158
+
159
+ input_shape = input_ids.size()
160
+ input_ids = input_ids.view(-1, input_shape[-1])
161
+
162
+ positions = torch.nonzero(input_ids == self.image_token_id)
163
+
164
+ select = False
165
+
166
+ target_device = self.img_projection[0].bias.device
167
+ target_dtype = self.img_projection[0].bias.dtype
168
+
169
+ if len(positions.tolist()) > 0:
170
+ # if self.use_hd_transform and img_sizes:
171
+ # img_embeds: (num_images, max_num_crops, 3, H, W)
172
+ # img_sizes: (num_images, 2).view(1, -1)
173
+
174
+ bs = img_embeds.shape[0]
175
+ # Nx(HW)xC
176
+ img_features = self.get_img_features(img_embeds.flatten(0, 1))
177
+ base_feat_height = base_feat_width = int(
178
+ img_features.shape[1]**0.5)
179
+
180
+ # bs x max_num_crops x (24x24) x C
181
+ img_features = img_features.view(
182
+ bs, -1, base_feat_height * base_feat_width, self.image_dim_out)
183
+ C = self.image_dim_out
184
+ H = base_feat_height
185
+
186
+ output_imgs = []
187
+ output_len = []
188
+
189
+ if isinstance(img_sizes, torch.Tensor):
190
+ img_sizes.squeeze_(0)
191
+
192
+ for _bs in range(bs):
193
+ h, w = img_sizes
194
+ h = h // 336
195
+ w = w // 336
196
+ B_ = h * w
197
+
198
+ # 1 x (24x24) x 1024
199
+ global_img_feature = img_features[_bs, :1]
200
+
201
+ # 1 x 12 x 12 x 4096
202
+ glb_img = global_img_feature \
203
+ .reshape(1, H // 2, 2, H // 2, 2,C) \
204
+ .permute(0, 1, 3, 2, 4, 5) \
205
+ .reshape(1, H // 2, H // 2, 4 * C)
206
+ temp_glb_GN = self.sub_GN.repeat(1, H // 2, 1, 1)
207
+
208
+ # 1 x 156 x 4096
209
+ glb_img = torch.cat([glb_img, temp_glb_GN],
210
+ dim=2).reshape(1, -1, 4 * C)
211
+
212
+ # (max_num_crops-1) x (12x12) x C
213
+ sub_img = img_features[_bs, 1:]
214
+ # 16x574x1024
215
+ # get rid of padding sub_img
216
+ sub_img = sub_img[:B_]
217
+
218
+ sub_img = sub_img.reshape(B_, H // 2, 2, H // 2, 2, C) \
219
+ .permute(0, 1, 3, 2, 4, 5).reshape(B_, -1, 4 * C)
220
+ sub_img = sub_img.reshape(1, h, w, 12, 12, -1) \
221
+ .permute(0, 1, 3, 2, 4, 5) \
222
+ .reshape(1, h * 12, w * 12, 4 * C)
223
+ temp_sub_GN = self.sub_GN.repeat(1, h * 12, 1, 1)
224
+ sub_img = torch.cat([sub_img, temp_sub_GN],
225
+ dim=2).reshape(1, -1, 4 * C)
226
+ # (1, num_img_tokens, 1024*4)
227
+
228
+ # glb + sub
229
+ if self.hd_transform_order == 'glb_sub':
230
+ output_imgs.append(
231
+ torch.cat([glb_img, self.glb_GN, sub_img], dim=1))
232
+ elif self.hd_transform_order == 'sub_glb':
233
+ output_imgs.append(
234
+ torch.cat([sub_img, self.glb_GN, glb_img], dim=1))
235
+
236
+ temp_len = int((h * w + 1) * 144 + 1 + (h + 1) * 12)
237
+ output_len.append(temp_len)
238
+
239
+ num_img_tokens = output_len
240
+ img_set_tensor = []
241
+ for _output_img in output_imgs:
242
+ img_feature_proj = self.img_projection(
243
+ _output_img.to(target_device, target_dtype))
244
+ img_set_tensor.append(img_feature_proj)
245
+ select = True
246
+
247
+ input_ids.clamp_min_(0).clamp_max_(self.vocab_size)
248
+
249
+ hidden_states = self.wte(input_ids)
250
+
251
+ if select:
252
+ idx = 0
253
+ for i, cnt in enumerate(num_img_tokens):
254
+ hidden_states[positions[idx, 0],
255
+ positions[idx, 1]:positions[idx, 1] +
256
+ cnt] = (img_set_tensor[i].to(
257
+ hidden_states.device, hidden_states.dtype))
258
+ idx += cnt
259
+
260
+ return hidden_states.squeeze(0)
261
+
262
+
263
+class Phi3VImagePixelInputs(TypedDict):
264
+ type: Literal["pixel_values"]
265
+ data: torch.Tensor
266
+ """Shape: (batch_size, 1 + num_patches, num_channels, height, width)"""
267
+
268
+ image_sizes: torch.Tensor
269
+ """Shape: (batch_size, 2)"""
270
+
271
+
272
+@MULTIMODAL_REGISTRY.register_image_pixel_input()
273
+@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
274
+class Phi3VForCausalLM(VisionLanguageModelBase):
275
+
276
+ def __init__(self,
277
+ config: PretrainedConfig,
278
+ vision_language_config: VisionLanguageConfig,
279
+ cache_config: Optional[CacheConfig] = None,
280
+ quant_config: Optional[QuantizationConfig] = None) -> None:
281
+ super().__init__(vision_language_config)
282
+ self.config = config
283
+ self.model = LlamaModel(config, cache_config, quant_config)
284
+ self.vision_embed_tokens = Phi3HDImageEmbedding(
285
+ vision_language_config, config, self.model.embed_tokens)
286
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
287
+ self.logits_processor = LogitsProcessor(config.vocab_size)
288
+ self.sampler = Sampler()
289
+
290
+ def _parse_and_validate_image_input(
291
+ self, **kwargs: object) -> Optional[Phi3VImagePixelInputs]:
292
+ pixel_values = kwargs.pop("pixel_values", None)
293
+ image_sizes = kwargs.pop("image_sizes", None)
294
+
295
+ expected_input_type = self.vision_language_config.image_input_type
296
+ ImageInputType = VisionLanguageConfig.ImageInputType
297
+
298
+ if expected_input_type != ImageInputType.PIXEL_VALUES:
299
+ raise ValueError(
300
+ f"Unexpected image input type: {expected_input_type}."
301
+ "Phi3v only support pixel_values input currently.")
302
+
303
+ if pixel_values is not None and image_sizes is not None:
304
+ return Phi3VImagePixelInputs(type="pixel_values",
305
+ data=pixel_values,
306
+ image_sizes=image_sizes)
307
+
308
+ return None
309
+
310
+ def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
311
+ kv_caches: List[torch.Tensor],
312
+ attn_metadata: AttentionMetadata, **kwargs: object):
313
+ image_input = self._parse_and_validate_image_input(**kwargs)
314
+
315
+ if image_input is not None:
316
+ inputs_embeds = self.vision_embed_tokens(
317
+ input_ids, image_input["data"], image_input["image_sizes"])
318
+
319
+ input_ids = None
320
+ else:
321
+ inputs_embeds = None
322
+
323
+ hidden_states = self.model(input_ids,
324
+ positions,
325
+ kv_caches,
326
+ attn_metadata,
327
+ inputs_embeds=inputs_embeds)
328
+
329
+ return hidden_states
330
+
331
+ def compute_logits(self, hidden_states: torch.Tensor,
332
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
333
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
334
+ sampling_metadata)
335
+ return logits
336
+
337
+ def sample(
338
+ self,
339
+ logits: torch.Tensor,
340
+ sampling_metadata: SamplingMetadata,
341
+ ) -> Optional[SamplerOutput]:
342
+ next_tokens = self.sampler(logits, sampling_metadata)
343
+ return next_tokens
344
+
345
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
346
+ stacked_params_mapping = [
347
+ # (param_name, shard_name, shard_id)
348
+ (".qkv_proj", ".q_proj", "q"),
349
+ (".qkv_proj", ".k_proj", "k"),
350
+ (".qkv_proj", ".v_proj", "v"),
351
+ (".gate_up_proj", ".gate_proj", 0),
352
+ (".gate_up_proj", ".up_proj", 1),
353
+ ]
354
+ params_dict = dict(self.named_parameters())
355
+ for name, loaded_weight in weights:
356
+ if "rotary_emb.inv_freq" in name:
357
+ continue
358
+ for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
359
+ if key_to_modify in name:
360
+ name = name.replace(key_to_modify, new_key)
361
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
362
+ # We only do sharding for language model
363
+ # and not vision model for now.
364
+ if "vision_embed_tokens" in name and self.vision_embed_tokens:
365
+ continue
366
+ if weight_name not in name:
367
+ continue
368
+ param = params_dict[name.replace(weight_name, param_name)]
369
+ weight_loader = param.weight_loader
370
+ weight_loader(param, loaded_weight, shard_id)
371
+ break
372
+ else:
373
+ # Skip loading extra bias for GPTQ models.
374
+ if name.endswith(".bias") and name not in params_dict:
375
+ continue
376
+ param = params_dict[name]
377
+ weight_loader = getattr(param, "weight_loader",
378
+ default_weight_loader)
379
+ weight_loader(param, loaded_weight)
vllm/multimodal/utils.py
+2
@@ -79,6 +79,8 @@ def get_full_image_text_prompt(image_prompt: str, text_prompt: str,
79
80
if config.hf_config.model_type in ("llava", "llava_next"):
81
full_prompt = f"{image_prompt}\n{text_prompt}"
82
+ elif config.hf_config.model_type == 'phi3_v':
83
+ full_prompt = f"{image_prompt}<s>\n{text_prompt}"
84
else:
85
raise ValueError(
86
f"Unsupported model type: {config.hf_config.model_type}")