@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / daef218b55
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] Initialize Phi-3-vision support (#4986)

Isotr0py committed Jun 18, 2024 at 10:34 UTC daef218b5595a8c744ee143223f4f0544619ea9f
8 files changed +571
docs/source/models/supported_models.rst
+4
@@ -135,6 +135,10 @@ Alongside each architecture, we include some popular models that use it.
135 - Phi-3-Small
136 - :code:`microsoft/Phi-3-small-8k-instruct`, :code:`microsoft/Phi-3-small-128k-instruct`, etc.
137 -
138 + * - :code:`Phi3VForCausalLM`
139 + - Phi-3-Vision
140 + - :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
141 + -
142 * - :code:`QWenLMHeadModel`
143 - Qwen
144 - :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
examples/phi3v_example.py new
+57
@@ -0,0 +1,57 @@
1 +import os
2 +import subprocess
3 +
4 +from PIL import Image
5 +
6 +from vllm import LLM, SamplingParams
7 +from vllm.multimodal.image import ImagePixelData
8 +
9 +
10 +def run_phi3v():
11 + model_path = "microsoft/Phi-3-vision-128k-instruct"
12 + llm = LLM(
13 + model=model_path,
14 + trust_remote_code=True,
15 + max_model_len=4096,
16 + image_input_type="pixel_values",
17 + image_token_id=32044,
18 + image_input_shape="1,3,1008,1344",
19 + image_feature_size=1921,
20 + disable_image_processor=False,
21 + )
22 +
23 + image = Image.open("images/cherry_blossom.jpg")
24 +
25 + # single-image prompt
26 + prompt = "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n" # noqa: E501
27 + prompt = prompt.replace("<|image_1|>", "<|image|>" * 1921 + "<s>")
28 +
29 + sampling_params = SamplingParams(temperature=0, max_tokens=64)
30 +
31 + outputs = llm.generate({
32 + "prompt": prompt,
33 + "sampling_params": sampling_params,
34 + "multi_modal_data": ImagePixelData(image),
35 + })
36 + for o in outputs:
37 + generated_text = o.outputs[0].text
38 + print(generated_text)
39 +
40 +
41 +if __name__ == "__main__":
42 + s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
43 + local_directory = "images"
44 +
45 + # Make sure the local directory exists or create it
46 + os.makedirs(local_directory, exist_ok=True)
47 +
48 + # Use AWS CLI to sync the directory, assume anonymous access
49 + subprocess.check_call([
50 + "aws",
51 + "s3",
52 + "sync",
53 + s3_bucket_path,
54 + local_directory,
55 + "--no-sign-request",
56 + ])
57 + run_phi3v()
requirements-test.txt
+1
@@ -14,6 +14,7 @@ peft
14 requests
15 ray
16 sentence-transformers # required for embedding
17 +torchvision # required for the image processor of phi3v
18
19 # Benchmarking
20 aiohttp
tests/conftest.py
+3
@@ -144,6 +144,7 @@ class HfRunner:
144 model_name: str,
145 dtype: str = "half",
146 *,
147 + model_kwargs: Optional[Dict[str, Any]] = None,
148 is_embedding_model: bool = False,
149 is_vision_model: bool = False,
150 ) -> None:
@@ -166,11 +167,13 @@ class HfRunner:
167 else:
168 auto_cls = AutoModelForCausalLM
169
170 + model_kwargs = model_kwargs if model_kwargs is not None else {}
171 self.model = self.wrap_device(
172 auto_cls.from_pretrained(
173 model_name,
174 torch_dtype=torch_dtype,
175 trust_remote_code=True,
176 + **model_kwargs,
177 ))
178
179 self.tokenizer = AutoTokenizer.from_pretrained(
tests/models/test_phi3v.py new
+124
@@ -0,0 +1,124 @@
1 +from typing import List, Tuple
2 +
3 +import pytest
4 +from transformers import AutoTokenizer
5 +
6 +from vllm.config import VisionLanguageConfig
7 +from vllm.utils import is_cpu
8 +
9 +from ..conftest import IMAGE_FILES
10 +
11 +pytestmark = pytest.mark.llava
12 +
13 +# The image token is placed before "user" on purpose so that the test can pass
14 +HF_IMAGE_PROMPTS = [
15 + "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
16 + "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
17 +]
18 +
19 +assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
20 +
21 +
22 +def iter_phi3v_configs(model_name: str):
23 + image_hw_to_feature_size = {
24 + (1008, 1344): 1921,
25 + }
26 +
27 + for (h, w), f in image_hw_to_feature_size.items():
28 + for input_type, input_shape in [
29 + (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
30 + ]:
31 + yield (model_name,
32 + VisionLanguageConfig(image_input_type=input_type,
33 + image_feature_size=f,
34 + image_token_id=32044,
35 + image_input_shape=input_shape,
36 + image_processor=model_name,
37 + image_processor_revision=None))
38 +
39 +
40 +model_and_vl_config = [
41 + *iter_phi3v_configs("microsoft/Phi-3-vision-128k-instruct"),
42 +]
43 +
44 +
45 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
46 + vlm_config: VisionLanguageConfig, model_id: str):
47 + """Sanitize vllm output to be comparable with hf output.
48 + The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
49 + x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
50 + It also reduces `output_str` from "<image><image>bla" to "bla".
51 + """
52 + input_ids, output_str = vllm_output
53 + image_token_id = vlm_config.image_token_id
54 +
55 + tokenizer = AutoTokenizer.from_pretrained(model_id)
56 + image_token_str = tokenizer.decode(image_token_id)
57 +
58 + hf_input_ids = [
59 + input_id if input_id != image_token_id else 0
60 + for idx, input_id in enumerate(input_ids)
61 + ]
62 + hf_output_str = output_str \
63 + .replace(image_token_str * vlm_config.image_feature_size, "") \
64 + .replace("<s>", " ").replace("<|user|>", "") \
65 + .replace("<|end|>\n<|assistant|>", " ")
66 +
67 + return hf_input_ids, hf_output_str
68 +
69 +
70 +target_dtype = "half"
71 +if is_cpu():
72 + target_dtype = "bfloat16"
73 +
74 +
75 +# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
76 +# Since we use _attn_implementation="eager" for hf_runner, here is
77 +# numeric difference for longer context and test can't pass
78 +@pytest.mark.parametrize("model_and_config", model_and_vl_config)
79 +@pytest.mark.parametrize("dtype", [target_dtype])
80 +@pytest.mark.parametrize("max_tokens", [8])
81 +def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
82 + model_and_config, dtype: str, max_tokens: int) -> None:
83 + """Inference result should be the same between hf and vllm.
84 +
85 + All the image fixtures for the test is under tests/images.
86 + For huggingface runner, we provide the PIL images as input.
87 + For vllm runner, we provide MultiModalData objects and corresponding
88 + vision language config as input.
89 + Note, the text input is also adjusted to abide by vllm contract.
90 + The text output is sanitized to be able to compare with hf.
91 + """
92 + model_id, vlm_config = model_and_config
93 +
94 + # use eager mode for hf runner, since phi3_v didn't work with flash_attn
95 + hf_model_kwargs = {"_attn_implementation": "eager"}
96 + with hf_runner(model_id, dtype=dtype,
97 + model_kwargs=hf_model_kwargs) as hf_model:
98 + hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
99 + max_tokens,
100 + images=hf_images)
101 +
102 + vllm_image_prompts = [
103 + p.replace("<|image_1|>",
104 + "<|image|>" * vlm_config.image_feature_size + "<s>")
105 + for p in HF_IMAGE_PROMPTS
106 + ]
107 +
108 + with vllm_runner(model_id,
109 + max_model_len=2048,
110 + dtype=dtype,
111 + enforce_eager=True,
112 + **vlm_config.as_cli_args_dict()) as vllm_model:
113 + vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
114 + max_tokens,
115 + images=vllm_images)
116 +
117 + for i in range(len(HF_IMAGE_PROMPTS)):
118 + hf_output_ids, hf_output_str = hf_outputs[i]
119 + vllm_output_ids, vllm_output_str = vllm_to_hf_output(
120 + vllm_outputs[i], vlm_config, model_id)
121 + assert hf_output_str == vllm_output_str, (
122 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
123 + assert hf_output_ids == vllm_output_ids, (
124 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
vllm/model_executor/models/__init__.py
+1
@@ -49,6 +49,7 @@ _GENERATION_MODELS = {
49 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
50 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
51 "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
52 + "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
53 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
54 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
55 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
vllm/model_executor/models/phi3v.py new
+379
@@ -0,0 +1,379 @@
1 +# coding=utf-8
2 +# Copyright 2024 The vLLM team.
3 +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
4 +#
5 +# Licensed under the Apache License, Version 2.0 (the "License");
6 +# you may not use this file except in compliance with the License.
7 +# You may obtain a copy of the License at
8 +#
9 +# http://www.apache.org/licenses/LICENSE-2.0
10 +#
11 +# Unless required by applicable law or agreed to in writing, software
12 +# distributed under the License is distributed on an "AS IS" BASIS,
13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 +# See the License for the specific language governing permissions and
15 +# limitations under the License.
16 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
17 +
18 +import torch
19 +import torch.nn as nn
20 +from transformers import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig
21 +from transformers.utils import logging
22 +
23 +from vllm.attention import AttentionMetadata
24 +from vllm.config import CacheConfig, VisionLanguageConfig
25 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
26 +from vllm.model_executor.layers.quantization.base_config import (
27 + QuantizationConfig)
28 +from vllm.model_executor.layers.sampler import Sampler
29 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
30 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31 +from vllm.model_executor.models.llama import LlamaModel
32 +from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
33 +from vllm.model_executor.sampling_metadata import SamplingMetadata
34 +from vllm.multimodal import MULTIMODAL_REGISTRY
35 +from vllm.multimodal.image import get_dummy_image_data
36 +from vllm.sequence import SamplerOutput
37 +
38 +logger = logging.get_logger(__name__)
39 +
40 +_KEYS_TO_MODIFY_MAPPING = {
41 + "model.vision_embed_tokens": "vision_embed_tokens",
42 +}
43 +
44 +CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
45 + hidden_act="quick_gelu",
46 + hidden_size=1024,
47 + image_size=336,
48 + intermediate_size=4096,
49 + num_attention_heads=16,
50 + num_channels=3,
51 + num_hidden_layers=24,
52 + patch_size=14,
53 + projection_dim=768)
54 +
55 +
56 +class Phi3ImageEmbeddingBase(nn.Module):
57 +
58 + def __init__(self, wte=None) -> None:
59 + super().__init__()
60 + self.wte = wte
61 + self.layer_idx: int
62 + self.type_feature: str
63 + self.img_processor: CLIPVisionModel
64 +
65 + def set_img_features(self, img_features: torch.FloatTensor) -> None:
66 + self.img_features = img_features
67 +
68 + def set_img_sizes(self, img_sizes: torch.LongTensor) -> None:
69 + self.img_sizes = img_sizes
70 +
71 + def get_img_features(self,
72 + img_embeds: torch.FloatTensor) -> torch.FloatTensor:
73 + LAYER_IDX = self.layer_idx
74 + TYPE_FEATURE = self.type_feature
75 +
76 + img_processor_output = self.img_processor(img_embeds,
77 + output_hidden_states=True)
78 + img_feature = img_processor_output.hidden_states[LAYER_IDX]
79 +
80 + if TYPE_FEATURE == "patch":
81 + patch_feature = img_feature[:, 1:]
82 + return patch_feature
83 +
84 + if TYPE_FEATURE == "cls_patch":
85 + return img_feature
86 +
87 + raise NotImplementedError
88 +
89 +
90 +# adapted from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_embedding_phi3_v.py
91 +class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
92 + """Phi3 Image embedding with HD transform."""
93 +
94 + def __init__(self,
95 + vision_language_config: VisionLanguageConfig,
96 + config: PretrainedConfig,
97 + wte=None) -> None:
98 + super().__init__(wte)
99 +
100 + self.image_token_id = vision_language_config.image_token_id
101 + # n_embed or hidden_size
102 + hidden_size = config.n_embd if hasattr(
103 + config, 'n_embd') else config.hidden_size
104 +
105 + clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
106 + self.img_processor = CLIPVisionModel(clip_config)
107 + image_dim_out = config.img_processor['image_dim_out']
108 + self.num_img_tokens = config.img_processor['num_img_tokens']
109 +
110 + self.image_dim_out = image_dim_out
111 + self.img_sizes = None
112 +
113 + # global_gn and sub_gn for hd transform, serves as line separator
114 + self.use_hd_transform = config.embd_layer.get('use_hd_transform',
115 + False)
116 + self.with_learnable_separator = config.embd_layer.get(
117 + 'with_learnable_separator', False)
118 + self.hd_transform_order = config.embd_layer.get(
119 + 'hd_transform_order', 'glb_sub')
120 + # with_hd_transform and with_learnable_separator should have same value
121 + assert self.use_hd_transform and self.with_learnable_separator
122 +
123 + # 1024 * 4, merge spatial to channel dimension
124 + self.glb_GN = nn.Parameter(torch.empty([1, 1, self.image_dim_out * 4]))
125 + self.sub_GN = nn.Parameter(
126 + torch.empty([1, 1, 1, self.image_dim_out * 4]))
127 +
128 + dim_projection = hidden_size
129 + depth = 2
130 + layers = [nn.Linear(image_dim_out * 4, dim_projection)]
131 + for _ in range(1, depth):
132 + layers.extend(
133 + [nn.GELU(),
134 + nn.Linear(dim_projection, dim_projection)])
135 + self.img_projection = nn.Sequential(*layers)
136 +
137 + self.vocab_size = config.vocab_size
138 + self.img_features = None
139 +
140 + self.layer_idx = config.img_processor.get('layer_idx', -2)
141 + self.type_feature = config.img_processor.get('type_feature', 'patch')
142 +
143 + def forward(self,
144 + input_ids: torch.LongTensor,
145 + pixel_values: torch.FloatTensor,
146 + image_sizes=None) -> torch.FloatTensor:
147 + """process and merge text embeddings with image embeddings."""
148 +
149 + img_embeds = pixel_values
150 + img_sizes = image_sizes
151 +
152 + if self.img_features is not None:
153 + img_embeds = self.img_features.clone()
154 + self.img_features = None
155 +
156 + if self.img_sizes is not None:
157 + img_sizes = self.img_sizes
158 +
159 + input_shape = input_ids.size()
160 + input_ids = input_ids.view(-1, input_shape[-1])
161 +
162 + positions = torch.nonzero(input_ids == self.image_token_id)
163 +
164 + select = False
165 +
166 + target_device = self.img_projection[0].bias.device
167 + target_dtype = self.img_projection[0].bias.dtype
168 +
169 + if len(positions.tolist()) > 0:
170 + # if self.use_hd_transform and img_sizes:
171 + # img_embeds: (num_images, max_num_crops, 3, H, W)
172 + # img_sizes: (num_images, 2).view(1, -1)
173 +
174 + bs = img_embeds.shape[0]
175 + # Nx(HW)xC
176 + img_features = self.get_img_features(img_embeds.flatten(0, 1))
177 + base_feat_height = base_feat_width = int(
178 + img_features.shape[1]**0.5)
179 +
180 + # bs x max_num_crops x (24x24) x C
181 + img_features = img_features.view(
182 + bs, -1, base_feat_height * base_feat_width, self.image_dim_out)
183 + C = self.image_dim_out
184 + H = base_feat_height
185 +
186 + output_imgs = []
187 + output_len = []
188 +
189 + if isinstance(img_sizes, torch.Tensor):
190 + img_sizes.squeeze_(0)
191 +
192 + for _bs in range(bs):
193 + h, w = img_sizes
194 + h = h // 336
195 + w = w // 336
196 + B_ = h * w
197 +
198 + # 1 x (24x24) x 1024
199 + global_img_feature = img_features[_bs, :1]
200 +
201 + # 1 x 12 x 12 x 4096
202 + glb_img = global_img_feature \
203 + .reshape(1, H // 2, 2, H // 2, 2,C) \
204 + .permute(0, 1, 3, 2, 4, 5) \
205 + .reshape(1, H // 2, H // 2, 4 * C)
206 + temp_glb_GN = self.sub_GN.repeat(1, H // 2, 1, 1)
207 +
208 + # 1 x 156 x 4096
209 + glb_img = torch.cat([glb_img, temp_glb_GN],
210 + dim=2).reshape(1, -1, 4 * C)
211 +
212 + # (max_num_crops-1) x (12x12) x C
213 + sub_img = img_features[_bs, 1:]
214 + # 16x574x1024
215 + # get rid of padding sub_img
216 + sub_img = sub_img[:B_]
217 +
218 + sub_img = sub_img.reshape(B_, H // 2, 2, H // 2, 2, C) \
219 + .permute(0, 1, 3, 2, 4, 5).reshape(B_, -1, 4 * C)
220 + sub_img = sub_img.reshape(1, h, w, 12, 12, -1) \
221 + .permute(0, 1, 3, 2, 4, 5) \
222 + .reshape(1, h * 12, w * 12, 4 * C)
223 + temp_sub_GN = self.sub_GN.repeat(1, h * 12, 1, 1)
224 + sub_img = torch.cat([sub_img, temp_sub_GN],
225 + dim=2).reshape(1, -1, 4 * C)
226 + # (1, num_img_tokens, 1024*4)
227 +
228 + # glb + sub
229 + if self.hd_transform_order == 'glb_sub':
230 + output_imgs.append(
231 + torch.cat([glb_img, self.glb_GN, sub_img], dim=1))
232 + elif self.hd_transform_order == 'sub_glb':
233 + output_imgs.append(
234 + torch.cat([sub_img, self.glb_GN, glb_img], dim=1))
235 +
236 + temp_len = int((h * w + 1) * 144 + 1 + (h + 1) * 12)
237 + output_len.append(temp_len)
238 +
239 + num_img_tokens = output_len
240 + img_set_tensor = []
241 + for _output_img in output_imgs:
242 + img_feature_proj = self.img_projection(
243 + _output_img.to(target_device, target_dtype))
244 + img_set_tensor.append(img_feature_proj)
245 + select = True
246 +
247 + input_ids.clamp_min_(0).clamp_max_(self.vocab_size)
248 +
249 + hidden_states = self.wte(input_ids)
250 +
251 + if select:
252 + idx = 0
253 + for i, cnt in enumerate(num_img_tokens):
254 + hidden_states[positions[idx, 0],
255 + positions[idx, 1]:positions[idx, 1] +
256 + cnt] = (img_set_tensor[i].to(
257 + hidden_states.device, hidden_states.dtype))
258 + idx += cnt
259 +
260 + return hidden_states.squeeze(0)
261 +
262 +
263 +class Phi3VImagePixelInputs(TypedDict):
264 + type: Literal["pixel_values"]
265 + data: torch.Tensor
266 + """Shape: (batch_size, 1 + num_patches, num_channels, height, width)"""
267 +
268 + image_sizes: torch.Tensor
269 + """Shape: (batch_size, 2)"""
270 +
271 +
272 +@MULTIMODAL_REGISTRY.register_image_pixel_input()
273 +@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
274 +class Phi3VForCausalLM(VisionLanguageModelBase):
275 +
276 + def __init__(self,
277 + config: PretrainedConfig,
278 + vision_language_config: VisionLanguageConfig,
279 + cache_config: Optional[CacheConfig] = None,
280 + quant_config: Optional[QuantizationConfig] = None) -> None:
281 + super().__init__(vision_language_config)
282 + self.config = config
283 + self.model = LlamaModel(config, cache_config, quant_config)
284 + self.vision_embed_tokens = Phi3HDImageEmbedding(
285 + vision_language_config, config, self.model.embed_tokens)
286 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
287 + self.logits_processor = LogitsProcessor(config.vocab_size)
288 + self.sampler = Sampler()
289 +
290 + def _parse_and_validate_image_input(
291 + self, **kwargs: object) -> Optional[Phi3VImagePixelInputs]:
292 + pixel_values = kwargs.pop("pixel_values", None)
293 + image_sizes = kwargs.pop("image_sizes", None)
294 +
295 + expected_input_type = self.vision_language_config.image_input_type
296 + ImageInputType = VisionLanguageConfig.ImageInputType
297 +
298 + if expected_input_type != ImageInputType.PIXEL_VALUES:
299 + raise ValueError(
300 + f"Unexpected image input type: {expected_input_type}."
301 + "Phi3v only support pixel_values input currently.")
302 +
303 + if pixel_values is not None and image_sizes is not None:
304 + return Phi3VImagePixelInputs(type="pixel_values",
305 + data=pixel_values,
306 + image_sizes=image_sizes)
307 +
308 + return None
309 +
310 + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
311 + kv_caches: List[torch.Tensor],
312 + attn_metadata: AttentionMetadata, **kwargs: object):
313 + image_input = self._parse_and_validate_image_input(**kwargs)
314 +
315 + if image_input is not None:
316 + inputs_embeds = self.vision_embed_tokens(
317 + input_ids, image_input["data"], image_input["image_sizes"])
318 +
319 + input_ids = None
320 + else:
321 + inputs_embeds = None
322 +
323 + hidden_states = self.model(input_ids,
324 + positions,
325 + kv_caches,
326 + attn_metadata,
327 + inputs_embeds=inputs_embeds)
328 +
329 + return hidden_states
330 +
331 + def compute_logits(self, hidden_states: torch.Tensor,
332 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
333 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
334 + sampling_metadata)
335 + return logits
336 +
337 + def sample(
338 + self,
339 + logits: torch.Tensor,
340 + sampling_metadata: SamplingMetadata,
341 + ) -> Optional[SamplerOutput]:
342 + next_tokens = self.sampler(logits, sampling_metadata)
343 + return next_tokens
344 +
345 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
346 + stacked_params_mapping = [
347 + # (param_name, shard_name, shard_id)
348 + (".qkv_proj", ".q_proj", "q"),
349 + (".qkv_proj", ".k_proj", "k"),
350 + (".qkv_proj", ".v_proj", "v"),
351 + (".gate_up_proj", ".gate_proj", 0),
352 + (".gate_up_proj", ".up_proj", 1),
353 + ]
354 + params_dict = dict(self.named_parameters())
355 + for name, loaded_weight in weights:
356 + if "rotary_emb.inv_freq" in name:
357 + continue
358 + for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
359 + if key_to_modify in name:
360 + name = name.replace(key_to_modify, new_key)
361 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
362 + # We only do sharding for language model
363 + # and not vision model for now.
364 + if "vision_embed_tokens" in name and self.vision_embed_tokens:
365 + continue
366 + if weight_name not in name:
367 + continue
368 + param = params_dict[name.replace(weight_name, param_name)]
369 + weight_loader = param.weight_loader
370 + weight_loader(param, loaded_weight, shard_id)
371 + break
372 + else:
373 + # Skip loading extra bias for GPTQ models.
374 + if name.endswith(".bias") and name not in params_dict:
375 + continue
376 + param = params_dict[name]
377 + weight_loader = getattr(param, "weight_loader",
378 + default_weight_loader)
379 + weight_loader(param, loaded_weight)
vllm/multimodal/utils.py
+2
@@ -79,6 +79,8 @@ def get_full_image_text_prompt(image_prompt: str, text_prompt: str,
79
80 if config.hf_config.model_type in ("llava", "llava_next"):
81 full_prompt = f"{image_prompt}\n{text_prompt}"
82 + elif config.hf_config.model_type == 'phi3_v':
83 + full_prompt = f"{image_prompt}<s>\n{text_prompt}"
84 else:
85 raise ValueError(
86 f"Unsupported model type: {config.hf_config.model_type}")