@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e6e42e4b17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core][VLM] Support image embeddings as input (#6613)

Roger Wang committed Aug 12, 2024 at 01:16 UTC e6e42e4b1759b8703bf6cb80aefa9c0ec35044f6
13 files changed +518 -139
docs/source/models/vlm.rst
+11
@@ -49,6 +49,17 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptI
49 "multi_modal_data": {"image": image},
50 })
51
52 + for o in outputs:
53 + generated_text = o.outputs[0].text
54 + print(generated_text)
55 +
56 + # Inference with image embeddings as input
57 + image_embeds = torch.load(...) # torch.Tensor of shape (1, image_feature_size, hidden_size of LM)
58 + outputs = llm.generate({
59 + "prompt": prompt,
60 + "multi_modal_data": {"image": image_embeds},
61 + })
62 +
63 for o in outputs:
64 generated_text = o.outputs[0].text
65 print(generated_text)
tests/models/test_llava_image_embeds.py new
+159
@@ -0,0 +1,159 @@
1 +from typing import List, Optional, Tuple, Type
2 +
3 +import pytest
4 +from transformers import AutoConfig, AutoTokenizer
5 +
6 +from vllm.sequence import SampleLogprobs
7 +
8 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9 +from .utils import check_logprobs_close
10 +
11 +pytestmark = pytest.mark.vlm
12 +
13 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
14 + "stop_sign":
15 + "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
16 + "cherry_blossom":
17 + "USER: <image>\nWhat is the season?\nASSISTANT:",
18 +})
19 +
20 +models = [
21 + "llava-hf/llava-1.5-7b-hf",
22 +]
23 +
24 +
25 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
26 + Optional[SampleLogprobs]],
27 + model: str):
28 + """Sanitize vllm output to be comparable with hf output."""
29 + output_ids, output_str, out_logprobs = vllm_output
30 +
31 + config = AutoConfig.from_pretrained(model)
32 + image_token_id = config.image_token_index
33 +
34 + tokenizer = AutoTokenizer.from_pretrained(model)
35 + eos_token_id = tokenizer.eos_token_id
36 +
37 + hf_output_ids = [
38 + token_id for idx, token_id in enumerate(output_ids)
39 + if token_id != image_token_id or output_ids[idx - 1] != image_token_id
40 + ]
41 +
42 + assert output_str[0] == " "
43 + hf_output_str = output_str[1:]
44 + if hf_output_ids[-1] == eos_token_id:
45 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
46 +
47 + return hf_output_ids, hf_output_str, out_logprobs
48 +
49 +
50 +def run_test(
51 + hf_runner: Type[HfRunner],
52 + vllm_runner: Type[VllmRunner],
53 + image_assets: _ImageAssets,
54 + model: str,
55 + *,
56 + size_factors: List[float],
57 + dtype: str,
58 + max_tokens: int,
59 + num_logprobs: int,
60 + tensor_parallel_size: int,
61 + distributed_executor_backend: Optional[str] = None,
62 +):
63 + """Inference result should be the same between hf and vllm.
64 +
65 + All the image fixtures for the test is under tests/images.
66 + For huggingface runner, we provide the PIL images as input.
67 + For vllm runner, we provide MultiModalDataDict objects
68 + and corresponding vision language config as input.
69 + Note, the text input is also adjusted to abide by vllm contract.
70 + The text output is sanitized to be able to compare with hf.
71 + """
72 +
73 + # vLLM to load from image embeddings
74 + vllm_images = [asset.image_embeds for asset in image_assets]
75 +
76 + # transformers to load from PIL images
77 + hf_images = [asset.pil_image for asset in image_assets]
78 +
79 + vllm_inputs_per_image = [(
80 + [prompt for _ in size_factors],
81 + [image for _ in size_factors],
82 + ) for image, prompt in zip(vllm_images, HF_IMAGE_PROMPTS)]
83 +
84 + hf_inputs_per_image = [(
85 + [prompt for _ in size_factors],
86 + [image for _ in size_factors],
87 + ) for image, prompt in zip(hf_images, HF_IMAGE_PROMPTS)]
88 +
89 + # NOTE: take care of the order. run vLLM first, and then run HF.
90 + # vLLM needs a fresh new process without cuda initialization.
91 + # if we run HF first, the cuda initialization will be done and it
92 + # will hurt multiprocessing backend with fork method (the default method).
93 +
94 + # max_model_len should be greater than image_feature_size
95 + with vllm_runner(model,
96 + dtype=dtype,
97 + tensor_parallel_size=tensor_parallel_size,
98 + distributed_executor_backend=distributed_executor_backend,
99 + enforce_eager=True) as vllm_model:
100 + vllm_outputs_per_image = [
101 + vllm_model.generate_greedy_logprobs(prompts,
102 + max_tokens,
103 + num_logprobs=num_logprobs,
104 + images=images)
105 + for prompts, images in vllm_inputs_per_image
106 + ]
107 +
108 + with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
109 + hf_outputs_per_image = [
110 + hf_model.generate_greedy_logprobs_limit(prompts,
111 + max_tokens,
112 + num_logprobs=num_logprobs,
113 + images=images)
114 + for prompts, images in hf_inputs_per_image
115 + ]
116 +
117 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
118 + vllm_outputs_per_image):
119 + # TODO: Check whether using original CLIPVisionModel can improve
120 + # consistency against HF
121 + check_logprobs_close(
122 + outputs_0_lst=hf_outputs,
123 + outputs_1_lst=[
124 + vllm_to_hf_output(vllm_output, model)
125 + for vllm_output in vllm_outputs
126 + ],
127 + name_0="hf",
128 + name_1="vllm",
129 + )
130 +
131 +
132 +@pytest.mark.parametrize("model", models)
133 +@pytest.mark.parametrize(
134 + "size_factors",
135 + [
136 + # No image
137 + [],
138 + # Single-scale
139 + [1.0],
140 + # Single-scale, batched
141 + [1.0, 1.0, 1.0],
142 + ],
143 +)
144 +@pytest.mark.parametrize("dtype", ["half"])
145 +@pytest.mark.parametrize("max_tokens", [128])
146 +@pytest.mark.parametrize("num_logprobs", [5])
147 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
148 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
149 + run_test(
150 + hf_runner,
151 + vllm_runner,
152 + image_assets,
153 + model,
154 + size_factors=size_factors,
155 + dtype=dtype,
156 + max_tokens=max_tokens,
157 + num_logprobs=num_logprobs,
158 + tensor_parallel_size=1,
159 + )
vllm/assets/image.py
+10
@@ -1,6 +1,7 @@
1 from dataclasses import dataclass
2 from typing import Literal
3
4 +import torch
5 from PIL import Image
6
7 from vllm.assets.base import get_vllm_public_assets
@@ -18,3 +19,12 @@ class ImageAsset:
19 image_path = get_vllm_public_assets(filename=f"{self.name}.jpg",
20 s3_prefix=VLM_IMAGES_DIR)
21 return Image.open(image_path)
22 +
23 + @property
24 + def image_embeds(self) -> torch.Tensor:
25 + """
26 + Image embeddings, only used for testing purposes with llava 1.5.
27 + """
28 + image_path = get_vllm_public_assets(filename=f"{self.name}.pt",
29 + s3_prefix=VLM_IMAGES_DIR)
30 + return torch.load(image_path)
vllm/model_executor/models/blip2.py
+49 -23
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
1 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3 import torch
4 import torch.nn as nn
@@ -28,6 +28,29 @@ _KEYS_TO_MODIFY_MAPPING = {
28 "language_model.model": "language_model",
29 }
30
31 +# We use this internally as placeholders since there is no image token
32 +# defined on the HuggingFace repo
33 +BLIP2_IMAGE_TOKEN = "<image>"
34 +BLIP2_IMAGE_TOKEN_ID = 50265
35 +
36 +
37 +class Blip2ImagePixelInputs(TypedDict):
38 + type: Literal["pixel_values"]
39 + data: torch.Tensor
40 + """Shape: (batch_size, num_channels, height, width)"""
41 +
42 +
43 +class Blip2ImageEmbeddingInputs(TypedDict):
44 + type: Literal["image_embeds"]
45 + data: torch.Tensor
46 + """Shape: `(batch_size, image_feature_size, hidden_size)`
47 +
48 + `hidden_size` must match the hidden size of language model backbone.
49 + """
50 +
51 +
52 +Blip2ImageInputs = Union[Blip2ImagePixelInputs, Blip2ImageEmbeddingInputs]
53 +
54
55 class Blip2QFormerMultiHeadAttention(nn.Module):
56
@@ -375,20 +398,6 @@ class Blip2QFormerModel(nn.Module):
398 return sequence_output
399
400
378 -class Blip2ImagePixelInputs(TypedDict):
379 - type: Literal["pixel_values"]
380 - data: torch.Tensor
381 - """Shape: (batch_size, num_channels, height, width)"""
382 -
383 -
384 -Blip2ImageInputs = Blip2ImagePixelInputs
385 -
386 -# We use this internally as placeholders since there is no image token
387 -# defined on the HuggingFace repo
388 -BLIP2_IMAGE_TOKEN = "<image>"
389 -BLIP2_IMAGE_TOKEN_ID = 50265
390 -
391 -
401 def get_blip2_image_feature_size(hf_config: Blip2Config) -> int:
402 return hf_config.num_query_tokens
403
@@ -506,18 +515,31 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
515 def _parse_and_validate_image_input(
516 self, **kwargs: object) -> Optional[Blip2ImageInputs]:
517 pixel_values = kwargs.pop("pixel_values", None)
518 + image_embeds = kwargs.pop("image_embeds", None)
519
510 - if pixel_values is None:
520 + if pixel_values is None and image_embeds is None:
521 return None
522
513 - if not isinstance(pixel_values, torch.Tensor):
514 - raise ValueError("Incorrect type of pixel values. "
515 - f"Got type: {type(pixel_values)}")
523 + if pixel_values is not None:
524 + if not isinstance(pixel_values, torch.Tensor):
525 + raise ValueError("Incorrect type of pixel values. "
526 + f"Got type: {type(pixel_values)}")
527
517 - return Blip2ImagePixelInputs(
518 - type="pixel_values",
519 - data=self._validate_pixel_values(pixel_values),
520 - )
528 + return Blip2ImagePixelInputs(
529 + type="pixel_values",
530 + data=self._validate_pixel_values(pixel_values),
531 + )
532 +
533 + if image_embeds is not None:
534 + if not isinstance(image_embeds, torch.Tensor):
535 + raise ValueError("Incorrect type of image embeddings. "
536 + f"Got type: {type(image_embeds)}")
537 + return Blip2ImageEmbeddingInputs(
538 + type="image_embeds",
539 + data=image_embeds,
540 + )
541 +
542 + raise AssertionError("This line should be unreachable.")
543
544 def _image_pixels_to_features(self, vision_model: BlipVisionModel,
545 pixel_values: torch.Tensor) -> torch.Tensor:
@@ -538,6 +560,10 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
560
561 def _process_image_input(self,
562 image_input: Blip2ImageInputs) -> torch.Tensor:
563 +
564 + if image_input["type"] == "image_embeds":
565 + return image_input["data"]
566 +
567 assert self.vision_model is not None
568 image_features = self._process_image_pixels(image_input)
569
vllm/model_executor/models/clip.py
+7 -1
@@ -88,7 +88,13 @@ def input_processor_for_clip(
88 tokenizer = cached_get_tokenizer(model_config.tokenizer)
89
90 if image_feature_size_override is None:
91 - image_feature_size = get_clip_image_feature_size(hf_config)
91 + image_data = multi_modal_data["image"]
92 + if isinstance(image_data, Image.Image):
93 + image_feature_size = get_clip_image_feature_size(hf_config)
94 + elif isinstance(image_data, torch.Tensor):
95 + image_feature_size = image_data.shape[0]
96 + else:
97 + raise TypeError(f"Invalid image type: {type(image_data)}")
98 else:
99 image_feature_size = image_feature_size_override
100
vllm/model_executor/models/fuyu.py
+10 -3
@@ -234,7 +234,8 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
234 cache_config=cache_config,
235 quant_config=quant_config)
236
237 - def _parse_and_validate_image_input(self, **kwargs: object):
237 + def _parse_and_validate_image_input(
238 + self, **kwargs: object) -> Optional[FuyuImagePixelInputs]:
239 image_patches = kwargs.pop("image_patches", None)
240
241 if isinstance(image_patches, torch.Tensor):
@@ -249,6 +250,13 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
250 data=image_patches)
251 return None
252
253 + def _process_image_input(
254 + self, image_input: FuyuImagePixelInputs) -> torch.Tensor:
255 +
256 + assert self.vision_embed_tokens is not None
257 + vision_embeddings, _ = self.vision_embed_tokens(image_input["data"])
258 + return vision_embeddings
259 +
260 def forward(
261 self,
262 input_ids: torch.Tensor,
@@ -261,8 +269,7 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
269 image_input = self._parse_and_validate_image_input(**kwargs)
270
271 if image_input is not None:
264 - vision_embeddings, _ = self.vision_embed_tokens(
265 - image_input["data"])
272 + vision_embeddings = self._process_image_input(image_input)
273 inputs_embeds = self.language_model.model.embed_tokens(input_ids)
274 inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
275 vision_embeddings,
vllm/model_executor/models/internvl.py
+54 -13
@@ -50,6 +50,19 @@ class InternVLImagePixelInputs(TypedDict):
50 """
51
52
53 +class InternVLImageEmbeddingInputs(TypedDict):
54 + type: Literal["image_embeds"]
55 + data: Union[torch.Tensor, List[torch.Tensor]]
56 + """Shape: `(batch_size, image_feature_size, hidden_size)`
57 +
58 + `hidden_size` must match the hidden size of language model backbone.
59 + """
60 +
61 +
62 +InternVLImageInputs = Union[InternVLImagePixelInputs,
63 + InternVLImageEmbeddingInputs]
64 +
65 +
66 # copied from https://huggingface.co/OpenGVLab/InternVL2-1B
67 def build_transform(input_size):
68 MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
@@ -193,8 +206,10 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
206 # add thumbnail image if num_blocks > 1
207 if hf_config.use_thumbnail and num_blocks > 1:
208 num_blocks += 1
209 + image_feature_size = num_blocks * num_patches
210 +
211 elif isinstance(image_data, torch.Tensor):
197 - raise NotImplementedError("Embeddings input is not supported yet")
212 + image_feature_size = image_data.shape[0]
213 else:
214 raise TypeError(f"Invalid image type: {type(image_data)}")
215
@@ -205,7 +220,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
220 prompt_token_ids = llm_inputs["prompt_token_ids"]
221 if prompt is None:
222 prompt = tokenizer.decode(prompt_token_ids)
208 - image_prompt = IMG_START + IMG_CONTEXT * num_blocks * num_patches + IMG_END
223 + image_prompt = IMG_START + IMG_CONTEXT * image_feature_size + IMG_END
224 new_prompt = prompt.replace('<image>', image_prompt, 1)
225 new_prompt_token_ids = tokenizer.encode(new_prompt)
226
@@ -378,23 +393,49 @@ class InternVLChatModel(nn.Module, SupportsVision):
393 return data
394
395 def _parse_and_validate_image_input(
381 - self, **kwargs: object) -> Optional[InternVLImagePixelInputs]:
396 + self, **kwargs: object) -> Optional[InternVLImageInputs]:
397 pixel_values = kwargs.pop("pixel_values", None)
398 image_token_id = kwargs.pop("image_token_id", None)
399 + image_embeds = kwargs.pop("image_embeds", None)
400
385 - if pixel_values is None:
401 + if pixel_values is None and image_embeds is None:
402 return None
403
404 + if image_embeds is not None:
405 + if not isinstance(image_embeds, torch.Tensor):
406 + raise ValueError("Incorrect type of image embeddings. "
407 + f"Got type: {type(image_embeds)}")
408 + return InternVLImageEmbeddingInputs(
409 + type="image_embeds",
410 + data=image_embeds,
411 + )
412 +
413 self.img_context_token_id = image_token_id[0]
414
390 - if not isinstance(pixel_values, (torch.Tensor, list)):
391 - raise ValueError("Incorrect type of pixel values. "
392 - f"Got type: {type(pixel_values)}")
415 + if pixel_values is not None:
416 + if not isinstance(pixel_values, (torch.Tensor, list)):
417 + raise ValueError("Incorrect type of pixel values. "
418 + f"Got type: {type(pixel_values)}")
419 +
420 + return InternVLImagePixelInputs(
421 + type="pixel_values",
422 + data=self._validate_pixel_values(pixel_values),
423 + )
424 +
425 + raise AssertionError("This line should be unreachable.")
426 +
427 + def _process_image_input(
428 + self,
429 + image_input: InternVLImageInputs,
430 + ) -> torch.Tensor:
431 +
432 + if image_input["type"] == "image_embeds":
433 + return image_input["data"]
434 +
435 + assert self.vision_model is not None
436 + image_embeds = self.extract_feature(image_input["data"])
437
394 - return InternVLImagePixelInputs(
395 - type="pixel_values",
396 - data=self._validate_pixel_values(pixel_values),
397 - )
438 + return image_embeds
439
440 def forward(
441 self,
@@ -409,9 +450,9 @@ class InternVLChatModel(nn.Module, SupportsVision):
450 if image_input is not None:
451 inputs_embeds = self.language_model.model.get_input_embeddings(
452 input_ids)
412 - vit_embeds = self.extract_feature(image_input["data"])
453 + vision_embeddings = self._process_image_input(image_input)
454 inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
414 - vit_embeds,
455 + vision_embeddings,
456 self.img_context_token_id)
457 input_ids = None
458 else:
vllm/model_executor/models/llava.py
+43 -18
@@ -27,6 +27,24 @@ from .utils import (filter_weights, init_vllm_registered_model,
27 merge_vision_embeddings)
28
29
30 +class LlavaImagePixelInputs(TypedDict):
31 + type: Literal["pixel_values"]
32 + data: torch.Tensor
33 + """Shape: `(batch_size, num_channels, height, width)`"""
34 +
35 +
36 +class LlavaImageEmbeddingInputs(TypedDict):
37 + type: Literal["image_embeds"]
38 + data: torch.Tensor
39 + """Shape: `(batch_size, image_feature_size, hidden_size)`
40 +
41 + `hidden_size` must match the hidden size of language model backbone.
42 + """
43 +
44 +
45 +LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageEmbeddingInputs]
46 +
47 +
48 # TODO(xwjiang): Run benchmark and decide if TP.
49 class LlavaMultiModalProjector(nn.Module):
50
@@ -49,15 +67,6 @@ class LlavaMultiModalProjector(nn.Module):
67 return hidden_states
68
69
52 -class LlavaImagePixelInputs(TypedDict):
53 - type: Literal["pixel_values"]
54 - data: torch.Tensor
55 - """Shape: `(batch_size, num_channels, height, width)`"""
56 -
57 -
58 -LlavaImageInputs = LlavaImagePixelInputs
59 -
60 -
70 def get_max_llava_image_tokens(ctx: InputContext):
71 hf_config = ctx.get_hf_config(LlavaConfig)
72 vision_config = hf_config.vision_config
@@ -210,18 +219,30 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
219 def _parse_and_validate_image_input(
220 self, **kwargs: object) -> Optional[LlavaImageInputs]:
221 pixel_values = kwargs.pop("pixel_values", None)
222 + image_embeds = kwargs.pop("image_embeds", None)
223
214 - if pixel_values is None:
224 + if pixel_values is None and image_embeds is None:
225 return None
226
217 - if not isinstance(pixel_values, torch.Tensor):
218 - raise ValueError("Incorrect type of pixel values. "
219 - f"Got type: {type(pixel_values)}")
220 -
221 - return LlavaImagePixelInputs(
222 - type="pixel_values",
223 - data=self._validate_pixel_values(pixel_values),
224 - )
227 + if pixel_values is not None:
228 + if not isinstance(pixel_values, torch.Tensor):
229 + raise ValueError("Incorrect type of pixel values. "
230 + f"Got type: {type(pixel_values)}")
231 + return LlavaImagePixelInputs(
232 + type="pixel_values",
233 + data=self._validate_pixel_values(pixel_values),
234 + )
235 +
236 + if image_embeds is not None:
237 + if not isinstance(image_embeds, torch.Tensor):
238 + raise ValueError("Incorrect type of image embeddings. "
239 + f"Got type: {type(image_embeds)}")
240 + return LlavaImageEmbeddingInputs(
241 + type="image_embeds",
242 + data=image_embeds,
243 + )
244 +
245 + raise AssertionError("This line should be unreachable.")
246
247 def _select_image_features(self, image_features: torch.Tensor, *,
248 strategy: str) -> torch.Tensor:
@@ -258,6 +279,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
279
280 def _process_image_input(self,
281 image_input: LlavaImageInputs) -> torch.Tensor:
282 +
283 + if image_input["type"] == "image_embeds":
284 + return image_input["data"]
285 +
286 assert self.vision_tower is not None
287 image_features = self._process_image_pixels(image_input)
288 return self.multi_modal_projector(image_features)
vllm/model_executor/models/llava_next.py
+43 -15
@@ -60,7 +60,17 @@ class LlavaNextImagePixelInputs(TypedDict):
60 """
61
62
63 -LlavaNextImageInputs = LlavaNextImagePixelInputs
63 +class LlavaNextImageEmbeddingInputs(TypedDict):
64 + type: Literal["image_embeds"]
65 + data: torch.Tensor
66 + """Shape: `(batch_size, image_feature_size, hidden_size)`
67 +
68 + `hidden_size` must match the hidden size of language model backbone.
69 + """
70 +
71 +
72 +LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
73 + LlavaNextImageEmbeddingInputs]
74
75
76 # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
@@ -208,7 +218,7 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
218 input_width=width,
219 )
220 elif isinstance(image_data, torch.Tensor):
211 - raise NotImplementedError("Embeddings input is not supported yet")
221 + image_feature_size = image_data.shape[0]
222 else:
223 raise TypeError(f"Invalid image type: {type(image_data)}")
224
@@ -320,26 +330,40 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
330 return data
331
332 def _parse_and_validate_image_input(
323 - self, **kwargs: object) -> Optional[LlavaNextImagePixelInputs]:
333 + self, **kwargs: object) -> Optional[LlavaNextImageInputs]:
334 pixel_values = kwargs.pop("pixel_values", None)
335 image_sizes = kwargs.pop("image_sizes", None)
336 + image_embeds = kwargs.pop("image_embeds", None)
337
327 - if pixel_values is None:
338 + if pixel_values is None and image_embeds is None:
339 return None
340
330 - if not isinstance(pixel_values, (torch.Tensor, list)):
331 - raise ValueError("Incorrect type of pixel values. "
332 - f"Got type: {type(pixel_values)}")
341 + if pixel_values is not None:
342 + if not isinstance(pixel_values, (torch.Tensor, list)):
343 + raise ValueError("Incorrect type of pixel values. "
344 + f"Got type: {type(pixel_values)}")
345
334 - if not isinstance(image_sizes, torch.Tensor):
335 - raise ValueError("Incorrect type of image sizes. "
336 - f"Got type: {type(image_sizes)}")
346 + if not isinstance(image_sizes, torch.Tensor):
347 + raise ValueError("Incorrect type of image sizes. "
348 + f"Got type: {type(image_sizes)}")
349
338 - return LlavaNextImagePixelInputs(
339 - type="pixel_values",
340 - data=self._validate_pixel_values(pixel_values),
341 - image_sizes=self._validate_image_sizes(image_sizes),
342 - )
350 + return LlavaNextImagePixelInputs(
351 + type="pixel_values",
352 + data=self._validate_pixel_values(pixel_values),
353 + image_sizes=self._validate_image_sizes(image_sizes),
354 + )
355 +
356 + if image_embeds is not None:
357 + if not isinstance(image_embeds, torch.Tensor):
358 + raise ValueError("Incorrect type of image embeds. "
359 + f"Got type: {type(image_embeds)}")
360 +
361 + return LlavaNextImageEmbeddingInputs(
362 + type="image_embeds",
363 + data=image_embeds,
364 + )
365 +
366 + raise AssertionError("This line should be unreachable.")
367
368 def _select_image_features(self, image_features: torch.Tensor, *,
369 strategy: str) -> torch.Tensor:
@@ -466,6 +490,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
490 self,
491 image_input: LlavaNextImageInputs,
492 ) -> Union[torch.Tensor, List[torch.Tensor]]:
493 +
494 + if image_input["type"] == "image_embeds":
495 + return [image_input["data"]]
496 +
497 patch_embeddings = self._process_image_pixels(image_input)
498
499 image_sizes = image_input.get("image_sizes")
vllm/model_executor/models/paligemma.py
+48 -32
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
1 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3 import torch
4 from torch import nn
@@ -31,6 +31,25 @@ _KEYS_TO_MODIFY_MAPPING = {
31 }
32
33
34 +class PaliGemmaImagePixelInputs(TypedDict):
35 + type: Literal["pixel_values"]
36 + data: torch.Tensor
37 + """Shape: (batch_size, num_channels, height, width)"""
38 +
39 +
40 +class PaliGemmaImageEmbeddingInputs(TypedDict):
41 + type: Literal["image_embeds"]
42 + data: torch.Tensor
43 + """Shape: `(batch_size, image_feature_size, hidden_size)`
44 +
45 + `hidden_size` must match the hidden size of language model backbone.
46 + """
47 +
48 +
49 +PaliGemmaImageInputs = Union[PaliGemmaImagePixelInputs,
50 + PaliGemmaImageEmbeddingInputs]
51 +
52 +
53 def get_max_paligemma_image_tokens(ctx: InputContext):
54 hf_config = ctx.get_hf_config(PaliGemmaConfig)
55 vision_config = hf_config.vision_config
@@ -107,15 +126,6 @@ class PaliGemmaMultiModalProjector(nn.Module):
126 return hidden_states
127
128
110 -class PaliGemmaImagePixelInputs(TypedDict):
111 - type: Literal["pixel_values"]
112 - data: torch.Tensor
113 - """Shape: (batch_size, num_channels, height, width)"""
114 -
115 -
116 -PaliGemmaImageInputs = PaliGemmaImagePixelInputs
117 -
118 -
129 @MULTIMODAL_REGISTRY.register_image_input_mapper()
130 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_paligemma_image_tokens)
131 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_paligemma)
@@ -163,18 +173,30 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
173 def _parse_and_validate_image_input(
174 self, **kwargs: object) -> Optional[PaliGemmaImageInputs]:
175 pixel_values = kwargs.pop("pixel_values", None)
176 + image_embeds = kwargs.pop("image_embeds", None)
177
167 - if pixel_values is None:
178 + if pixel_values is None and image_embeds is None:
179 return None
180
170 - if not isinstance(pixel_values, torch.Tensor):
171 - raise ValueError("Incorrect type of pixel values. "
172 - f"Got type: {type(pixel_values)}")
173 -
174 - return PaliGemmaImagePixelInputs(
175 - type="pixel_values",
176 - data=self._validate_pixel_values(pixel_values),
177 - )
181 + if pixel_values is not None:
182 + if not isinstance(pixel_values, torch.Tensor):
183 + raise ValueError("Incorrect type of pixel values. "
184 + f"Got type: {type(pixel_values)}")
185 + return PaliGemmaImagePixelInputs(
186 + type="pixel_values",
187 + data=self._validate_pixel_values(pixel_values),
188 + )
189 +
190 + if image_embeds is not None:
191 + if not isinstance(image_embeds, torch.Tensor):
192 + raise ValueError("Incorrect type of image embeddings. "
193 + f"Got type: {type(image_embeds)}")
194 + return PaliGemmaImageEmbeddingInputs(
195 + type="image_embeds",
196 + data=image_embeds,
197 + )
198 +
199 + raise AssertionError("This line should be unreachable.")
200
201 def _image_pixels_to_features(
202 self,
@@ -187,27 +209,21 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
209
210 return image_features
211
190 - def _process_image_pixels(
212 + def _process_image_input(
213 self,
192 - inputs: PaliGemmaImagePixelInputs,
214 + image_input: PaliGemmaImageInputs,
215 ) -> torch.Tensor:
194 - assert self.vision_tower is not None
216
196 - pixel_values = inputs["data"]
217 + if image_input["type"] == "image_embeds":
218 + return image_input["data"]
219
198 - return self._image_pixels_to_features(
220 + assert self.vision_tower is not None
221 + pixel_values = image_input["data"]
222 + image_features = self._image_pixels_to_features(
223 self.vision_tower,
224 pixel_values,
225 )
226
203 - def _process_image_input(
204 - self,
205 - image_input: PaliGemmaImageInputs,
206 - ) -> torch.Tensor:
207 -
208 - assert self.vision_tower is not None
209 - image_features = self._process_image_pixels(image_input, )
210 -
227 return self.multi_modal_projector(image_features)
228
229 def forward(self,
vllm/model_executor/models/phi3v.py
+73 -32
@@ -70,6 +70,36 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
70 projection_dim=768)
71
72
73 +class Phi3VImagePixelInputs(TypedDict):
74 + type: Literal["pixel_values"]
75 + data: Union[torch.Tensor, List[torch.Tensor]]
76 + """
77 + Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
78 +
79 + Note that `num_patches` may be different for each batch, in which case
80 + the data is passed as a list instead of a batched tensor.
81 + """
82 +
83 + image_sizes: torch.Tensor
84 + """
85 + Shape: `(batch_size, 2)`
86 +
87 + This should be in `(height, width)` format.
88 + """
89 +
90 +
91 +class Phi3VImageEmbeddingInputs(TypedDict):
92 + type: Literal["image_embeds"]
93 + data: Union[torch.Tensor, List[torch.Tensor]]
94 + """Shape: `(batch_size, image_feature_size, hidden_size)`
95 +
96 + `hidden_size` must match the hidden size of language model backbone.
97 + """
98 +
99 +
100 +Phi3VImageInputs = Union[Phi3VImagePixelInputs, Phi3VImageEmbeddingInputs]
101 +
102 +
103 class Phi3ImageEmbeddingBase(nn.Module):
104
105 def __init__(self) -> None:
@@ -257,24 +287,6 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
287 return image_features_hd_newline
288
289
260 -class Phi3VImagePixelInputs(TypedDict):
261 - type: Literal["pixel_values"]
262 - data: Union[torch.Tensor, List[torch.Tensor]]
263 - """
264 - Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
265 -
266 - Note that `num_patches` may be different for each batch, in which case
267 - the data is passed as a list instead of a batched tensor.
268 - """
269 -
270 - image_sizes: torch.Tensor
271 - """
272 - Shape: `(batch_size, 2)`
273 -
274 - This should be in `(height, width)` format.
275 - """
276 -
277 -
290 # Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L57
291 def _calc_padded_size(*, width: int, height: int, padding_unit: int = 336):
292 target_height = int(np.ceil(height / padding_unit) * padding_unit)
@@ -390,7 +402,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
402 input_width=w,
403 input_height=h)
404 elif isinstance(image_data, torch.Tensor):
393 - raise NotImplementedError("Embeddings input is not supported yet")
405 + image_feature_size = image_data.shape[0]
406 else:
407 raise TypeError(f"Invalid image type: {type(image_data)}")
408
@@ -494,25 +506,55 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
506 return data
507
508 def _parse_and_validate_image_input(
497 - self, **kwargs: object) -> Optional[Phi3VImagePixelInputs]:
509 + self, **kwargs: object) -> Optional[Phi3VImageInputs]:
510 pixel_values = kwargs.pop("pixel_values", None)
511 image_sizes = kwargs.pop("image_sizes", None)
512 + image_embeds = kwargs.pop("image_embeds", None)
513
514 if pixel_values is None:
515 return None
516
504 - if not isinstance(pixel_values, (torch.Tensor, list)):
505 - raise ValueError("Incorrect type of pixel values. "
506 - f"Got type: {type(pixel_values)}")
517 + if pixel_values is None and image_embeds is None:
518 + return None
519 +
520 + if pixel_values is not None:
521 + if not isinstance(pixel_values, (torch.Tensor, list)):
522 + raise ValueError("Incorrect type of pixel values. "
523 + f"Got type: {type(pixel_values)}")
524 +
525 + if not isinstance(image_sizes, torch.Tensor):
526 + raise ValueError("Incorrect type of image sizes. "
527 + f"Got type: {type(image_sizes)}")
528 +
529 + return Phi3VImagePixelInputs(
530 + type="pixel_values",
531 + data=self._validate_pixel_values(pixel_values),
532 + image_sizes=self._validate_image_sizes(image_sizes))
533 +
534 + if image_embeds is not None:
535 + if not isinstance(image_embeds, torch.Tensor):
536 + raise ValueError("Incorrect type of image embeddings. "
537 + f"Got type: {type(image_embeds)}")
538 + return Phi3VImageEmbeddingInputs(
539 + type="image_embeds",
540 + data=image_embeds,
541 + )
542 +
543 + raise AssertionError("This line should be unreachable.")
544 +
545 + def _process_image_input(
546 + self,
547 + image_input: Phi3VImageInputs,
548 + ) -> torch.Tensor:
549 +
550 + if image_input["type"] == "image_embeds":
551 + return image_input["data"]
552
508 - if not isinstance(image_sizes, torch.Tensor):
509 - raise ValueError("Incorrect type of image sizes. "
510 - f"Got type: {type(image_sizes)}")
553 + assert self.vision_embed_tokens is not None
554 + image_embeds = self.vision_embed_tokens(image_input["data"],
555 + image_input["image_sizes"])
556
512 - return Phi3VImagePixelInputs(
513 - type="pixel_values",
514 - data=self._validate_pixel_values(pixel_values),
515 - image_sizes=self._validate_image_sizes(image_sizes))
557 + return image_embeds
558
559 def forward(self,
560 input_ids: torch.Tensor,
@@ -524,8 +566,7 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
566 image_input = self._parse_and_validate_image_input(**kwargs)
567
568 if image_input is not None:
527 - vision_embeddings = self.vision_embed_tokens(
528 - image_input["data"], image_input["image_sizes"])
569 + vision_embeddings = self._process_image_input(image_input)
570 inputs_embeds = self.model.get_input_embeddings(input_ids)
571 inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
572 vision_embeddings,
vllm/model_executor/models/siglip.py
+7 -1
@@ -97,7 +97,13 @@ def input_processor_for_siglip(
97 tokenizer = cached_get_tokenizer(model_config.tokenizer)
98
99 if image_feature_size_override is None:
100 - image_feature_size = get_siglip_image_feature_size(hf_config)
100 + image_data = multi_modal_data["image"]
101 + if isinstance(image_data, Image.Image):
102 + image_feature_size = get_siglip_image_feature_size(hf_config)
103 + elif isinstance(image_data, torch.Tensor):
104 + image_feature_size = image_data.shape[0]
105 + else:
106 + raise TypeError(f"Invalid image type: {type(image_data)}")
107 else:
108 image_feature_size = image_feature_size_override
109
vllm/multimodal/image.py
+4 -1
@@ -115,6 +115,7 @@ class ImagePlugin(MultiModalPlugin):
115 data: object) -> MultiModalInputs:
116 model_config = ctx.model_config
117
118 + # PIL image
119 if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
120 image_processor = self._get_hf_image_processor(model_config)
121 if image_processor is None:
@@ -129,8 +130,10 @@ class ImagePlugin(MultiModalPlugin):
130 raise
131
132 return MultiModalInputs(batch_data)
133 +
134 + # Image embedding
135 elif isinstance(data, torch.Tensor) or is_list_of(data, torch.Tensor):
133 - raise NotImplementedError("Embeddings input is not supported yet")
136 + return MultiModalInputs({"image_embeds": data})
137
138 raise TypeError(f"Invalid image type: {type(data)}")
139