A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core][VLM] Support image embeddings as input (#6613)
Roger Wang committed
Aug 12, 2024 at 01:16 UTC
e6e42e4b1759b8703bf6cb80aefa9c0ec35044f6
13 files changed
+518
-139
docs/source/models/vlm.rst
+11
@@ -49,6 +49,17 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptI
49
"multi_modal_data": {"image": image},
50
})
51
52
+ for o in outputs:
53
+ generated_text = o.outputs[0].text
54
+ print(generated_text)
55
+
56
+ # Inference with image embeddings as input
57
+ image_embeds = torch.load(...) # torch.Tensor of shape (1, image_feature_size, hidden_size of LM)
58
+ outputs = llm.generate({
59
+ "prompt": prompt,
60
+ "multi_modal_data": {"image": image_embeds},
61
+ })
62
+
63
for o in outputs:
64
generated_text = o.outputs[0].text
65
print(generated_text)
tests/models/test_llava_image_embeds.py
new
+159
@@ -0,0 +1,159 @@
1
+from typing import List, Optional, Tuple, Type
2
+
3
+import pytest
4
+from transformers import AutoConfig, AutoTokenizer
5
+
6
+from vllm.sequence import SampleLogprobs
7
+
8
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9
+from .utils import check_logprobs_close
10
+
11
+pytestmark = pytest.mark.vlm
12
+
13
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
14
+ "stop_sign":
15
+ "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
16
+ "cherry_blossom":
17
+ "USER: <image>\nWhat is the season?\nASSISTANT:",
18
+})
19
+
20
+models = [
21
+ "llava-hf/llava-1.5-7b-hf",
22
+]
23
+
24
+
25
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
26
+ Optional[SampleLogprobs]],
27
+ model: str):
28
+ """Sanitize vllm output to be comparable with hf output."""
29
+ output_ids, output_str, out_logprobs = vllm_output
30
+
31
+ config = AutoConfig.from_pretrained(model)
32
+ image_token_id = config.image_token_index
33
+
34
+ tokenizer = AutoTokenizer.from_pretrained(model)
35
+ eos_token_id = tokenizer.eos_token_id
36
+
37
+ hf_output_ids = [
38
+ token_id for idx, token_id in enumerate(output_ids)
39
+ if token_id != image_token_id or output_ids[idx - 1] != image_token_id
40
+ ]
41
+
42
+ assert output_str[0] == " "
43
+ hf_output_str = output_str[1:]
44
+ if hf_output_ids[-1] == eos_token_id:
45
+ hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
46
+
47
+ return hf_output_ids, hf_output_str, out_logprobs
48
+
49
+
50
+def run_test(
51
+ hf_runner: Type[HfRunner],
52
+ vllm_runner: Type[VllmRunner],
53
+ image_assets: _ImageAssets,
54
+ model: str,
55
+ *,
56
+ size_factors: List[float],
57
+ dtype: str,
58
+ max_tokens: int,
59
+ num_logprobs: int,
60
+ tensor_parallel_size: int,
61
+ distributed_executor_backend: Optional[str] = None,
62
+):
63
+ """Inference result should be the same between hf and vllm.
64
+
65
+ All the image fixtures for the test is under tests/images.
66
+ For huggingface runner, we provide the PIL images as input.
67
+ For vllm runner, we provide MultiModalDataDict objects
68
+ and corresponding vision language config as input.
69
+ Note, the text input is also adjusted to abide by vllm contract.
70
+ The text output is sanitized to be able to compare with hf.
71
+ """
72
+
73
+ # vLLM to load from image embeddings
74
+ vllm_images = [asset.image_embeds for asset in image_assets]
75
+
76
+ # transformers to load from PIL images
77
+ hf_images = [asset.pil_image for asset in image_assets]
78
+
79
+ vllm_inputs_per_image = [(
80
+ [prompt for _ in size_factors],
81
+ [image for _ in size_factors],
82
+ ) for image, prompt in zip(vllm_images, HF_IMAGE_PROMPTS)]
83
+
84
+ hf_inputs_per_image = [(
85
+ [prompt for _ in size_factors],
86
+ [image for _ in size_factors],
87
+ ) for image, prompt in zip(hf_images, HF_IMAGE_PROMPTS)]
88
+
89
+ # NOTE: take care of the order. run vLLM first, and then run HF.
90
+ # vLLM needs a fresh new process without cuda initialization.
91
+ # if we run HF first, the cuda initialization will be done and it
92
+ # will hurt multiprocessing backend with fork method (the default method).
93
+
94
+ # max_model_len should be greater than image_feature_size
95
+ with vllm_runner(model,
96
+ dtype=dtype,
97
+ tensor_parallel_size=tensor_parallel_size,
98
+ distributed_executor_backend=distributed_executor_backend,
99
+ enforce_eager=True) as vllm_model:
100
+ vllm_outputs_per_image = [
101
+ vllm_model.generate_greedy_logprobs(prompts,
102
+ max_tokens,
103
+ num_logprobs=num_logprobs,
104
+ images=images)
105
+ for prompts, images in vllm_inputs_per_image
106
+ ]
107
+
108
+ with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
109
+ hf_outputs_per_image = [
110
+ hf_model.generate_greedy_logprobs_limit(prompts,
111
+ max_tokens,
112
+ num_logprobs=num_logprobs,
113
+ images=images)
114
+ for prompts, images in hf_inputs_per_image
115
+ ]
116
+
117
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
118
+ vllm_outputs_per_image):
119
+ # TODO: Check whether using original CLIPVisionModel can improve
120
+ # consistency against HF
121
+ check_logprobs_close(
122
+ outputs_0_lst=hf_outputs,
123
+ outputs_1_lst=[
124
+ vllm_to_hf_output(vllm_output, model)
125
+ for vllm_output in vllm_outputs
126
+ ],
127
+ name_0="hf",
128
+ name_1="vllm",
129
+ )
130
+
131
+
132
+@pytest.mark.parametrize("model", models)
133
+@pytest.mark.parametrize(
134
+ "size_factors",
135
+ [
136
+ # No image
137
+ [],
138
+ # Single-scale
139
+ [1.0],
140
+ # Single-scale, batched
141
+ [1.0, 1.0, 1.0],
142
+ ],
143
+)
144
+@pytest.mark.parametrize("dtype", ["half"])
145
+@pytest.mark.parametrize("max_tokens", [128])
146
+@pytest.mark.parametrize("num_logprobs", [5])
147
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
148
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
149
+ run_test(
150
+ hf_runner,
151
+ vllm_runner,
152
+ image_assets,
153
+ model,
154
+ size_factors=size_factors,
155
+ dtype=dtype,
156
+ max_tokens=max_tokens,
157
+ num_logprobs=num_logprobs,
158
+ tensor_parallel_size=1,
159
+ )
vllm/assets/image.py
+10
@@ -1,6 +1,7 @@
1
from dataclasses import dataclass
2
from typing import Literal
3
4
+import torch
5
from PIL import Image
6
7
from vllm.assets.base import get_vllm_public_assets
@@ -18,3 +19,12 @@ class ImageAsset:
19
image_path = get_vllm_public_assets(filename=f"{self.name}.jpg",
20
s3_prefix=VLM_IMAGES_DIR)
21
return Image.open(image_path)
22
+
23
+ @property
24
+ def image_embeds(self) -> torch.Tensor:
25
+ """
26
+ Image embeddings, only used for testing purposes with llava 1.5.
27
+ """
28
+ image_path = get_vllm_public_assets(filename=f"{self.name}.pt",
29
+ s3_prefix=VLM_IMAGES_DIR)
30
+ return torch.load(image_path)
vllm/model_executor/models/blip2.py
+49
-23
@@ -1,4 +1,4 @@
1
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
1
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3
import torch
4
import torch.nn as nn
@@ -28,6 +28,29 @@ _KEYS_TO_MODIFY_MAPPING = {
28
"language_model.model": "language_model",
29
}
30
31
+# We use this internally as placeholders since there is no image token
32
+# defined on the HuggingFace repo
33
+BLIP2_IMAGE_TOKEN = "<image>"
34
+BLIP2_IMAGE_TOKEN_ID = 50265
35
+
36
+
37
+class Blip2ImagePixelInputs(TypedDict):
38
+ type: Literal["pixel_values"]
39
+ data: torch.Tensor
40
+ """Shape: (batch_size, num_channels, height, width)"""
41
+
42
+
43
+class Blip2ImageEmbeddingInputs(TypedDict):
44
+ type: Literal["image_embeds"]
45
+ data: torch.Tensor
46
+ """Shape: `(batch_size, image_feature_size, hidden_size)`
47
+
48
+ `hidden_size` must match the hidden size of language model backbone.
49
+ """
50
+
51
+
52
+Blip2ImageInputs = Union[Blip2ImagePixelInputs, Blip2ImageEmbeddingInputs]
53
+
54
55
class Blip2QFormerMultiHeadAttention(nn.Module):
56
@@ -375,20 +398,6 @@ class Blip2QFormerModel(nn.Module):
398
return sequence_output
399
400
378
-class Blip2ImagePixelInputs(TypedDict):
379
- type: Literal["pixel_values"]
380
- data: torch.Tensor
381
- """Shape: (batch_size, num_channels, height, width)"""
382
-
383
-
384
-Blip2ImageInputs = Blip2ImagePixelInputs
385
-
386
-# We use this internally as placeholders since there is no image token
387
-# defined on the HuggingFace repo
388
-BLIP2_IMAGE_TOKEN = "<image>"
389
-BLIP2_IMAGE_TOKEN_ID = 50265
390
-
391
-
401
def get_blip2_image_feature_size(hf_config: Blip2Config) -> int:
402
return hf_config.num_query_tokens
403
@@ -506,18 +515,31 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
515
def _parse_and_validate_image_input(
516
self, **kwargs: object) -> Optional[Blip2ImageInputs]:
517
pixel_values = kwargs.pop("pixel_values", None)
518
+ image_embeds = kwargs.pop("image_embeds", None)
519
510
- if pixel_values is None:
520
+ if pixel_values is None and image_embeds is None:
521
return None
522
513
- if not isinstance(pixel_values, torch.Tensor):
514
- raise ValueError("Incorrect type of pixel values. "
515
- f"Got type: {type(pixel_values)}")
523
+ if pixel_values is not None:
524
+ if not isinstance(pixel_values, torch.Tensor):
525
+ raise ValueError("Incorrect type of pixel values. "
526
+ f"Got type: {type(pixel_values)}")
527
517
- return Blip2ImagePixelInputs(
518
- type="pixel_values",
519
- data=self._validate_pixel_values(pixel_values),
520
- )
528
+ return Blip2ImagePixelInputs(
529
+ type="pixel_values",
530
+ data=self._validate_pixel_values(pixel_values),
531
+ )
532
+
533
+ if image_embeds is not None:
534
+ if not isinstance(image_embeds, torch.Tensor):
535
+ raise ValueError("Incorrect type of image embeddings. "
536
+ f"Got type: {type(image_embeds)}")
537
+ return Blip2ImageEmbeddingInputs(
538
+ type="image_embeds",
539
+ data=image_embeds,
540
+ )
541
+
542
+ raise AssertionError("This line should be unreachable.")
543
544
def _image_pixels_to_features(self, vision_model: BlipVisionModel,
545
pixel_values: torch.Tensor) -> torch.Tensor:
@@ -538,6 +560,10 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
560
561
def _process_image_input(self,
562
image_input: Blip2ImageInputs) -> torch.Tensor:
563
+
564
+ if image_input["type"] == "image_embeds":
565
+ return image_input["data"]
566
+
567
assert self.vision_model is not None
568
image_features = self._process_image_pixels(image_input)
569
vllm/model_executor/models/clip.py
+7
-1
@@ -88,7 +88,13 @@ def input_processor_for_clip(
88
tokenizer = cached_get_tokenizer(model_config.tokenizer)
89
90
if image_feature_size_override is None:
91
- image_feature_size = get_clip_image_feature_size(hf_config)
91
+ image_data = multi_modal_data["image"]
92
+ if isinstance(image_data, Image.Image):
93
+ image_feature_size = get_clip_image_feature_size(hf_config)
94
+ elif isinstance(image_data, torch.Tensor):
95
+ image_feature_size = image_data.shape[0]
96
+ else:
97
+ raise TypeError(f"Invalid image type: {type(image_data)}")
98
else:
99
image_feature_size = image_feature_size_override
100
vllm/model_executor/models/fuyu.py
+10
-3
@@ -234,7 +234,8 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
234
cache_config=cache_config,
235
quant_config=quant_config)
236
237
- def _parse_and_validate_image_input(self, **kwargs: object):
237
+ def _parse_and_validate_image_input(
238
+ self, **kwargs: object) -> Optional[FuyuImagePixelInputs]:
239
image_patches = kwargs.pop("image_patches", None)
240
241
if isinstance(image_patches, torch.Tensor):
@@ -249,6 +250,13 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
250
data=image_patches)
251
return None
252
253
+ def _process_image_input(
254
+ self, image_input: FuyuImagePixelInputs) -> torch.Tensor:
255
+
256
+ assert self.vision_embed_tokens is not None
257
+ vision_embeddings, _ = self.vision_embed_tokens(image_input["data"])
258
+ return vision_embeddings
259
+
260
def forward(
261
self,
262
input_ids: torch.Tensor,
@@ -261,8 +269,7 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
269
image_input = self._parse_and_validate_image_input(**kwargs)
270
271
if image_input is not None:
264
- vision_embeddings, _ = self.vision_embed_tokens(
265
- image_input["data"])
272
+ vision_embeddings = self._process_image_input(image_input)
273
inputs_embeds = self.language_model.model.embed_tokens(input_ids)
274
inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
275
vision_embeddings,
vllm/model_executor/models/internvl.py
+54
-13
@@ -50,6 +50,19 @@ class InternVLImagePixelInputs(TypedDict):
50
"""
51
52
53
+class InternVLImageEmbeddingInputs(TypedDict):
54
+ type: Literal["image_embeds"]
55
+ data: Union[torch.Tensor, List[torch.Tensor]]
56
+ """Shape: `(batch_size, image_feature_size, hidden_size)`
57
+
58
+ `hidden_size` must match the hidden size of language model backbone.
59
+ """
60
+
61
+
62
+InternVLImageInputs = Union[InternVLImagePixelInputs,
63
+ InternVLImageEmbeddingInputs]
64
+
65
+
66
# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
67
def build_transform(input_size):
68
MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
@@ -193,8 +206,10 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
206
# add thumbnail image if num_blocks > 1
207
if hf_config.use_thumbnail and num_blocks > 1:
208
num_blocks += 1
209
+ image_feature_size = num_blocks * num_patches
210
+
211
elif isinstance(image_data, torch.Tensor):
197
- raise NotImplementedError("Embeddings input is not supported yet")
212
+ image_feature_size = image_data.shape[0]
213
else:
214
raise TypeError(f"Invalid image type: {type(image_data)}")
215
@@ -205,7 +220,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
220
prompt_token_ids = llm_inputs["prompt_token_ids"]
221
if prompt is None:
222
prompt = tokenizer.decode(prompt_token_ids)
208
- image_prompt = IMG_START + IMG_CONTEXT * num_blocks * num_patches + IMG_END
223
+ image_prompt = IMG_START + IMG_CONTEXT * image_feature_size + IMG_END
224
new_prompt = prompt.replace('<image>', image_prompt, 1)
225
new_prompt_token_ids = tokenizer.encode(new_prompt)
226
@@ -378,23 +393,49 @@ class InternVLChatModel(nn.Module, SupportsVision):
393
return data
394
395
def _parse_and_validate_image_input(
381
- self, **kwargs: object) -> Optional[InternVLImagePixelInputs]:
396
+ self, **kwargs: object) -> Optional[InternVLImageInputs]:
397
pixel_values = kwargs.pop("pixel_values", None)
398
image_token_id = kwargs.pop("image_token_id", None)
399
+ image_embeds = kwargs.pop("image_embeds", None)
400
385
- if pixel_values is None:
401
+ if pixel_values is None and image_embeds is None:
402
return None
403
404
+ if image_embeds is not None:
405
+ if not isinstance(image_embeds, torch.Tensor):
406
+ raise ValueError("Incorrect type of image embeddings. "
407
+ f"Got type: {type(image_embeds)}")
408
+ return InternVLImageEmbeddingInputs(
409
+ type="image_embeds",
410
+ data=image_embeds,
411
+ )
412
+
413
self.img_context_token_id = image_token_id[0]
414
390
- if not isinstance(pixel_values, (torch.Tensor, list)):
391
- raise ValueError("Incorrect type of pixel values. "
392
- f"Got type: {type(pixel_values)}")
415
+ if pixel_values is not None:
416
+ if not isinstance(pixel_values, (torch.Tensor, list)):
417
+ raise ValueError("Incorrect type of pixel values. "
418
+ f"Got type: {type(pixel_values)}")
419
+
420
+ return InternVLImagePixelInputs(
421
+ type="pixel_values",
422
+ data=self._validate_pixel_values(pixel_values),
423
+ )
424
+
425
+ raise AssertionError("This line should be unreachable.")
426
+
427
+ def _process_image_input(
428
+ self,
429
+ image_input: InternVLImageInputs,
430
+ ) -> torch.Tensor:
431
+
432
+ if image_input["type"] == "image_embeds":
433
+ return image_input["data"]
434
+
435
+ assert self.vision_model is not None
436
+ image_embeds = self.extract_feature(image_input["data"])
437
394
- return InternVLImagePixelInputs(
395
- type="pixel_values",
396
- data=self._validate_pixel_values(pixel_values),
397
- )
438
+ return image_embeds
439
440
def forward(
441
self,
@@ -409,9 +450,9 @@ class InternVLChatModel(nn.Module, SupportsVision):
450
if image_input is not None:
451
inputs_embeds = self.language_model.model.get_input_embeddings(
452
input_ids)
412
- vit_embeds = self.extract_feature(image_input["data"])
453
+ vision_embeddings = self._process_image_input(image_input)
454
inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
414
- vit_embeds,
455
+ vision_embeddings,
456
self.img_context_token_id)
457
input_ids = None
458
else:
vllm/model_executor/models/llava.py
+43
-18
@@ -27,6 +27,24 @@ from .utils import (filter_weights, init_vllm_registered_model,
27
merge_vision_embeddings)
28
29
30
+class LlavaImagePixelInputs(TypedDict):
31
+ type: Literal["pixel_values"]
32
+ data: torch.Tensor
33
+ """Shape: `(batch_size, num_channels, height, width)`"""
34
+
35
+
36
+class LlavaImageEmbeddingInputs(TypedDict):
37
+ type: Literal["image_embeds"]
38
+ data: torch.Tensor
39
+ """Shape: `(batch_size, image_feature_size, hidden_size)`
40
+
41
+ `hidden_size` must match the hidden size of language model backbone.
42
+ """
43
+
44
+
45
+LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageEmbeddingInputs]
46
+
47
+
48
# TODO(xwjiang): Run benchmark and decide if TP.
49
class LlavaMultiModalProjector(nn.Module):
50
@@ -49,15 +67,6 @@ class LlavaMultiModalProjector(nn.Module):
67
return hidden_states
68
69
52
-class LlavaImagePixelInputs(TypedDict):
53
- type: Literal["pixel_values"]
54
- data: torch.Tensor
55
- """Shape: `(batch_size, num_channels, height, width)`"""
56
-
57
-
58
-LlavaImageInputs = LlavaImagePixelInputs
59
-
60
-
70
def get_max_llava_image_tokens(ctx: InputContext):
71
hf_config = ctx.get_hf_config(LlavaConfig)
72
vision_config = hf_config.vision_config
@@ -210,18 +219,30 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
219
def _parse_and_validate_image_input(
220
self, **kwargs: object) -> Optional[LlavaImageInputs]:
221
pixel_values = kwargs.pop("pixel_values", None)
222
+ image_embeds = kwargs.pop("image_embeds", None)
223
214
- if pixel_values is None:
224
+ if pixel_values is None and image_embeds is None:
225
return None
226
217
- if not isinstance(pixel_values, torch.Tensor):
218
- raise ValueError("Incorrect type of pixel values. "
219
- f"Got type: {type(pixel_values)}")
220
-
221
- return LlavaImagePixelInputs(
222
- type="pixel_values",
223
- data=self._validate_pixel_values(pixel_values),
224
- )
227
+ if pixel_values is not None:
228
+ if not isinstance(pixel_values, torch.Tensor):
229
+ raise ValueError("Incorrect type of pixel values. "
230
+ f"Got type: {type(pixel_values)}")
231
+ return LlavaImagePixelInputs(
232
+ type="pixel_values",
233
+ data=self._validate_pixel_values(pixel_values),
234
+ )
235
+
236
+ if image_embeds is not None:
237
+ if not isinstance(image_embeds, torch.Tensor):
238
+ raise ValueError("Incorrect type of image embeddings. "
239
+ f"Got type: {type(image_embeds)}")
240
+ return LlavaImageEmbeddingInputs(
241
+ type="image_embeds",
242
+ data=image_embeds,
243
+ )
244
+
245
+ raise AssertionError("This line should be unreachable.")
246
247
def _select_image_features(self, image_features: torch.Tensor, *,
248
strategy: str) -> torch.Tensor:
@@ -258,6 +279,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
279
280
def _process_image_input(self,
281
image_input: LlavaImageInputs) -> torch.Tensor:
282
+
283
+ if image_input["type"] == "image_embeds":
284
+ return image_input["data"]
285
+
286
assert self.vision_tower is not None
287
image_features = self._process_image_pixels(image_input)
288
return self.multi_modal_projector(image_features)
vllm/model_executor/models/llava_next.py
+43
-15
@@ -60,7 +60,17 @@ class LlavaNextImagePixelInputs(TypedDict):
60
"""
61
62
63
-LlavaNextImageInputs = LlavaNextImagePixelInputs
63
+class LlavaNextImageEmbeddingInputs(TypedDict):
64
+ type: Literal["image_embeds"]
65
+ data: torch.Tensor
66
+ """Shape: `(batch_size, image_feature_size, hidden_size)`
67
+
68
+ `hidden_size` must match the hidden size of language model backbone.
69
+ """
70
+
71
+
72
+LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
73
+ LlavaNextImageEmbeddingInputs]
74
75
76
# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
@@ -208,7 +218,7 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
218
input_width=width,
219
)
220
elif isinstance(image_data, torch.Tensor):
211
- raise NotImplementedError("Embeddings input is not supported yet")
221
+ image_feature_size = image_data.shape[0]
222
else:
223
raise TypeError(f"Invalid image type: {type(image_data)}")
224
@@ -320,26 +330,40 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
330
return data
331
332
def _parse_and_validate_image_input(
323
- self, **kwargs: object) -> Optional[LlavaNextImagePixelInputs]:
333
+ self, **kwargs: object) -> Optional[LlavaNextImageInputs]:
334
pixel_values = kwargs.pop("pixel_values", None)
335
image_sizes = kwargs.pop("image_sizes", None)
336
+ image_embeds = kwargs.pop("image_embeds", None)
337
327
- if pixel_values is None:
338
+ if pixel_values is None and image_embeds is None:
339
return None
340
330
- if not isinstance(pixel_values, (torch.Tensor, list)):
331
- raise ValueError("Incorrect type of pixel values. "
332
- f"Got type: {type(pixel_values)}")
341
+ if pixel_values is not None:
342
+ if not isinstance(pixel_values, (torch.Tensor, list)):
343
+ raise ValueError("Incorrect type of pixel values. "
344
+ f"Got type: {type(pixel_values)}")
345
334
- if not isinstance(image_sizes, torch.Tensor):
335
- raise ValueError("Incorrect type of image sizes. "
336
- f"Got type: {type(image_sizes)}")
346
+ if not isinstance(image_sizes, torch.Tensor):
347
+ raise ValueError("Incorrect type of image sizes. "
348
+ f"Got type: {type(image_sizes)}")
349
338
- return LlavaNextImagePixelInputs(
339
- type="pixel_values",
340
- data=self._validate_pixel_values(pixel_values),
341
- image_sizes=self._validate_image_sizes(image_sizes),
342
- )
350
+ return LlavaNextImagePixelInputs(
351
+ type="pixel_values",
352
+ data=self._validate_pixel_values(pixel_values),
353
+ image_sizes=self._validate_image_sizes(image_sizes),
354
+ )
355
+
356
+ if image_embeds is not None:
357
+ if not isinstance(image_embeds, torch.Tensor):
358
+ raise ValueError("Incorrect type of image embeds. "
359
+ f"Got type: {type(image_embeds)}")
360
+
361
+ return LlavaNextImageEmbeddingInputs(
362
+ type="image_embeds",
363
+ data=image_embeds,
364
+ )
365
+
366
+ raise AssertionError("This line should be unreachable.")
367
368
def _select_image_features(self, image_features: torch.Tensor, *,
369
strategy: str) -> torch.Tensor:
@@ -466,6 +490,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
490
self,
491
image_input: LlavaNextImageInputs,
492
) -> Union[torch.Tensor, List[torch.Tensor]]:
493
+
494
+ if image_input["type"] == "image_embeds":
495
+ return [image_input["data"]]
496
+
497
patch_embeddings = self._process_image_pixels(image_input)
498
499
image_sizes = image_input.get("image_sizes")
vllm/model_executor/models/paligemma.py
+48
-32
@@ -1,4 +1,4 @@
1
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
1
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3
import torch
4
from torch import nn
@@ -31,6 +31,25 @@ _KEYS_TO_MODIFY_MAPPING = {
31
}
32
33
34
+class PaliGemmaImagePixelInputs(TypedDict):
35
+ type: Literal["pixel_values"]
36
+ data: torch.Tensor
37
+ """Shape: (batch_size, num_channels, height, width)"""
38
+
39
+
40
+class PaliGemmaImageEmbeddingInputs(TypedDict):
41
+ type: Literal["image_embeds"]
42
+ data: torch.Tensor
43
+ """Shape: `(batch_size, image_feature_size, hidden_size)`
44
+
45
+ `hidden_size` must match the hidden size of language model backbone.
46
+ """
47
+
48
+
49
+PaliGemmaImageInputs = Union[PaliGemmaImagePixelInputs,
50
+ PaliGemmaImageEmbeddingInputs]
51
+
52
+
53
def get_max_paligemma_image_tokens(ctx: InputContext):
54
hf_config = ctx.get_hf_config(PaliGemmaConfig)
55
vision_config = hf_config.vision_config
@@ -107,15 +126,6 @@ class PaliGemmaMultiModalProjector(nn.Module):
126
return hidden_states
127
128
110
-class PaliGemmaImagePixelInputs(TypedDict):
111
- type: Literal["pixel_values"]
112
- data: torch.Tensor
113
- """Shape: (batch_size, num_channels, height, width)"""
114
-
115
-
116
-PaliGemmaImageInputs = PaliGemmaImagePixelInputs
117
-
118
-
129
@MULTIMODAL_REGISTRY.register_image_input_mapper()
130
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_paligemma_image_tokens)
131
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_paligemma)
@@ -163,18 +173,30 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
173
def _parse_and_validate_image_input(
174
self, **kwargs: object) -> Optional[PaliGemmaImageInputs]:
175
pixel_values = kwargs.pop("pixel_values", None)
176
+ image_embeds = kwargs.pop("image_embeds", None)
177
167
- if pixel_values is None:
178
+ if pixel_values is None and image_embeds is None:
179
return None
180
170
- if not isinstance(pixel_values, torch.Tensor):
171
- raise ValueError("Incorrect type of pixel values. "
172
- f"Got type: {type(pixel_values)}")
173
-
174
- return PaliGemmaImagePixelInputs(
175
- type="pixel_values",
176
- data=self._validate_pixel_values(pixel_values),
177
- )
181
+ if pixel_values is not None:
182
+ if not isinstance(pixel_values, torch.Tensor):
183
+ raise ValueError("Incorrect type of pixel values. "
184
+ f"Got type: {type(pixel_values)}")
185
+ return PaliGemmaImagePixelInputs(
186
+ type="pixel_values",
187
+ data=self._validate_pixel_values(pixel_values),
188
+ )
189
+
190
+ if image_embeds is not None:
191
+ if not isinstance(image_embeds, torch.Tensor):
192
+ raise ValueError("Incorrect type of image embeddings. "
193
+ f"Got type: {type(image_embeds)}")
194
+ return PaliGemmaImageEmbeddingInputs(
195
+ type="image_embeds",
196
+ data=image_embeds,
197
+ )
198
+
199
+ raise AssertionError("This line should be unreachable.")
200
201
def _image_pixels_to_features(
202
self,
@@ -187,27 +209,21 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
209
210
return image_features
211
190
- def _process_image_pixels(
212
+ def _process_image_input(
213
self,
192
- inputs: PaliGemmaImagePixelInputs,
214
+ image_input: PaliGemmaImageInputs,
215
) -> torch.Tensor:
194
- assert self.vision_tower is not None
216
196
- pixel_values = inputs["data"]
217
+ if image_input["type"] == "image_embeds":
218
+ return image_input["data"]
219
198
- return self._image_pixels_to_features(
220
+ assert self.vision_tower is not None
221
+ pixel_values = image_input["data"]
222
+ image_features = self._image_pixels_to_features(
223
self.vision_tower,
224
pixel_values,
225
)
226
203
- def _process_image_input(
204
- self,
205
- image_input: PaliGemmaImageInputs,
206
- ) -> torch.Tensor:
207
-
208
- assert self.vision_tower is not None
209
- image_features = self._process_image_pixels(image_input, )
210
-
227
return self.multi_modal_projector(image_features)
228
229
def forward(self,
vllm/model_executor/models/phi3v.py
+73
-32
@@ -70,6 +70,36 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
70
projection_dim=768)
71
72
73
+class Phi3VImagePixelInputs(TypedDict):
74
+ type: Literal["pixel_values"]
75
+ data: Union[torch.Tensor, List[torch.Tensor]]
76
+ """
77
+ Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
78
+
79
+ Note that `num_patches` may be different for each batch, in which case
80
+ the data is passed as a list instead of a batched tensor.
81
+ """
82
+
83
+ image_sizes: torch.Tensor
84
+ """
85
+ Shape: `(batch_size, 2)`
86
+
87
+ This should be in `(height, width)` format.
88
+ """
89
+
90
+
91
+class Phi3VImageEmbeddingInputs(TypedDict):
92
+ type: Literal["image_embeds"]
93
+ data: Union[torch.Tensor, List[torch.Tensor]]
94
+ """Shape: `(batch_size, image_feature_size, hidden_size)`
95
+
96
+ `hidden_size` must match the hidden size of language model backbone.
97
+ """
98
+
99
+
100
+Phi3VImageInputs = Union[Phi3VImagePixelInputs, Phi3VImageEmbeddingInputs]
101
+
102
+
103
class Phi3ImageEmbeddingBase(nn.Module):
104
105
def __init__(self) -> None:
@@ -257,24 +287,6 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
287
return image_features_hd_newline
288
289
260
-class Phi3VImagePixelInputs(TypedDict):
261
- type: Literal["pixel_values"]
262
- data: Union[torch.Tensor, List[torch.Tensor]]
263
- """
264
- Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
265
-
266
- Note that `num_patches` may be different for each batch, in which case
267
- the data is passed as a list instead of a batched tensor.
268
- """
269
-
270
- image_sizes: torch.Tensor
271
- """
272
- Shape: `(batch_size, 2)`
273
-
274
- This should be in `(height, width)` format.
275
- """
276
-
277
-
290
# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L57
291
def _calc_padded_size(*, width: int, height: int, padding_unit: int = 336):
292
target_height = int(np.ceil(height / padding_unit) * padding_unit)
@@ -390,7 +402,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
402
input_width=w,
403
input_height=h)
404
elif isinstance(image_data, torch.Tensor):
393
- raise NotImplementedError("Embeddings input is not supported yet")
405
+ image_feature_size = image_data.shape[0]
406
else:
407
raise TypeError(f"Invalid image type: {type(image_data)}")
408
@@ -494,25 +506,55 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
506
return data
507
508
def _parse_and_validate_image_input(
497
- self, **kwargs: object) -> Optional[Phi3VImagePixelInputs]:
509
+ self, **kwargs: object) -> Optional[Phi3VImageInputs]:
510
pixel_values = kwargs.pop("pixel_values", None)
511
image_sizes = kwargs.pop("image_sizes", None)
512
+ image_embeds = kwargs.pop("image_embeds", None)
513
514
if pixel_values is None:
515
return None
516
504
- if not isinstance(pixel_values, (torch.Tensor, list)):
505
- raise ValueError("Incorrect type of pixel values. "
506
- f"Got type: {type(pixel_values)}")
517
+ if pixel_values is None and image_embeds is None:
518
+ return None
519
+
520
+ if pixel_values is not None:
521
+ if not isinstance(pixel_values, (torch.Tensor, list)):
522
+ raise ValueError("Incorrect type of pixel values. "
523
+ f"Got type: {type(pixel_values)}")
524
+
525
+ if not isinstance(image_sizes, torch.Tensor):
526
+ raise ValueError("Incorrect type of image sizes. "
527
+ f"Got type: {type(image_sizes)}")
528
+
529
+ return Phi3VImagePixelInputs(
530
+ type="pixel_values",
531
+ data=self._validate_pixel_values(pixel_values),
532
+ image_sizes=self._validate_image_sizes(image_sizes))
533
+
534
+ if image_embeds is not None:
535
+ if not isinstance(image_embeds, torch.Tensor):
536
+ raise ValueError("Incorrect type of image embeddings. "
537
+ f"Got type: {type(image_embeds)}")
538
+ return Phi3VImageEmbeddingInputs(
539
+ type="image_embeds",
540
+ data=image_embeds,
541
+ )
542
+
543
+ raise AssertionError("This line should be unreachable.")
544
+
545
+ def _process_image_input(
546
+ self,
547
+ image_input: Phi3VImageInputs,
548
+ ) -> torch.Tensor:
549
+
550
+ if image_input["type"] == "image_embeds":
551
+ return image_input["data"]
552
508
- if not isinstance(image_sizes, torch.Tensor):
509
- raise ValueError("Incorrect type of image sizes. "
510
- f"Got type: {type(image_sizes)}")
553
+ assert self.vision_embed_tokens is not None
554
+ image_embeds = self.vision_embed_tokens(image_input["data"],
555
+ image_input["image_sizes"])
556
512
- return Phi3VImagePixelInputs(
513
- type="pixel_values",
514
- data=self._validate_pixel_values(pixel_values),
515
- image_sizes=self._validate_image_sizes(image_sizes))
557
+ return image_embeds
558
559
def forward(self,
560
input_ids: torch.Tensor,
@@ -524,8 +566,7 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
566
image_input = self._parse_and_validate_image_input(**kwargs)
567
568
if image_input is not None:
527
- vision_embeddings = self.vision_embed_tokens(
528
- image_input["data"], image_input["image_sizes"])
569
+ vision_embeddings = self._process_image_input(image_input)
570
inputs_embeds = self.model.get_input_embeddings(input_ids)
571
inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
572
vision_embeddings,
vllm/model_executor/models/siglip.py
+7
-1
@@ -97,7 +97,13 @@ def input_processor_for_siglip(
97
tokenizer = cached_get_tokenizer(model_config.tokenizer)
98
99
if image_feature_size_override is None:
100
- image_feature_size = get_siglip_image_feature_size(hf_config)
100
+ image_data = multi_modal_data["image"]
101
+ if isinstance(image_data, Image.Image):
102
+ image_feature_size = get_siglip_image_feature_size(hf_config)
103
+ elif isinstance(image_data, torch.Tensor):
104
+ image_feature_size = image_data.shape[0]
105
+ else:
106
+ raise TypeError(f"Invalid image type: {type(image_data)}")
107
else:
108
image_feature_size = image_feature_size_override
109
vllm/multimodal/image.py
+4
-1
@@ -115,6 +115,7 @@ class ImagePlugin(MultiModalPlugin):
115
data: object) -> MultiModalInputs:
116
model_config = ctx.model_config
117
118
+ # PIL image
119
if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
120
image_processor = self._get_hf_image_processor(model_config)
121
if image_processor is None:
@@ -129,8 +130,10 @@ class ImagePlugin(MultiModalPlugin):
130
raise
131
132
return MultiModalInputs(batch_data)
133
+
134
+ # Image embedding
135
elif isinstance(data, torch.Tensor) or is_list_of(data, torch.Tensor):
133
- raise NotImplementedError("Embeddings input is not supported yet")
136
+ return MultiModalInputs({"image_embeds": data})
137
138
raise TypeError(f"Invalid image type: {type(data)}")
139