A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Support nested structures for TensorSchema (#26212)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 4, 2025 at 16:20 UTC
44ea85137a43b0cb6711441331613fc164ab1441
5 files changed
+274
-292
tests/utils_/test_tensor_schema.py
+39
-29
@@ -6,37 +6,39 @@ import torch
6
7
from vllm.model_executor.models.glm4_1v import Glm4vImageEmbeddingInputs
8
from vllm.model_executor.models.granite_speech import GraniteSpeechAudioInputs
9
+from vllm.model_executor.models.hyperclovax_vision import (
10
+ HCXVisionVideoPixelInputs)
11
from vllm.model_executor.models.phi3v import Phi3VImagePixelInputs
12
13
14
def test_tensor_schema_valid_tensor():
15
Phi3VImagePixelInputs(
14
- data=torch.randn(16, 64, 3, 32, 32),
16
+ pixel_values=torch.randn(16, 64, 3, 32, 32),
17
image_sizes=torch.randint(0, 256, (16, 2)),
18
)
19
20
21
def test_tensor_schema_optional_fields():
22
Phi3VImagePixelInputs(
21
- data=torch.randn(16, 64, 3, 32, 32),
23
+ pixel_values=torch.randn(16, 64, 3, 32, 32),
24
image_sizes=None,
25
)
26
25
- Phi3VImagePixelInputs(data=torch.randn(16, 64, 3, 32, 32), )
27
+ Phi3VImagePixelInputs(pixel_values=torch.randn(16, 64, 3, 32, 32))
28
29
30
def test_tensor_schema_constant_dim_failure():
31
with pytest.raises(ValueError, match="dim\\[2\\] expected 3, got 4"):
32
Phi3VImagePixelInputs(
31
- data=torch.randn(16, 64, 4, 32, 32), # dim[2] = 4
33
+ pixel_values=torch.randn(16, 64, 4, 32, 32), # dim[2] = 4
34
image_sizes=torch.randint(0, 256, (16, 2)),
35
)
36
37
38
def test_tensor_schema_invalid_types_in_list():
37
- with pytest.raises(ValueError, match="is not a torch.Tensor"):
39
+ with pytest.raises(TypeError, match="is not one of the expected types"):
40
Phi3VImagePixelInputs(
39
- data=[
41
+ pixel_values=[
42
torch.randn(64, 3, 32, 32),
43
"not_a_tensor",
44
torch.randn(64, 3, 32, 32),
@@ -48,27 +50,28 @@ def test_tensor_schema_invalid_types_in_list():
50
def test_tensor_schema_rank_mismatch():
51
with pytest.raises(ValueError, match="has rank 3 but expected 5"):
52
Phi3VImagePixelInputs(
51
- data=torch.randn(16, 64, 3),
53
+ pixel_values=torch.randn(16, 64, 3),
54
image_sizes=torch.randint(0, 256, (16, 2)),
55
)
56
57
58
def test_tensor_schema_missing_required_field():
57
- with pytest.raises(ValueError, match="Required field 'data' is missing"):
59
+ with pytest.raises(ValueError,
60
+ match="Required field 'pixel_values' is missing"):
61
Phi3VImagePixelInputs(image_sizes=torch.randint(0, 256, (16, 2)), )
62
63
64
def test_tensor_schema_symbolic_dim_mismatch():
65
with pytest.raises(ValueError, match="expected 'bn'=12, got 16"):
66
Phi3VImagePixelInputs(
64
- data=torch.randn(12, 64, 3, 32, 32),
67
+ pixel_values=torch.randn(12, 64, 3, 32, 32),
68
image_sizes=torch.randint(0, 256, (16, 2)),
69
)
70
71
72
def test_tensor_schema_list_tensor_valid():
73
Phi3VImagePixelInputs(
71
- data=[torch.randn(64, 3, 32, 32) for _ in range(16)],
74
+ pixel_values=[torch.randn(64, 3, 32, 32) for _ in range(16)],
75
image_sizes=torch.randint(0, 256, (16, 2)),
76
)
77
@@ -76,39 +79,46 @@ def test_tensor_schema_list_tensor_valid():
79
def test_tensor_schema_variable_patch_counts_valid():
80
# Each image has a different number of patches (p)
81
# Each tensor has shape (p, 3, 32, 32)
79
- data = [
80
- torch.randn(16, 3, 32, 32), # p = 16
81
- torch.randn(32, 3, 32, 32), # p = 32
82
- torch.randn(64, 3, 32, 32), # p = 64
83
- ]
84
- image_sizes = torch.randint(0, 256, (3, 2)) # bn = 3
82
Phi3VImagePixelInputs(
86
- data=data,
87
- image_sizes=image_sizes,
83
+ pixel_values=[
84
+ torch.randn(16, 3, 32, 32), # p = 16
85
+ torch.randn(32, 3, 32, 32), # p = 32
86
+ torch.randn(64, 3, 32, 32), # p = 64
87
+ ],
88
+ image_sizes=torch.randint(0, 256, (3, 2)), # bn = 3
89
)
90
91
92
def test_tensor_schema_tuple_tensor_valid():
93
Phi3VImagePixelInputs(
93
- data=tuple(torch.randn(64, 3, 32, 32) for _ in range(16)),
94
+ pixel_values=tuple(torch.randn(64, 3, 32, 32) for _ in range(16)),
95
image_sizes=torch.randint(0, 256, (16, 2)),
96
)
97
98
99
+def test_tensor_schema_double_nested_tensors():
100
+ x = torch.rand(4, 3, 32, 32)
101
+ y = torch.rand(2, 3, 32, 32)
102
+
103
+ HCXVisionVideoPixelInputs(pixel_values_videos=([x, y, x], [y], [x, y]))
104
+
105
+
106
def test_tensor_schema_inconsistent_shapes_in_list():
107
with pytest.raises(ValueError, match="contains inconsistent shapes"):
108
Phi3VImagePixelInputs(
101
- data=[torch.randn(64, 3, 32, 32),
102
- torch.randn(64, 3, 16, 16)] +
103
- [torch.randn(64, 3, 32, 32) for _ in range(14)],
109
+ pixel_values=[
110
+ torch.randn(64, 3, 32, 32),
111
+ torch.randn(64, 3, 16, 16),
112
+ *(torch.randn(64, 3, 32, 32) for _ in range(14)),
113
+ ],
114
image_sizes=torch.randint(0, 256, (16, 2)),
115
)
116
117
118
def test_tensor_schema_empty_list():
109
- with pytest.raises(ValueError, match="is an empty list"):
119
+ with pytest.raises(ValueError, match="is an empty sequence"):
120
Phi3VImagePixelInputs(
111
- data=[],
121
+ pixel_values=[],
122
image_sizes=torch.randint(0, 256, (0, 2)),
123
)
124
@@ -117,18 +127,18 @@ def test_tensor_schema_validation_disabled_skips_shape_check():
127
# This should NOT raise, because validation is turned off
128
# This would normally fail (dim[2] should be 3, not 4)
129
Phi3VImagePixelInputs(
120
- data=torch.randn(16, 64, 4, 32, 32),
130
+ pixel_values=torch.randn(16, 64, 4, 32, 32),
131
image_sizes=torch.randint(0, 256, (16, 2)),
132
validate=False,
133
)
134
135
136
def test_tensor_schema_with_valid_resolve_binding_dims():
127
- data = torch.randn(16, 64, 3, 336, 336) # h=336, w=336
137
+ pixel_values = torch.randn(16, 64, 3, 336, 336) # h=336, w=336
138
image_sizes = torch.randint(0, 256, (16, 2))
139
140
Phi3VImagePixelInputs(
131
- data=data,
141
+ pixel_values=pixel_values,
142
image_sizes=image_sizes,
143
resolve_bindings={
144
"h": 336,
@@ -138,13 +148,13 @@ def test_tensor_schema_with_valid_resolve_binding_dims():
148
149
150
def test_tensor_schema_with_invalid_resolve_binding_dims():
141
- data = torch.randn(16, 64, 3, 36, 36) # h=36, w=36
151
+ pixel_values = torch.randn(16, 64, 3, 36, 36) # h=36, w=36
152
image_sizes = torch.randint(0, 256, (16, 2))
153
154
# Should raise because 'h' and 'w' don't match resolve bindings
155
with pytest.raises(ValueError, match="dim\\[3\\] expected 336, got 36"):
156
Phi3VImagePixelInputs(
147
- data=data,
157
+ pixel_values=pixel_values,
158
image_sizes=image_sizes,
159
resolve_bindings={
160
"h": 336,
vllm/model_executor/models/glm4_1v.py
+2
-2
@@ -29,7 +29,7 @@
29
import math
30
from collections.abc import Iterable, Mapping, Sequence
31
from functools import partial
32
-from typing import Annotated, Any, Callable, Literal, Optional, Union, override
32
+from typing import Annotated, Any, Callable, Literal, Optional, Union
33
34
import numpy as np
35
import torch
@@ -1170,7 +1170,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1170
"video.height override (%d) exceeds model's "
1171
"maximum height (%d), will be ignored",
1172
overrides.height, height)
1173
- height = min(height, override.height)
1173
+ height = min(height, overrides.height)
1174
1175
video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)
1176
video_items = []
vllm/model_executor/models/hyperclovax_vision.py
+177
-210
@@ -2,27 +2,16 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
# copied from : https://github.com/huggingface/transformers
4
import ast
5
-import sys
5
from collections import defaultdict
6
from collections.abc import Iterable, Mapping, Sequence
7
from functools import partial
9
-from itertools import chain
10
-from typing import Any, Literal, Optional, TypedDict, Union
8
+from itertools import accumulate
9
+from typing import Annotated, Any, Literal, Optional, Union
10
11
import numpy as np
13
-import PIL
14
-from einops import rearrange
15
-from PIL import Image
16
-
17
-if sys.version_info >= (3, 11):
18
- import typing
19
- Unpack = typing.Unpack
20
-else:
21
- import typing_extensions
22
- Unpack = typing_extensions.Unpack
23
-
12
import torch
13
import torch.nn as nn
14
+from einops import rearrange
15
from timm.layers import LayerNorm, LayerNorm2d
16
from timm.models.regnet import RegStage
17
from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig
@@ -42,11 +31,13 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
31
PromptReplacement, PromptUpdate)
32
from vllm.multimodal.profiling import BaseDummyInputsBuilder
33
from vllm.sequence import IntermediateTensors
34
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
35
36
from .clip import CLIPVisionModel
37
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
38
from .siglip import SiglipVisionModel
49
-from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
39
+from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
40
+ maybe_prefix)
41
from .vision import get_vision_encoder_info
42
43
EOT = "<|endofturn|>"
@@ -69,28 +60,42 @@ def get_num_combined_frames(
60
return num_canvases + (leftover_frames > 0)
61
62
72
-class HCXVisionMultimodalPixelInputs(TypedDict):
73
- type: Literal["pixel_values"]
74
- pixel_values_images: list[torch.Tensor]
63
+class HCXVisionImagePixelInputs(TensorSchema):
64
"""
76
- Shape: `[(num_grids, num_channels, height, width), ...]` if anyres
77
-
78
- Note that `height` or `width` may be different per batch and image,
79
- in which case the data is passed as a list instead of a batched tensor.
65
+ Dimensions:
66
+ - n: Number of images
67
+ - g: Number of grids
68
+ - c: Number of channels (3)
69
+ - h: Height
70
+ - w: Width
71
"""
81
- image_sizes_images: list[tuple[Union[int, float]]]
82
- """
83
- Shape: `[(height, width), ...]`
84
- """
85
- vision_query_lengths_images: list[Union[int, float]]
86
- pixel_values_videos: list[tuple[Union[int, float]]]
72
+ type: Literal["pixel_values"] = "pixel_values"
73
+ pixel_values_images: Annotated[
74
+ list[torch.Tensor],
75
+ TensorShape("n", "g", 3, "h", "w", dynamic_dims={"g"})]
76
+ image_sizes_images: Annotated[torch.Tensor, TensorShape("n", 2)]
77
+
78
+
79
+HCXVisionImageInputs = HCXVisionImagePixelInputs
80
+
81
+
82
+class HCXVisionVideoPixelInputs(TensorSchema):
83
"""
88
- Shape: `[(num_grids, num_channels, height, width), ...]` if anyres
84
+ Dimensions:
85
+ - n: Number of videos
86
+ - f: Number of frames
87
+ - g: Number of grids
88
+ - c: Number of channels (3)
89
+ - h: Height
90
+ - w: Width
91
"""
90
- vision_query_lengths_videos: list[Union[int, float]]
92
+ type: Literal["pixel_values_videos"] = "pixel_values_videos"
93
+ pixel_values_videos: Annotated[
94
+ list[list[torch.Tensor]],
95
+ TensorShape("n", "f", "g", 3, "h", "w", dynamic_dims={"f", "g"})]
96
97
93
-HCXVisionMultimodalInputs = Union[HCXVisionMultimodalPixelInputs]
98
+HCXVisionVideoInputs = HCXVisionVideoPixelInputs
99
100
101
class HCXVisionProcessingInfo(BaseProcessingInfo):
@@ -191,27 +196,9 @@ class HCXVisionMultiModalProcessor(
196
mm_kwargs: Mapping[str, object],
197
tok_kwargs: Mapping[str, object],
198
) -> BatchFeature:
194
-
195
- def replace_multimodal_token(
196
- token_ids: torch.Tensor,
197
- target_token: int,
198
- repeats: list[int],
199
- ):
200
- output = list[int]()
201
- _repeats_idx = 0
202
- for token_id in token_ids:
203
- if token_id == target_token:
204
- output += [token_id.item()] * repeats[_repeats_idx]
205
- _repeats_idx += 1
206
- else:
207
- output += [token_id.item()]
208
-
209
- return torch.tensor(output, device=token_ids.device)
210
-
199
for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
212
- if video_arr.dtype == np.uint8:
213
- continue
214
- mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
200
+ if video_arr.dtype != np.uint8:
201
+ mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
202
203
processed_outputs = self.info.ctx.call_hf_processor(
204
hf_processor=self.info.get_hf_processor(**mm_kwargs),
@@ -223,20 +210,16 @@ class HCXVisionMultiModalProcessor(
210
) # text-only
211
212
if len(mm_data) > 0:
226
- # batchify input as a single item
227
- images = mm_data.get("images", None)
228
- batched_images = None if images is None else [images]
229
-
230
- # list of video in single conversation
231
- videos = mm_data.get("videos", None)
232
- batched_videos = None if videos is None else [videos]
213
+ images = mm_data.get("images")
214
+ videos = mm_data.get("videos")
215
216
+ # batchify input as a single item
217
_processed_outputs = self.info.ctx.call_hf_processor(
218
hf_processor=self.info.get_hf_processor(**mm_kwargs),
219
data=dict(
220
text=None,
238
- images=batched_images,
239
- videos=batched_videos,
221
+ images=None if images is None else [images],
222
+ videos=None if videos is None else [videos],
223
),
224
) # mm-only
225
@@ -246,51 +229,43 @@ class HCXVisionMultiModalProcessor(
229
_processed_outputs[k] = v[0]
230
231
if images:
249
- tokenizer = self.info.get_tokenizer()
250
- image_token_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
251
- processed_outputs["input_ids"] = torch.stack([
252
- replace_multimodal_token(
253
- token_ids=_input_ids,
254
- target_token=image_token_id,
255
- repeats=_processed_outputs[
256
- "vision_query_lengths_images"],
257
- ) for _input_ids in processed_outputs["input_ids"]
258
- ],
259
- dim=0)
232
+ _processed_outputs["image_sizes_images"] = torch.tensor(
233
+ _processed_outputs["image_sizes_images"])
234
+ _processed_outputs[
235
+ "vision_query_lengths_images"] = torch.tensor(
236
+ _processed_outputs["vision_query_lengths_images"])
237
238
if videos:
262
- _num_per_videos = [
263
- get_num_combined_frames(len(video)) for video in videos
239
+ _idx_per_video = [
240
+ 0, *accumulate(
241
+ get_num_combined_frames(len(video))
242
+ for video in videos)
243
]
244
_processed_outputs["pixel_values_videos"] = [
245
_processed_outputs["pixel_values_videos"]
267
- [sum(_num_per_videos[:_i]):sum(_num_per_videos[:_i + 1])]
268
- for _i in range(len(videos))
246
+ [_idx_per_video[i]:_idx_per_video[i + 1]]
247
+ for i in range(len(videos))
248
]
249
_processed_outputs["vision_query_lengths_videos"] = [
271
- _processed_outputs["vision_query_lengths_videos"]
272
- [sum(_num_per_videos[:_i]):sum(_num_per_videos[:_i + 1])]
273
- for _i in range(len(videos))
250
+ torch.tensor(
251
+ _processed_outputs["vision_query_lengths_videos"]
252
+ [_idx_per_video[i]:_idx_per_video[i + 1]])
253
+ for i in range(len(videos))
254
]
255
276
- tokenizer = self.info.get_tokenizer()
277
- video_token_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN)
278
- processed_outputs["input_ids"] = torch.stack([
279
- replace_multimodal_token(
280
- token_ids=_input_ids,
281
- target_token=video_token_id,
282
- repeats=[
283
- sum(lens) for lens in
284
- _processed_outputs["vision_query_lengths_videos"]
285
- ],
286
- ) for _input_ids in processed_outputs["input_ids"]
287
- ],
288
- dim=0)
289
-
256
processed_outputs.update(_processed_outputs)
257
258
return processed_outputs
259
260
+ def _hf_processor_applies_updates(
261
+ self,
262
+ prompt_text: str,
263
+ mm_items: MultiModalDataItems,
264
+ hf_processor_mm_kwargs: Mapping[str, object],
265
+ tokenization_kwargs: Mapping[str, object],
266
+ ) -> bool:
267
+ return False
268
+
269
def _get_prompt_updates(
270
self,
271
mm_items: MultiModalDataItems,
@@ -311,11 +286,11 @@ class HCXVisionMultiModalProcessor(
286
out_item = out_mm_kwargs[modality][item_idx]
287
288
if modality == "image":
314
- lens = out_item["vision_query_lengths_images"].data
289
+ lens = out_item["vision_query_lengths_images"].data.tolist()
290
num_tokens = self.info.get_num_image_tokens(
291
vision_query_length=lens)
292
elif modality == "video":
318
- lens = out_item["vision_query_lengths_videos"].data
293
+ lens = out_item["vision_query_lengths_videos"].data.tolist()
294
num_tokens = self.info.get_num_video_tokens(
295
vision_query_length=lens)
296
else:
@@ -343,26 +318,11 @@ class HCXVisionMultiModalProcessor(
318
hf_processor_mm_kwargs: Mapping[str, object],
319
) -> Mapping[str, MultiModalFieldConfig]:
320
return dict(
346
- # image
321
pixel_values_images=MultiModalFieldConfig.batched("image"),
322
image_sizes_images=MultiModalFieldConfig.batched("image"),
323
vision_query_lengths_images=MultiModalFieldConfig.batched("image"),
350
- num_queries_vis_abstractors_images=MultiModalFieldConfig.batched(
351
- "image"),
352
- num_queries_vis_abstractors_slow_images=MultiModalFieldConfig.
353
- batched("image"),
354
- first_last_frames_slows_images=MultiModalFieldConfig.batched(
355
- "image"),
356
- # video
324
pixel_values_videos=MultiModalFieldConfig.batched("video"),
358
- image_sizes_videos=MultiModalFieldConfig.batched("video"),
325
vision_query_lengths_videos=MultiModalFieldConfig.batched("video"),
360
- num_queries_vis_abstractors_videos=MultiModalFieldConfig.batched(
361
- "video"),
362
- num_queries_vis_abstractors_slow_videos=MultiModalFieldConfig.
363
- batched("video"),
364
- first_last_frames_slows_videos=MultiModalFieldConfig.batched(
365
- "video"),
326
)
327
328
@@ -617,6 +577,7 @@ class HCXVisionCAbstractor(nn.Module):
577
info=_build_hcxvision_hf_info,
578
dummy_inputs=HCXVisionDummyInputsBuilder)
579
class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
580
+ merge_by_field_config = True
581
582
packed_modules_mapping = {
583
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -692,55 +653,94 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
653
654
raise ValueError("Only image or video modality is supported")
655
656
+ def _parse_and_validate_image_input(
657
+ self,
658
+ **kwargs: object,
659
+ ) -> Optional[HCXVisionImageInputs]:
660
+ pixel_values_images = kwargs.pop("pixel_values_images", None)
661
+
662
+ if pixel_values_images is None:
663
+ return None
664
+
665
+ image_sizes_images = kwargs.pop("image_sizes_images")
666
+
667
+ return HCXVisionImagePixelInputs(
668
+ pixel_values_images=pixel_values_images,
669
+ image_sizes_images=image_sizes_images,
670
+ )
671
+
672
+ def _parse_and_validate_video_input(
673
+ self,
674
+ **kwargs: object,
675
+ ) -> Optional[HCXVisionVideoInputs]:
676
+ pixel_values_videos = kwargs.pop("pixel_values_videos", None)
677
+
678
+ if pixel_values_videos is None:
679
+ return None
680
+
681
+ return HCXVisionVideoPixelInputs(
682
+ pixel_values_videos=pixel_values_videos, )
683
+
684
+ def _process_image_input(
685
+ self,
686
+ image_input: HCXVisionImageInputs,
687
+ ) -> tuple[torch.Tensor, ...]:
688
+ return self.forward_images(
689
+ pixel_values_images=image_input["pixel_values_images"],
690
+ image_sizes_images=image_input["image_sizes_images"],
691
+ )
692
+
693
+ def _process_video_input(
694
+ self,
695
+ video_input: HCXVisionVideoInputs,
696
+ ) -> tuple[torch.Tensor, ...]:
697
+ return self.forward_videos(
698
+ pixel_values_videos=video_input["pixel_values_videos"], )
699
+
700
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
701
+ modalities = {}
702
+
703
+ # Preserve the order of modalities if there are multiple of them
704
+ # from the order of kwargs.
705
+ for input_key in kwargs:
706
+ if (input_key == "pixel_values_images"
707
+ and "images" not in modalities):
708
+ modalities["images"] = self._parse_and_validate_image_input(
709
+ **kwargs)
710
+ if (input_key == "pixel_values_videos"
711
+ and "videos" not in modalities):
712
+ modalities["videos"] = self._parse_and_validate_video_input(
713
+ **kwargs)
714
+
715
+ return modalities
716
+
717
def get_language_model(self) -> torch.nn.Module:
718
return self.language_model
719
720
def get_multimodal_embeddings(
721
self,
700
- **kwargs: Unpack[HCXVisionMultimodalInputs],
722
+ **kwargs: object,
723
) -> MultiModalEmbeddings:
724
+ modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
725
+ if not modalities:
726
+ return []
727
+
728
+ # The result multimodal_embeddings is tuple of tensors, with each
729
+ # tensor correspoending to a multimodal data item (image or video).
730
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
731
+
732
+ # NOTE: It is important to iterate over the keys in this dictionary
733
+ # to preserve the order of the modalities.
734
+ for modality in modalities:
735
+ if modality == "images":
736
+ image_input = modalities["images"]
737
+ vision_embeddings = self._process_image_input(image_input)
738
+ multimodal_embeddings += vision_embeddings
739
+ if modality == "videos":
740
+ video_input = modalities["videos"]
741
+ video_embeddings = self._process_video_input(video_input)
742
+ multimodal_embeddings += video_embeddings
743
703
- multimodal_embeddings = list()
704
- if kwargs.get("pixel_values_images") is not None:
705
- for _pixel_values_images, _image_sizes_images in zip(
706
- kwargs["pixel_values_images"],
707
- kwargs["image_sizes_images"]):
708
- _pixel_values_images = _pixel_values_images.unsqueeze(dim=0)
709
- _image_sizes_images = _image_sizes_images.unsqueeze(dim=0)
710
- _len_pixel_values_images = [
711
- len(pixel_value) for pixel_value in _pixel_values_images
712
- ]
713
- if isinstance(_image_sizes_images, torch.Tensor):
714
- _image_sizes_images = _image_sizes_images.detach().cpu(
715
- ).tolist()
716
- _multimodal_embeddings_images = self.forward_images(
717
- pixel_values_images=_pixel_values_images,
718
- image_sizes_images=_image_sizes_images,
719
- len_pixel_values_images=_len_pixel_values_images,
720
- )
721
- _multimodal_embeddings_images = torch.cat(
722
- _multimodal_embeddings_images, dim=0)
723
- multimodal_embeddings.append(_multimodal_embeddings_images)
724
-
725
- if kwargs.get("pixel_values_videos") is not None:
726
- for _pixel_values_videos, _vision_query_lengths_videos in zip(
727
- kwargs["pixel_values_videos"],
728
- kwargs["vision_query_lengths_videos"]):
729
- _len_pixel_values_videos = [
730
- len(_vision_query_lengths)
731
- for _vision_query_lengths in _vision_query_lengths_videos
732
- ]
733
- _c, _w, _h = _pixel_values_videos.shape[-3:]
734
- _pixel_values_videos = _pixel_values_videos.reshape(
735
- sum(_len_pixel_values_videos), -1, _c, _w,
736
- _h).unsqueeze(dim=0)
737
- _multimodal_embeddings_videos = self.forward_videos(
738
- pixel_values_videos=_pixel_values_videos,
739
- len_pixel_values_videos=_len_pixel_values_videos,
740
- )
741
- _multimodal_embeddings_videos = torch.cat(
742
- _multimodal_embeddings_videos, dim=0)
743
- multimodal_embeddings.append(_multimodal_embeddings_videos)
744
return multimodal_embeddings
745
746
def forward(
@@ -762,28 +762,20 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
762
763
def forward_images(
764
self,
765
- pixel_values_images: list[list[torch.FloatTensor]],
766
- image_sizes_images: list[list[tuple[int, int]]],
767
- len_pixel_values_images: list[int],
768
- ) -> list[list[torch.Tensor]]:
769
- if sum(len_pixel_values_images) == 0:
770
- return None
771
-
772
- concat_pixel_values_images = torch.cat(list(
773
- chain(*pixel_values_images)),
774
- dim=0)
765
+ pixel_values_images: list[torch.Tensor],
766
+ image_sizes_images: torch.Tensor,
767
+ ) -> tuple[torch.Tensor, ...]:
768
+ pixel_values_image_flat = flatten_bn(pixel_values_images, concat=True)
769
770
visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
771
image_forward_outs = self.vision_model(
778
- concat_pixel_values_images)[:, visual_token_idx:]
772
+ pixel_values_image_flat)[:, visual_token_idx:]
773
774
image_forward_outs = image_forward_outs.to(
775
dtype=self.mm_projector.dtype)
776
image_forward_outs = self.mm_projector(image_forward_outs) # b (h w) d
777
784
- split_sizes = [
785
- pixel_value.shape[0] for pixel_value in chain(*pixel_values_images)
786
- ]
778
+ split_sizes = [len(item) for item in pixel_values_images]
779
image_forward_outs = torch.split(image_forward_outs,
780
split_sizes,
781
dim=0)
@@ -791,10 +783,7 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
783
# newline for anyres postprocessing
784
image_features = anyres_postprocessing(
785
image_forward_outs=image_forward_outs,
794
- image_sizes=[
795
- image_size for image_sizes in image_sizes_images
796
- for image_size in image_sizes
797
- ],
786
+ image_sizes=image_sizes_images.tolist(),
787
num_queries_vis_abstractor=self.config.
788
num_queries_vis_abstractor_image,
789
unpad=self.config.unpad,
@@ -803,26 +792,21 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
792
image_newline=self.image_newline,
793
possible_resolutions=self.config.possible_resolutions,
794
)
806
- return image_features
795
+
796
+ return tuple(image_features)
797
798
def forward_videos(
799
self,
810
- pixel_values_videos: list[list[torch.FloatTensor]],
811
- len_pixel_values_videos: list[int],
812
- ) -> list[torch.Tensor]:
813
-
814
- len_video_grids = sum(len_pixel_values_videos)
815
- if len_video_grids == 0:
816
- return None
817
-
818
- # Run Vision Model
819
- concat_pixel_values_videos = torch.cat(list(
820
- chain(*pixel_values_videos)),
821
- dim=0)
800
+ pixel_values_videos: list[list[torch.Tensor]],
801
+ ) -> tuple[torch.Tensor, ...]:
802
+ pixel_values_videos_flat = flatten_bn(
803
+ [frame for frames in pixel_values_videos for frame in frames],
804
+ concat=True,
805
+ )
806
807
visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
808
video_forward_outs = self.vision_model(
825
- concat_pixel_values_videos)[:, visual_token_idx:]
809
+ pixel_values_videos_flat)[:, visual_token_idx:]
810
811
video_forward_outs = video_forward_outs.to(
812
dtype=self.mm_projector.dtype)
@@ -905,7 +889,11 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
889
) == 0, f"target_features is not empty!! {target_features}"
890
assert len(video_groups) == len(video_features)
891
908
- return video_features
892
+ feats_per_video = [len(video) for video in pixel_values_videos]
893
+ idxs_per_video = [0, *accumulate(feats_per_video)]
894
+ return tuple(
895
+ torch.cat(video_features[idxs_per_video[i]:idxs_per_video[i + 1]])
896
+ for i in range(len(feats_per_video)))
897
898
def _prepare_multimodal_kwargs(self, **kwargs: object):
899
output = defaultdict(list)
@@ -1111,15 +1099,15 @@ def reshape_and_unpad_image_features(
1099
1100
1101
def anyres_postprocessing(
1114
- image_forward_outs: list[torch.FloatTensor],
1102
+ image_forward_outs: list[torch.Tensor],
1103
image_sizes: list[list[int]],
1104
possible_resolutions: list[tuple[int, int]],
1105
patch_size: int,
1106
grid_size: int,
1119
- image_newline: torch.FloatTensor,
1107
+ image_newline: torch.Tensor,
1108
num_queries_vis_abstractor: int = -1,
1109
unpad: bool = False,
1122
-) -> list[torch.FloatTensor]:
1110
+) -> list[torch.Tensor]:
1111
height = width = grid_size // patch_size
1112
1113
if num_queries_vis_abstractor > 0:
@@ -1147,26 +1135,5 @@ def anyres_postprocessing(
1135
(image_feature, image_newline[None].to(image_feature.device)),
1136
dim=0)
1137
new_image_features.append(image_feature)
1150
- image_features = new_image_features
1151
- return image_features
1152
-
1153
-
1154
-def resize_image(
1155
- image: Union[np.ndarray, PIL.Image.Image],
1156
- max_side: int = 378,
1157
-) -> np.ndarray:
1158
- image_arr = image
1159
- if isinstance(image, np.ndarray):
1160
- image = Image.fromarray(image)
1161
-
1162
- width, height = image.size
1163
- cur_max_size = max(width, height)
1164
- if cur_max_size <= max_side:
1165
- return image_arr
1166
-
1167
- scale = max_side / cur_max_size
1168
- width = int(width * scale)
1169
- height = int(height * scale)
1170
- image = image.resize((width, height), Image.LANCZOS)
1171
- image_arr = np.array(image)
1172
- return image_arr
1138
+
1139
+ return new_image_features
vllm/model_executor/models/phi3v.py
+3
-3
@@ -109,7 +109,7 @@ class Phi3VImagePixelInputs(TensorSchema):
109
type: Literal["pixel_values", "image_embeds"] = "pixel_values"
110
111
# Supports either a stacked tensor or a list of (p, 3, h, w) tensors
112
- data: Annotated[
112
+ pixel_values: Annotated[
113
Union[torch.Tensor, list[torch.Tensor]],
114
TensorShape("bn", "p", 3, "h", "w", dynamic_dims={"p"}
115
), # 'p' may vary across items
@@ -594,7 +594,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
594
if pixel_values is not None:
595
return Phi3VImagePixelInputs(
596
type="pixel_values",
597
- data=flatten_bn(pixel_values),
597
+ pixel_values=flatten_bn(pixel_values),
598
image_sizes=flatten_bn(image_sizes, concat=True),
599
resolve_bindings={
600
"h": CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size,
@@ -628,7 +628,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
628
)
629
630
assert self.vision_embed_tokens is not None
631
- image_embeds = self.vision_embed_tokens(image_input["data"],
631
+ image_embeds = self.vision_embed_tokens(image_input["pixel_values"],
632
image_input["image_sizes"])
633
634
return image_embeds
vllm/utils/tensor_schema.py
+53
-48
@@ -94,34 +94,63 @@ class TensorSchema:
94
return False
95
return True
96
97
- def _validate_nested_tensors(
98
- self,
99
- value: Union[list[torch.Tensor], tuple[torch.Tensor, ...]],
100
- field_name: str,
101
- expected_shape: tuple[Union[int, str], ...],
102
- dynamic_dims: set[str],
97
+ def _fmt_indexer(self, idxs: tuple[int, ...]) -> str:
98
+ if not idxs:
99
+ return ""
100
+
101
+ return str(list(idxs))
102
+
103
+ def _validate_field(
104
+ self,
105
+ value: object,
106
+ field_name: str,
107
+ expected_shape: tuple[Union[int, str], ...],
108
+ dynamic_dims: set[str],
109
+ leading_idxs: tuple[int, ...] = (),
110
) -> tuple[int, ...]:
104
- """Validate a list/tuple of tensors and return the actual shape."""
111
+ """Validate a field and return the actual shape."""
112
+ if isinstance(value, (int, float)):
113
+ return () # Scalar
114
+ if isinstance(value, torch.Tensor):
115
+ return value.shape
116
+
117
+ if not isinstance(value, (list, tuple)):
118
+ raise TypeError(
119
+ f"{field_name}{self._fmt_indexer(leading_idxs)} is not "
120
+ f"one of the expected types: int, float, Tensor, list, tuple. "
121
+ f"Got: {type(value)}")
122
+
123
+ if len(value) == 0:
124
+ raise ValueError(f"{field_name}{self._fmt_indexer(leading_idxs)} "
125
+ f"is an empty sequence")
126
+
127
# Ensure all tensors in the list have the same
128
# shape, besides dynamic dimensions
107
- first = value[0]
129
for i, v in enumerate(value):
109
- if not isinstance(v, torch.Tensor):
110
- raise ValueError(f"{field_name}[{i}] is not a "
111
- f"torch.Tensor")
112
- if not self._match_shape_with_dynamic(
113
- v.shape,
114
- first.shape,
130
+ shape = self._validate_field(
131
+ v,
132
+ field_name,
133
+ expected_shape[1:],
134
+ dynamic_dims,
135
+ leading_idxs=leading_idxs + (i, ),
136
+ )
137
+
138
+ if i == 0:
139
+ first_shape = shape
140
+ elif not self._match_shape_with_dynamic(
141
+ shape,
142
+ first_shape,
143
expected_shape,
144
dynamic_dims,
145
):
118
- raise ValueError(f"{field_name} contains inconsistent "
119
- f"shapes: {first.shape} vs {v.shape} "
120
- f"at index {i}")
146
+ raise ValueError(
147
+ f"{field_name}{self._fmt_indexer(leading_idxs)} "
148
+ f"contains inconsistent shapes: {first_shape} "
149
+ f"(index 0) vs {shape} (index {i})")
150
151
# Treat the list as a stacked tensor:
152
# shape = (len(list), *tensor.shape)
124
- return (len(value), ) + first.shape
153
+ return (len(value), ) + first_shape
154
155
def _validate_tensor_shape_expected(
156
self,
@@ -187,36 +216,12 @@ class TensorSchema:
216
for arg in args:
217
if isinstance(arg, TensorShape):
218
expected_shape = arg.resolve(**self._resolve_bindings)
190
- if isinstance(value, (list, tuple)):
191
- # list/tuple of Tensors → shape = (len(value), ...)
192
- if value and isinstance(value[0], torch.Tensor):
193
- actual_shape = self._validate_nested_tensors(
194
- value, field_name, expected_shape,
195
- arg.dynamic_dims)
196
- elif value:
197
- # list/tuple of scalars → shape = (len(value),)
198
- actual_shape = (len(value), )
199
- else:
200
- raise ValueError(
201
- f"{field_name} is an empty list")
202
-
203
- # Tensor → shape = tensor.shape
204
- elif isinstance(value, torch.Tensor):
205
- actual_shape = value.shape
206
-
207
- # Otherwise, it's an unsupported type
208
- else:
209
- type_names = []
210
- for arg in args:
211
- if hasattr(arg, "__name__"):
212
- type_names.append(str(arg.__name__))
213
- else:
214
- type_names.append(str(arg))
215
-
216
- expected_types = ", ".join(type_names)
217
- raise ValueError(
218
- f"{field_name} is not one of the expected "
219
- f"types: {expected_types}")
219
+ actual_shape = self._validate_field(
220
+ value,
221
+ field_name,
222
+ expected_shape,
223
+ arg.dynamic_dims,
224
+ )
225
226
self._validate_tensor_shape_expected(
227
actual_shape, expected_shape, field_name,