A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Initial support of multimodal models for V1 re-arch (#10699)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Dec 8, 2024 at 04:50 UTC
a11f3265282c712d1d9fa75368e2a8c40019fbb7
11 files changed
+284
-69
vllm/engine/arg_utils.py
+8
-8
@@ -1050,9 +1050,12 @@ class EngineArgs:
1050
# long context (> 32K) models. This is to avoid OOM errors in the
1051
# initial memory profiling phase.
1052
1053
- # Chunked prefill is currently disabled for multimodal models by
1054
- # default.
1055
- if use_long_context and not model_config.is_multimodal_model:
1053
+ # For multimodal models, chunked prefill is disabled by default in
1054
+ # V0, but enabled by design in V1
1055
+ if model_config.is_multimodal_model:
1056
+ self.enable_chunked_prefill = bool(envs.VLLM_USE_V1)
1057
+
1058
+ elif use_long_context:
1059
is_gpu = device_config.device_type == "cuda"
1060
use_sliding_window = (model_config.get_sliding_window()
1061
is not None)
@@ -1241,12 +1244,9 @@ class EngineArgs:
1244
Override the EngineConfig's configs based on the usage context for V1.
1245
"""
1246
assert envs.VLLM_USE_V1, "V1 is not enabled"
1244
- # TODO (ywang96): Enable APC by default when VLM supports it.
1247
if engine_config.model_config.is_multimodal_model:
1246
- logger.warning(
1247
- "Prefix caching is currently not supported for multimodal "
1248
- "models and has been disabled.")
1249
- engine_config.cache_config.enable_prefix_caching = False
1248
+ # TODO (ywang96): Enable APC by default when VLM supports it.
1249
+ assert not engine_config.cache_config.enable_prefix_caching
1250
1251
1252
@dataclass
vllm/model_executor/models/interfaces.py
+5
@@ -36,6 +36,11 @@ class SupportsMultiModal(Protocol):
36
"""
37
Returns multimodal embeddings generated from multimodal kwargs
38
to be merged with text embeddings.
39
+
40
+ The output embeddings must be one of the following formats:
41
+ - A list or tuple of 2D tensors, where each tensor corresponds to
42
+ each input image.
43
+ - A single 3D tensor, with the batch dimension grouping the 2D tensors.
44
"""
45
...
46
vllm/model_executor/models/internvl.py
+57
-11
@@ -26,7 +26,7 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,
26
InternVisionPatchModel)
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
29
-from vllm.multimodal.inputs import NestedTensors
29
+from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
30
from vllm.multimodal.utils import cached_get_tokenizer
31
from vllm.sequence import IntermediateTensors
32
from vllm.utils import is_list_of
@@ -52,12 +52,18 @@ class InternVLImagePixelInputs(TypedDict):
52
Shape:
53
`(batch_size * num_images * (1 + num_patches), num_channels, height, width)`
54
"""
55
+ patches_per_image: List[int]
56
+ """
57
+ List of number of total patches for each image in the batch.
58
+ """
59
60
61
class InternVLImageEmbeddingInputs(TypedDict):
62
type: Literal["image_embeds"]
59
- data: torch.Tensor
60
- """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
63
+ data: NestedTensors
64
+ """
65
+ A tensor of shape `(num_images, total_image_feature_size, hidden_size)`
66
+ or a list of tensors of shape `(total_image_feature_size, hidden_size)`
67
68
`hidden_size` must match the hidden size of language model backbone.
69
"""
@@ -349,10 +355,32 @@ class InternVLInputPipeline:
355
new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
356
num_patches)
357
new_prompt_token_ids = tokenizer.encode(new_prompt)
358
+ img_context_token_id = tokenizer.encode(self.img_context_token,
359
+ add_special_tokens=False)
360
+ assert len(img_context_token_id) == 1, \
361
+ (f"Invalid image token '{self.img_context_token}': A valid image "
362
+ f"token encodes to a single token ID, got {img_context_token_id}.")
363
+ img_context_token_id = img_context_token_id[0]
364
+
365
+ # Get precise tracking of placeholder positions
366
+ token_idx = image_idx = 0
367
+ placeholder_ranges = []
368
+ while token_idx < len(new_prompt_token_ids):
369
+ if new_prompt_token_ids[token_idx] == img_context_token_id:
370
+ curr_image_featue_size = image_feature_sizes[image_idx]
371
+ placeholder_ranges.append(
372
+ PlaceholderRange(offset=token_idx,
373
+ length=curr_image_featue_size))
374
+ image_idx += 1
375
+ token_idx += curr_image_featue_size
376
+ else:
377
+ token_idx += 1
378
353
- return token_inputs(prompt=prompt,
354
- prompt_token_ids=new_prompt_token_ids,
355
- multi_modal_data=multi_modal_data)
379
+ return token_inputs(
380
+ prompt=prompt,
381
+ prompt_token_ids=new_prompt_token_ids,
382
+ multi_modal_data=multi_modal_data,
383
+ multi_modal_placeholders={"image": placeholder_ranges})
384
385
def input_mapper(
386
self,
@@ -614,26 +642,46 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
642
if not isinstance(pixel_values, (torch.Tensor, list)):
643
raise ValueError("Incorrect type of pixel values. "
644
f"Got type: {type(pixel_values)}")
645
+
646
+ patches_per_image = []
647
+ for request_pixel_values in pixel_values:
648
+ for image_pixel_values in request_pixel_values:
649
+ patches_per_image.append(image_pixel_values.shape[0])
650
# We need to flatten (B, N, P) to (B*N*P),
651
# so we call flatten_bn twice.
652
return InternVLImagePixelInputs(
653
type="pixel_values",
654
data=self._validate_pixel_values(
655
flatten_bn(flatten_bn(pixel_values), concat=True)),
623
- )
656
+ patches_per_image=patches_per_image)
657
658
raise AssertionError("This line should be unreachable.")
659
660
def _process_image_input(
661
self,
662
image_input: InternVLImageInputs,
630
- ) -> torch.Tensor:
663
+ ) -> Tuple[torch.Tensor]:
664
if image_input["type"] == "image_embeds":
665
return image_input["data"]
666
667
assert self.vision_model is not None
668
+
669
image_embeds = self.extract_feature(image_input["data"])
670
671
+ patches_per_image = image_input["patches_per_image"]
672
+ if len(patches_per_image) == 1:
673
+ image_embeds = image_embeds.unsqueeze(0)
674
+ return image_embeds
675
+
676
+ # NOTE: Image embeddings are split into separate tensors for each image
677
+ # by the size of each embedding.
678
+ feature_size = image_embeds.shape[1]
679
+ image_embeds = image_embeds.view(-1,
680
+ self.config.text_config.hidden_size)
681
+ image_feature_sizes = [
682
+ num_patches * feature_size for num_patches in patches_per_image
683
+ ]
684
+ image_embeds = image_embeds.split(image_feature_sizes)
685
return image_embeds
686
687
def _set_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -696,13 +744,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
744
"inputs_embeds": inputs_embeds,
745
}
746
747
+ # Only required if the model is mono-architecture
748
if self.visual_token_mask is not None:
700
- # overwrite visual_token_mask and img_context_token_id back to None,
701
- # so that this doesn't need to depend on encoder output
749
forward_kwargs.update(
750
{"visual_token_mask": self.visual_token_mask})
751
self.visual_token_mask = None
705
- self.img_context_token_id = None
752
753
hidden_states = self.language_model.model(**forward_kwargs)
754
return hidden_states
vllm/model_executor/models/molmo.py
+63
-9
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
37
ParallelLMHead, VocabParallelEmbedding)
38
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40
-from vllm.multimodal.inputs import NestedTensors
40
+from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
41
from vllm.multimodal.utils import cached_get_tokenizer
42
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
43
SequenceData)
@@ -46,12 +46,16 @@ from vllm.transformers_utils.processor import get_processor
46
from .interfaces import SupportsMultiModal, SupportsPP
47
from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
48
make_empty_intermediate_tensors_factory, make_layers,
49
- maybe_prefix)
49
+ maybe_prefix, merge_multimodal_embeddings)
50
51
# TODO: hard-coded for now. Consider making it configurable.
52
VIT_LAYERS = [-2, -9]
53
NUM_PREFIX_TOKENS = 1
54
ADDITIONAL_VOCAB_SIZE = 128
55
+DEFAULT_IMAGE_PATCH_TOKEN_ID = 152066
56
+DEFAULT_IM_START_TOKEN_ID = 152067
57
+DEFAULT_IM_END_TOKEN_ID = 152064
58
+DEFAULT_IM_COL_TOKEN_ID = 152065
59
60
61
class MolmoImageInputs(TypedDict):
@@ -75,6 +79,11 @@ class MolmoImageInputs(TypedDict):
79
`(batch_size, num_crops, num_patch)`
80
"""
81
82
+ image_start_end: Tuple[int, int]
83
+ """Starting and ending index of placeholder
84
+ tokens
85
+ """
86
+
87
88
@dataclass
89
class VisionBackboneConfig:
@@ -918,6 +927,8 @@ def image_input_mapper_for_molmo(
927
ctx: InputContext,
928
data: object,
929
):
930
+ if isinstance(data, list):
931
+ data = data[0]
932
return MultiModalKwargs(data)
933
934
@@ -967,7 +978,22 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
978
if "image_masks" in out:
979
dummy_imgdata["image_masks"] = out["image_masks"]
980
dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long)
970
- return DummyData(dummy_seqdata, {"image": dummy_imgdata})
981
+ size = 0
982
+ offset = -1
983
+ for i in range(len(token_ids)):
984
+ if token_ids[i] in (DEFAULT_IMAGE_PATCH_TOKEN_ID,
985
+ DEFAULT_IM_START_TOKEN_ID, DEFAULT_IM_END_TOKEN_ID,
986
+ DEFAULT_IM_COL_TOKEN_ID):
987
+ if offset < 0:
988
+ offset = i
989
+ size += 1
990
+ dummy_imgdata["image_start_end"] = (offset, offset + size)
991
+ return DummyData(seq_data=dummy_seqdata,
992
+ multi_modal_data={"image": dummy_imgdata},
993
+ multi_modal_placeholders={
994
+ "image":
995
+ [PlaceholderRange(offset=offset, length=size)]
996
+ })
997
998
999
def pad_images(
@@ -1055,19 +1081,34 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1081
if image_masks is not None:
1082
image_data["image_masks"] = image_masks
1083
1058
- image_data["seq_len"] = torch.tensor(len(out["input_ids"]),
1084
+ new_prompt_token_ids = out["input_ids"].tolist()
1085
+ image_data["seq_len"] = torch.tensor(len(new_prompt_token_ids),
1086
dtype=torch.long)
1087
1088
multi_modal_data = dict(image=image_data)
1089
+ size = 0
1090
+ offset = -1
1091
+ for i in range(len(new_prompt_token_ids)):
1092
+ if new_prompt_token_ids[i] in (DEFAULT_IMAGE_PATCH_TOKEN_ID,
1093
+ DEFAULT_IM_START_TOKEN_ID,
1094
+ DEFAULT_IM_END_TOKEN_ID,
1095
+ DEFAULT_IM_COL_TOKEN_ID):
1096
+ if offset < 0:
1097
+ offset = i
1098
+ size += 1
1099
+ image_data["image_start_end"] = (offset, offset + size)
1100
1101
prompt = inputs.get("prompt")
1102
if prompt is None:
1065
- prompt = tokenizer.decode(out["input_ids"])
1103
+ prompt = tokenizer.decode(new_prompt_token_ids)
1104
1105
return token_inputs(
1068
- prompt_token_ids=out["input_ids"],
1106
+ prompt_token_ids=new_prompt_token_ids,
1107
prompt=prompt,
1108
multi_modal_data=multi_modal_data,
1109
+ multi_modal_placeholders={
1110
+ "image": [PlaceholderRange(offset=offset, length=size)]
1111
+ },
1112
)
1113
1114
@@ -1113,6 +1154,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1154
) -> Optional[MolmoImageInputs]:
1155
images = kwargs.pop("images", None)
1156
image_masks = kwargs.pop("image_masks", None)
1157
+ image_start_end = kwargs.pop("image_start_end", None)
1158
if images is None:
1159
return None
1160
@@ -1130,6 +1172,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1172
image_input_idx=image_input_idx,
1173
seq_len=seq_len,
1174
image_masks=image_masks,
1175
+ image_start_end=image_start_end,
1176
)
1177
1178
def _process_image_input(
@@ -1178,9 +1221,16 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1221
1222
# Note: In this original implementation from AI2, the final
1223
# vision_embeddings will be always be the same length
1181
- # of input embedddings, which is not very efficient.
1182
- # TODO(ywang96): see if this can be optimized.
1224
+ # of input embeddings.
1225
vision_embeddings = torch.einsum('nd,nm->md', image_features, mat)
1226
+
1227
+ # Split by the sizes of the input sequences. For each full embedding,
1228
+ # extract the actual vision embeddings to be merged.
1229
+ vision_embeddings = list(vision_embeddings.split(seq_len.tolist()))
1230
+ for i in range(len(vision_embeddings)):
1231
+ start, end = image_input['image_start_end'][i]
1232
+ vision_embeddings[i] = vision_embeddings[i][start:end]
1233
+
1234
return vision_embeddings
1235
1236
def get_input_embeddings(
@@ -1190,7 +1240,11 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1240
) -> torch.Tensor:
1241
inputs_embeds = self.model.get_input_embeddings(input_ids)
1242
if multimodal_embeddings is not None:
1193
- inputs_embeds = inputs_embeds + multimodal_embeddings
1243
+ inputs_embeds = merge_multimodal_embeddings(
1244
+ input_ids, inputs_embeds, multimodal_embeddings, [
1245
+ DEFAULT_IMAGE_PATCH_TOKEN_ID, DEFAULT_IM_START_TOKEN_ID,
1246
+ DEFAULT_IM_END_TOKEN_ID, DEFAULT_IM_COL_TOKEN_ID
1247
+ ])
1248
return inputs_embeds
1249
1250
def forward(
vllm/model_executor/models/pixtral.py
+92
-29
@@ -48,6 +48,9 @@ try:
48
except ImportError:
49
USE_XFORMERS_OPS = False
50
51
+PIXTRAL_IMAGE_BREAK_ID = 12
52
+PIXTRAL_IMAGE_END_ID = 13
53
+
54
55
def get_max_pixtral_image_tokens(ctx: InputContext):
56
tokenizer = cached_get_tokenizer(
@@ -68,7 +71,6 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
71
tokenizer_mode=ctx.model_config.tokenizer_mode)
72
73
mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
71
- patch_size = mm_encoder.mm_config.image_patch_size
74
image_token_id = mm_encoder.special_ids.img
75
76
mm_config = ctx.model_config.multimodal_config
@@ -78,8 +80,8 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
80
size = 256
81
image = Image.new("RGB", (size, size), color=0)
82
81
- image_feature_size = (size**2) // (patch_size**2)
82
-
83
+ encoding = tokenizer.instruct.mm_encoder(ImageChunk(image=image))
84
+ image_feature_size = len(encoding.tokens)
85
num_image_tokens = image_feature_size * num_images
86
seq_data = SequenceData.from_prompt_token_counts(
87
(image_token_id, num_image_tokens),
@@ -101,14 +103,13 @@ def input_mapper_for_pixtral(ctx: InputContext,
103
104
Args:
105
ctx: Context of the loaded model.
104
- data: data potentially containing image/image embeddings to be mapped
105
- to pixel_values in .forward() for a visual QWenLMHeadModel model.
106
+ data: data potentially containing PIL images to be processed
107
+ and mapped to `images`.
108
109
Returns:
110
MultiModalKwargs containing the stacked normalized images tensor or
111
image embeddings.
112
"""
111
- # Early exit if we have provided an image to a language only Qwen model
113
model_config = ctx.model_config
114
tokenizer = cached_get_tokenizer(
115
model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
@@ -116,35 +117,67 @@ def input_mapper_for_pixtral(ctx: InputContext,
117
data_list = data if isinstance(data, list) else [data]
118
119
images = []
120
+ image_tokens_list = []
121
for image_data in data_list:
122
image = ImageChunk(image=image_data)
123
encoding = tokenizer.instruct.mm_encoder(image)
124
image = torch.from_numpy(encoding.image).to(device="cuda",
125
dtype=torch.float16)
126
images.append(image)
127
+ image_tokens_list.append(encoding.tokens)
128
126
- return MultiModalKwargs({"images": images})
129
+ image_tokens = torch.tensor([
130
+ token_id for image_tokens in image_tokens_list
131
+ for token_id in image_tokens
132
+ ])
133
+ return MultiModalKwargs({"images": images, "image_tokens": image_tokens})
134
135
136
def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
137
multi_modal_data = inputs.get("multi_modal_data")
131
- if multi_modal_data is not None and "image" in multi_modal_data:
132
- tokenizer = cached_get_tokenizer(
133
- ctx.model_config.tokenizer,
134
- tokenizer_mode=ctx.model_config.tokenizer_mode)
135
-
136
- mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
137
- image_token_id = mm_encoder.special_ids.img
138
+ if multi_modal_data is None or "image" not in multi_modal_data:
139
+ return inputs
140
139
- if image_token_id not in inputs['prompt_token_ids']:
140
- raise ValueError(
141
- f"You've passed {inputs=} without {image_token_id=}"
142
- " Make sure to process your input via mistral_common's"
143
- " tokenizer or pass a chat completion request. For more"
144
- " For more info, see: "
145
- "https://github.com/vllm-project/vllm/issues/8411.")
141
+ prompt_token_ids = inputs.get("prompt_token_ids")
142
+ prompt = inputs.get("prompt")
143
+ tokenizer = cached_get_tokenizer(
144
+ ctx.model_config.tokenizer,
145
+ tokenizer_mode=ctx.model_config.tokenizer_mode)
146
147
- return inputs
147
+ mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
148
+ image_token_id = mm_encoder.special_ids.img
149
+ image_break_id = mm_encoder.special_ids.img_break
150
+ image_end_id = mm_encoder.special_ids.img_end
151
+
152
+ if image_token_id not in inputs['prompt_token_ids']:
153
+ raise ValueError(
154
+ f"You've passed {inputs=} without {image_token_id=}"
155
+ " Make sure to process your input via mistral_common's"
156
+ " tokenizer or pass a chat completion request. For more"
157
+ " For more info, see: "
158
+ "https://github.com/vllm-project/vllm/issues/8411.")
159
+
160
+ # Get precise tracking of placeholder positions
161
+ placeholder_ranges = []
162
+ curr_offset = -1
163
+ curr_length = 0
164
+ for i in range(len(prompt_token_ids)):
165
+ if prompt_token_ids[i] in (image_token_id, image_break_id):
166
+ if curr_offset < 0:
167
+ curr_offset = i
168
+ curr_length += 1
169
+ elif prompt_token_ids[i] == image_end_id:
170
+ curr_length += 1
171
+ placeholder_ranges.append(
172
+ PlaceholderRange(offset=curr_offset, length=curr_length))
173
+ curr_offset = -1
174
+ curr_length = 0
175
+ else:
176
+ pass
177
+ return token_inputs(prompt=prompt,
178
+ prompt_token_ids=prompt_token_ids,
179
+ multi_modal_data=multi_modal_data,
180
+ multi_modal_placeholders={"image": placeholder_ranges})
181
182
183
@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_pixtral)
@@ -192,11 +225,29 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
225
return get_sampler()
226
227
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
195
- image_input = self._parse_and_validate_image_input(**kwargs)
228
+ image_input, image_tokens = self._parse_and_validate_image_input(
229
+ **kwargs)
230
if image_input is None:
231
return None
232
+
233
vision_embeddings = self._process_image_input(image_input)
199
- return vision_embeddings
234
+
235
+ # NOTE: We patch the outputs of the vision encoder with embeddings
236
+ # from `[IMG_BREAK]` and `[IMG_END]` tokens.
237
+ image_embeds = self.language_model.get_input_embeddings(image_tokens)
238
+ image_token_mask = image_tokens == self.vision_args.image_token_id
239
+ image_embeds[image_token_mask] = vision_embeddings
240
+
241
+ # NOTE: Image embeddings are split into separate tensors for each image
242
+ # by the indices of `[IMG_END]` token.
243
+ split_indices = torch.where(
244
+ image_tokens == PIXTRAL_IMAGE_END_ID)[0] + 1
245
+ if len(split_indices) <= 1:
246
+ # Do not split, return as tensor of shape [1, fs, hs]
247
+ return image_embeds.unsqueeze(0)
248
+
249
+ image_embeds = image_embeds.tensor_split(split_indices.cpu())
250
+ return image_embeds
251
252
def get_input_embeddings(
253
self,
@@ -206,8 +257,10 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
257
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
258
if multimodal_embeddings is not None:
259
inputs_embeds = merge_multimodal_embeddings(
209
- input_ids, inputs_embeds, multimodal_embeddings,
210
- self.vision_args.image_token_id)
260
+ input_ids, inputs_embeds, multimodal_embeddings, [
261
+ self.vision_args.image_token_id, PIXTRAL_IMAGE_END_ID,
262
+ PIXTRAL_IMAGE_BREAK_ID
263
+ ])
264
return inputs_embeds
265
266
def forward(
@@ -245,10 +298,11 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
298
def _parse_and_validate_image_input(
299
self,
300
images: Optional[Union[List[List[torch.Tensor]], List[torch.Tensor],
248
- torch.Tensor]] = None
301
+ torch.Tensor]] = None,
302
+ image_tokens: Optional[torch.Tensor] = None,
303
) -> Optional[List[torch.Tensor]]:
304
if images is None:
251
- return None
305
+ return None, None
306
307
if isinstance(images, torch.Tensor):
308
# if passed as batch take all images
@@ -267,7 +321,16 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
321
322
images = flatten_images
323
270
- return images
324
+ if isinstance(image_tokens, torch.Tensor):
325
+ # image_tokens are batched
326
+ image_tokens = image_tokens.flatten()
327
+ elif isinstance(image_tokens, list):
328
+ # image_tokens are of different lengths thus passed as a list
329
+ image_tokens = torch.cat(image_tokens)
330
+
331
+ assert image_tokens.dim() == 1
332
+
333
+ return images, image_tokens
334
335
def _process_image_input(self,
336
image_input: List[torch.Tensor]) -> torch.Tensor:
vllm/model_executor/models/utils.py
+27
-1
@@ -409,16 +409,42 @@ def merge_multimodal_embeddings(
409
input_ids: torch.Tensor,
410
inputs_embeds: torch.Tensor,
411
multimodal_embeddings: NestedTensors,
412
- placeholder_token_id: int,
412
+ placeholder_token_id: Union[int, List[int]],
413
) -> torch.Tensor:
414
"""
415
Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
416
positions in ``inputs_embeds`` corresponding to placeholder tokens in
417
``input_ids``.
418
+
419
+ ``placeholder_token_id`` can be a list of token ids (e.g, token ids
420
+ of img_start, img_break, and img_end tokens) when needed: This means
421
+ the order of these tokens in the ``input_ids`` MUST MATCH the order of
422
+ their embeddings in ``multimodal_embeddings`` since we need to
423
+ slice-merge instead of individually scattering.
424
+
425
+ For example, if input_ids is "TTTTTSIIIBIIIBIIIETTT", where
426
+ - T is text token
427
+ - S is image start token
428
+ - I is image embedding token
429
+ - B is image break token
430
+ - E is image end token.
431
+
432
+ Then the image embeddings (that correspond to I's) from vision encoder
433
+ must be padded with embeddings of S, B, and E in the same order of
434
+ input_ids for a correct embedding merge.
435
436
Note:
437
This updates ``inputs_embeds`` in place.
438
"""
439
+ if isinstance(placeholder_token_id, list):
440
+ placeholder_token_id = torch.tensor(placeholder_token_id,
441
+ device=input_ids.device)
442
+ return _merge_multimodal_embeddings(
443
+ inputs_embeds,
444
+ torch.isin(input_ids, placeholder_token_id),
445
+ multimodal_embeddings,
446
+ )
447
+
448
return _merge_multimodal_embeddings(
449
inputs_embeds,
450
(input_ids == placeholder_token_id),
vllm/multimodal/inputs.py
+2
-1
@@ -96,7 +96,8 @@ class PlaceholderRange(TypedDict):
96
"""The length of the placeholder."""
97
98
99
-NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
99
+NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor,
100
+ Tuple[torch.Tensor, ...]]
101
"""
102
Uses a list instead of a tensor if the dimensions of each element do not match.
103
"""
vllm/multimodal/utils.py
+6
-4
@@ -535,11 +535,13 @@ def repeat_and_pad_placeholder_tokens(
535
return new_prompt, new_token_ids, placeholder_ranges
536
537
538
-def consecutive_placeholder_ranges(num_items: int,
539
- item_size: int) -> List[PlaceholderRange]:
538
+def consecutive_placeholder_ranges(
539
+ num_items: int,
540
+ item_size: int,
541
+ initial_offset: int = 0) -> List[PlaceholderRange]:
542
"""Returns a list of consecutive PlaceholderRanges of a fixed size"""
543
544
return [
543
- PlaceholderRange(offset=i * item_size, length=item_size)
544
- for i in range(num_items)
545
+ PlaceholderRange(offset=initial_offset + i * item_size,
546
+ length=item_size) for i in range(num_items)
547
]
vllm/v1/core/scheduler.py
+2
-2
@@ -73,12 +73,12 @@ class Scheduler:
73
# has the Transformer architecture (e.g., ViT).
74
# FIXME(woosuk): Below are placeholder values. We need to calculate the
75
# actual values from the configurations.
76
- self.max_num_encoder_input_tokens = 2048
76
+ self.max_num_encoder_input_tokens = 16384
77
# NOTE(woosuk): For the models without encoder (e.g., text-only models),
78
# the encoder cache will not be initialized and used, regardless of
79
# the cache size. This is because the memory space for the encoder cache
80
# is preallocated in the profiling run.
81
- self.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
81
+ self.encoder_cache_manager = EncoderCacheManager(cache_size=16384)
82
83
def schedule(self) -> "SchedulerOutput":
84
# NOTE(woosuk) on the scheduling algorithm:
vllm/v1/engine/llm_engine.py
+21
-3
@@ -1,5 +1,7 @@
1
from typing import Dict, List, Mapping, Optional, Type, Union
2
3
+from typing_extensions import TypeVar
4
+
5
from vllm.config import VllmConfig
6
from vllm.engine.arg_utils import EngineArgs
7
from vllm.engine.metrics_types import StatLoggerBase
@@ -12,7 +14,8 @@ from vllm.outputs import RequestOutput
14
from vllm.pooling_params import PoolingParams
15
from vllm.prompt_adapter.request import PromptAdapterRequest
16
from vllm.sampling_params import SamplingParams
15
-from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
17
+from vllm.transformers_utils.tokenizer_group import (
18
+ BaseTokenizerGroup, init_tokenizer_from_configs)
19
from vllm.usage.usage_lib import UsageContext
20
from vllm.v1.engine.core_client import EngineCoreClient
21
from vllm.v1.engine.detokenizer import Detokenizer
@@ -21,6 +24,8 @@ from vllm.v1.executor.gpu_executor import GPUExecutor
24
25
logger = init_logger(__name__)
26
27
+_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
28
+
29
30
class LLMEngine:
31
"""Legacy LLMEngine for backwards compatibility."""
@@ -169,5 +174,18 @@ class LLMEngine:
174
def stop_profile(self):
175
self.engine_core.profile(False)
176
172
- def get_tokenizer_group(self, group_type):
173
- pass
177
+ def get_tokenizer_group(
178
+ self,
179
+ group_type: Type[_G] = BaseTokenizerGroup,
180
+ ) -> _G:
181
+ tokenizer_group = self.tokenizer
182
+
183
+ if tokenizer_group is None:
184
+ raise ValueError("Unable to get tokenizer because "
185
+ "skip_tokenizer_init is True")
186
+ if not isinstance(tokenizer_group, group_type):
187
+ raise TypeError("Invalid type of tokenizer group. "
188
+ f"Expected type: {group_type}, but "
189
+ f"found type: {type(tokenizer_group)}")
190
+
191
+ return tokenizer_group
vllm/v1/engine/mm_input_mapper.py
+1
-1
@@ -33,7 +33,7 @@ class MMInputMapper:
33
num_images = len(image_inputs)
34
for i in range(num_images):
35
mm_input = self.multi_modal_input_mapper(
36
- {"image": [image_inputs[i]]},
36
+ {"image": image_inputs[i]},
37
mm_processor_kwargs=mm_processor_kwargs,
38
)
39
mm_inputs.append(mm_input)