@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a11f326528
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Initial support of multimodal models for V1 re-arch (#10699)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Dec 8, 2024 at 04:50 UTC a11f3265282c712d1d9fa75368e2a8c40019fbb7
11 files changed +284 -69
vllm/engine/arg_utils.py
+8 -8
@@ -1050,9 +1050,12 @@ class EngineArgs:
1050 # long context (> 32K) models. This is to avoid OOM errors in the
1051 # initial memory profiling phase.
1052
1053 - # Chunked prefill is currently disabled for multimodal models by
1054 - # default.
1055 - if use_long_context and not model_config.is_multimodal_model:
1053 + # For multimodal models, chunked prefill is disabled by default in
1054 + # V0, but enabled by design in V1
1055 + if model_config.is_multimodal_model:
1056 + self.enable_chunked_prefill = bool(envs.VLLM_USE_V1)
1057 +
1058 + elif use_long_context:
1059 is_gpu = device_config.device_type == "cuda"
1060 use_sliding_window = (model_config.get_sliding_window()
1061 is not None)
@@ -1241,12 +1244,9 @@ class EngineArgs:
1244 Override the EngineConfig's configs based on the usage context for V1.
1245 """
1246 assert envs.VLLM_USE_V1, "V1 is not enabled"
1244 - # TODO (ywang96): Enable APC by default when VLM supports it.
1247 if engine_config.model_config.is_multimodal_model:
1246 - logger.warning(
1247 - "Prefix caching is currently not supported for multimodal "
1248 - "models and has been disabled.")
1249 - engine_config.cache_config.enable_prefix_caching = False
1248 + # TODO (ywang96): Enable APC by default when VLM supports it.
1249 + assert not engine_config.cache_config.enable_prefix_caching
1250
1251
1252 @dataclass
vllm/model_executor/models/interfaces.py
+5
@@ -36,6 +36,11 @@ class SupportsMultiModal(Protocol):
36 """
37 Returns multimodal embeddings generated from multimodal kwargs
38 to be merged with text embeddings.
39 +
40 + The output embeddings must be one of the following formats:
41 + - A list or tuple of 2D tensors, where each tensor corresponds to
42 + each input image.
43 + - A single 3D tensor, with the batch dimension grouping the 2D tensors.
44 """
45 ...
46
vllm/model_executor/models/internvl.py
+57 -11
@@ -26,7 +26,7 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
29 -from vllm.multimodal.inputs import NestedTensors
29 +from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
30 from vllm.multimodal.utils import cached_get_tokenizer
31 from vllm.sequence import IntermediateTensors
32 from vllm.utils import is_list_of
@@ -52,12 +52,18 @@ class InternVLImagePixelInputs(TypedDict):
52 Shape:
53 `(batch_size * num_images * (1 + num_patches), num_channels, height, width)`
54 """
55 + patches_per_image: List[int]
56 + """
57 + List of number of total patches for each image in the batch.
58 + """
59
60
61 class InternVLImageEmbeddingInputs(TypedDict):
62 type: Literal["image_embeds"]
59 - data: torch.Tensor
60 - """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
63 + data: NestedTensors
64 + """
65 + A tensor of shape `(num_images, total_image_feature_size, hidden_size)`
66 + or a list of tensors of shape `(total_image_feature_size, hidden_size)`
67
68 `hidden_size` must match the hidden size of language model backbone.
69 """
@@ -349,10 +355,32 @@ class InternVLInputPipeline:
355 new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
356 num_patches)
357 new_prompt_token_ids = tokenizer.encode(new_prompt)
358 + img_context_token_id = tokenizer.encode(self.img_context_token,
359 + add_special_tokens=False)
360 + assert len(img_context_token_id) == 1, \
361 + (f"Invalid image token '{self.img_context_token}': A valid image "
362 + f"token encodes to a single token ID, got {img_context_token_id}.")
363 + img_context_token_id = img_context_token_id[0]
364 +
365 + # Get precise tracking of placeholder positions
366 + token_idx = image_idx = 0
367 + placeholder_ranges = []
368 + while token_idx < len(new_prompt_token_ids):
369 + if new_prompt_token_ids[token_idx] == img_context_token_id:
370 + curr_image_featue_size = image_feature_sizes[image_idx]
371 + placeholder_ranges.append(
372 + PlaceholderRange(offset=token_idx,
373 + length=curr_image_featue_size))
374 + image_idx += 1
375 + token_idx += curr_image_featue_size
376 + else:
377 + token_idx += 1
378
353 - return token_inputs(prompt=prompt,
354 - prompt_token_ids=new_prompt_token_ids,
355 - multi_modal_data=multi_modal_data)
379 + return token_inputs(
380 + prompt=prompt,
381 + prompt_token_ids=new_prompt_token_ids,
382 + multi_modal_data=multi_modal_data,
383 + multi_modal_placeholders={"image": placeholder_ranges})
384
385 def input_mapper(
386 self,
@@ -614,26 +642,46 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
642 if not isinstance(pixel_values, (torch.Tensor, list)):
643 raise ValueError("Incorrect type of pixel values. "
644 f"Got type: {type(pixel_values)}")
645 +
646 + patches_per_image = []
647 + for request_pixel_values in pixel_values:
648 + for image_pixel_values in request_pixel_values:
649 + patches_per_image.append(image_pixel_values.shape[0])
650 # We need to flatten (B, N, P) to (B*N*P),
651 # so we call flatten_bn twice.
652 return InternVLImagePixelInputs(
653 type="pixel_values",
654 data=self._validate_pixel_values(
655 flatten_bn(flatten_bn(pixel_values), concat=True)),
623 - )
656 + patches_per_image=patches_per_image)
657
658 raise AssertionError("This line should be unreachable.")
659
660 def _process_image_input(
661 self,
662 image_input: InternVLImageInputs,
630 - ) -> torch.Tensor:
663 + ) -> Tuple[torch.Tensor]:
664 if image_input["type"] == "image_embeds":
665 return image_input["data"]
666
667 assert self.vision_model is not None
668 +
669 image_embeds = self.extract_feature(image_input["data"])
670
671 + patches_per_image = image_input["patches_per_image"]
672 + if len(patches_per_image) == 1:
673 + image_embeds = image_embeds.unsqueeze(0)
674 + return image_embeds
675 +
676 + # NOTE: Image embeddings are split into separate tensors for each image
677 + # by the size of each embedding.
678 + feature_size = image_embeds.shape[1]
679 + image_embeds = image_embeds.view(-1,
680 + self.config.text_config.hidden_size)
681 + image_feature_sizes = [
682 + num_patches * feature_size for num_patches in patches_per_image
683 + ]
684 + image_embeds = image_embeds.split(image_feature_sizes)
685 return image_embeds
686
687 def _set_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -696,13 +744,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
744 "inputs_embeds": inputs_embeds,
745 }
746
747 + # Only required if the model is mono-architecture
748 if self.visual_token_mask is not None:
700 - # overwrite visual_token_mask and img_context_token_id back to None,
701 - # so that this doesn't need to depend on encoder output
749 forward_kwargs.update(
750 {"visual_token_mask": self.visual_token_mask})
751 self.visual_token_mask = None
705 - self.img_context_token_id = None
752
753 hidden_states = self.language_model.model(**forward_kwargs)
754 return hidden_states
vllm/model_executor/models/molmo.py
+63 -9
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
37 ParallelLMHead, VocabParallelEmbedding)
38 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40 -from vllm.multimodal.inputs import NestedTensors
40 +from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
41 from vllm.multimodal.utils import cached_get_tokenizer
42 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
43 SequenceData)
@@ -46,12 +46,16 @@ from vllm.transformers_utils.processor import get_processor
46 from .interfaces import SupportsMultiModal, SupportsPP
47 from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
48 make_empty_intermediate_tensors_factory, make_layers,
49 - maybe_prefix)
49 + maybe_prefix, merge_multimodal_embeddings)
50
51 # TODO: hard-coded for now. Consider making it configurable.
52 VIT_LAYERS = [-2, -9]
53 NUM_PREFIX_TOKENS = 1
54 ADDITIONAL_VOCAB_SIZE = 128
55 +DEFAULT_IMAGE_PATCH_TOKEN_ID = 152066
56 +DEFAULT_IM_START_TOKEN_ID = 152067
57 +DEFAULT_IM_END_TOKEN_ID = 152064
58 +DEFAULT_IM_COL_TOKEN_ID = 152065
59
60
61 class MolmoImageInputs(TypedDict):
@@ -75,6 +79,11 @@ class MolmoImageInputs(TypedDict):
79 `(batch_size, num_crops, num_patch)`
80 """
81
82 + image_start_end: Tuple[int, int]
83 + """Starting and ending index of placeholder
84 + tokens
85 + """
86 +
87
88 @dataclass
89 class VisionBackboneConfig:
@@ -918,6 +927,8 @@ def image_input_mapper_for_molmo(
927 ctx: InputContext,
928 data: object,
929 ):
930 + if isinstance(data, list):
931 + data = data[0]
932 return MultiModalKwargs(data)
933
934
@@ -967,7 +978,22 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
978 if "image_masks" in out:
979 dummy_imgdata["image_masks"] = out["image_masks"]
980 dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long)
970 - return DummyData(dummy_seqdata, {"image": dummy_imgdata})
981 + size = 0
982 + offset = -1
983 + for i in range(len(token_ids)):
984 + if token_ids[i] in (DEFAULT_IMAGE_PATCH_TOKEN_ID,
985 + DEFAULT_IM_START_TOKEN_ID, DEFAULT_IM_END_TOKEN_ID,
986 + DEFAULT_IM_COL_TOKEN_ID):
987 + if offset < 0:
988 + offset = i
989 + size += 1
990 + dummy_imgdata["image_start_end"] = (offset, offset + size)
991 + return DummyData(seq_data=dummy_seqdata,
992 + multi_modal_data={"image": dummy_imgdata},
993 + multi_modal_placeholders={
994 + "image":
995 + [PlaceholderRange(offset=offset, length=size)]
996 + })
997
998
999 def pad_images(
@@ -1055,19 +1081,34 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1081 if image_masks is not None:
1082 image_data["image_masks"] = image_masks
1083
1058 - image_data["seq_len"] = torch.tensor(len(out["input_ids"]),
1084 + new_prompt_token_ids = out["input_ids"].tolist()
1085 + image_data["seq_len"] = torch.tensor(len(new_prompt_token_ids),
1086 dtype=torch.long)
1087
1088 multi_modal_data = dict(image=image_data)
1089 + size = 0
1090 + offset = -1
1091 + for i in range(len(new_prompt_token_ids)):
1092 + if new_prompt_token_ids[i] in (DEFAULT_IMAGE_PATCH_TOKEN_ID,
1093 + DEFAULT_IM_START_TOKEN_ID,
1094 + DEFAULT_IM_END_TOKEN_ID,
1095 + DEFAULT_IM_COL_TOKEN_ID):
1096 + if offset < 0:
1097 + offset = i
1098 + size += 1
1099 + image_data["image_start_end"] = (offset, offset + size)
1100
1101 prompt = inputs.get("prompt")
1102 if prompt is None:
1065 - prompt = tokenizer.decode(out["input_ids"])
1103 + prompt = tokenizer.decode(new_prompt_token_ids)
1104
1105 return token_inputs(
1068 - prompt_token_ids=out["input_ids"],
1106 + prompt_token_ids=new_prompt_token_ids,
1107 prompt=prompt,
1108 multi_modal_data=multi_modal_data,
1109 + multi_modal_placeholders={
1110 + "image": [PlaceholderRange(offset=offset, length=size)]
1111 + },
1112 )
1113
1114
@@ -1113,6 +1154,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1154 ) -> Optional[MolmoImageInputs]:
1155 images = kwargs.pop("images", None)
1156 image_masks = kwargs.pop("image_masks", None)
1157 + image_start_end = kwargs.pop("image_start_end", None)
1158 if images is None:
1159 return None
1160
@@ -1130,6 +1172,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1172 image_input_idx=image_input_idx,
1173 seq_len=seq_len,
1174 image_masks=image_masks,
1175 + image_start_end=image_start_end,
1176 )
1177
1178 def _process_image_input(
@@ -1178,9 +1221,16 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1221
1222 # Note: In this original implementation from AI2, the final
1223 # vision_embeddings will be always be the same length
1181 - # of input embedddings, which is not very efficient.
1182 - # TODO(ywang96): see if this can be optimized.
1224 + # of input embeddings.
1225 vision_embeddings = torch.einsum('nd,nm->md', image_features, mat)
1226 +
1227 + # Split by the sizes of the input sequences. For each full embedding,
1228 + # extract the actual vision embeddings to be merged.
1229 + vision_embeddings = list(vision_embeddings.split(seq_len.tolist()))
1230 + for i in range(len(vision_embeddings)):
1231 + start, end = image_input['image_start_end'][i]
1232 + vision_embeddings[i] = vision_embeddings[i][start:end]
1233 +
1234 return vision_embeddings
1235
1236 def get_input_embeddings(
@@ -1190,7 +1240,11 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1240 ) -> torch.Tensor:
1241 inputs_embeds = self.model.get_input_embeddings(input_ids)
1242 if multimodal_embeddings is not None:
1193 - inputs_embeds = inputs_embeds + multimodal_embeddings
1243 + inputs_embeds = merge_multimodal_embeddings(
1244 + input_ids, inputs_embeds, multimodal_embeddings, [
1245 + DEFAULT_IMAGE_PATCH_TOKEN_ID, DEFAULT_IM_START_TOKEN_ID,
1246 + DEFAULT_IM_END_TOKEN_ID, DEFAULT_IM_COL_TOKEN_ID
1247 + ])
1248 return inputs_embeds
1249
1250 def forward(
vllm/model_executor/models/pixtral.py
+92 -29
@@ -48,6 +48,9 @@ try:
48 except ImportError:
49 USE_XFORMERS_OPS = False
50
51 +PIXTRAL_IMAGE_BREAK_ID = 12
52 +PIXTRAL_IMAGE_END_ID = 13
53 +
54
55 def get_max_pixtral_image_tokens(ctx: InputContext):
56 tokenizer = cached_get_tokenizer(
@@ -68,7 +71,6 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
71 tokenizer_mode=ctx.model_config.tokenizer_mode)
72
73 mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
71 - patch_size = mm_encoder.mm_config.image_patch_size
74 image_token_id = mm_encoder.special_ids.img
75
76 mm_config = ctx.model_config.multimodal_config
@@ -78,8 +80,8 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
80 size = 256
81 image = Image.new("RGB", (size, size), color=0)
82
81 - image_feature_size = (size**2) // (patch_size**2)
82 -
83 + encoding = tokenizer.instruct.mm_encoder(ImageChunk(image=image))
84 + image_feature_size = len(encoding.tokens)
85 num_image_tokens = image_feature_size * num_images
86 seq_data = SequenceData.from_prompt_token_counts(
87 (image_token_id, num_image_tokens),
@@ -101,14 +103,13 @@ def input_mapper_for_pixtral(ctx: InputContext,
103
104 Args:
105 ctx: Context of the loaded model.
104 - data: data potentially containing image/image embeddings to be mapped
105 - to pixel_values in .forward() for a visual QWenLMHeadModel model.
106 + data: data potentially containing PIL images to be processed
107 + and mapped to `images`.
108
109 Returns:
110 MultiModalKwargs containing the stacked normalized images tensor or
111 image embeddings.
112 """
111 - # Early exit if we have provided an image to a language only Qwen model
113 model_config = ctx.model_config
114 tokenizer = cached_get_tokenizer(
115 model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
@@ -116,35 +117,67 @@ def input_mapper_for_pixtral(ctx: InputContext,
117 data_list = data if isinstance(data, list) else [data]
118
119 images = []
120 + image_tokens_list = []
121 for image_data in data_list:
122 image = ImageChunk(image=image_data)
123 encoding = tokenizer.instruct.mm_encoder(image)
124 image = torch.from_numpy(encoding.image).to(device="cuda",
125 dtype=torch.float16)
126 images.append(image)
127 + image_tokens_list.append(encoding.tokens)
128
126 - return MultiModalKwargs({"images": images})
129 + image_tokens = torch.tensor([
130 + token_id for image_tokens in image_tokens_list
131 + for token_id in image_tokens
132 + ])
133 + return MultiModalKwargs({"images": images, "image_tokens": image_tokens})
134
135
136 def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
137 multi_modal_data = inputs.get("multi_modal_data")
131 - if multi_modal_data is not None and "image" in multi_modal_data:
132 - tokenizer = cached_get_tokenizer(
133 - ctx.model_config.tokenizer,
134 - tokenizer_mode=ctx.model_config.tokenizer_mode)
135 -
136 - mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
137 - image_token_id = mm_encoder.special_ids.img
138 + if multi_modal_data is None or "image" not in multi_modal_data:
139 + return inputs
140
139 - if image_token_id not in inputs['prompt_token_ids']:
140 - raise ValueError(
141 - f"You've passed {inputs=} without {image_token_id=}"
142 - " Make sure to process your input via mistral_common's"
143 - " tokenizer or pass a chat completion request. For more"
144 - " For more info, see: "
145 - "https://github.com/vllm-project/vllm/issues/8411.")
141 + prompt_token_ids = inputs.get("prompt_token_ids")
142 + prompt = inputs.get("prompt")
143 + tokenizer = cached_get_tokenizer(
144 + ctx.model_config.tokenizer,
145 + tokenizer_mode=ctx.model_config.tokenizer_mode)
146
147 - return inputs
147 + mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
148 + image_token_id = mm_encoder.special_ids.img
149 + image_break_id = mm_encoder.special_ids.img_break
150 + image_end_id = mm_encoder.special_ids.img_end
151 +
152 + if image_token_id not in inputs['prompt_token_ids']:
153 + raise ValueError(
154 + f"You've passed {inputs=} without {image_token_id=}"
155 + " Make sure to process your input via mistral_common's"
156 + " tokenizer or pass a chat completion request. For more"
157 + " For more info, see: "
158 + "https://github.com/vllm-project/vllm/issues/8411.")
159 +
160 + # Get precise tracking of placeholder positions
161 + placeholder_ranges = []
162 + curr_offset = -1
163 + curr_length = 0
164 + for i in range(len(prompt_token_ids)):
165 + if prompt_token_ids[i] in (image_token_id, image_break_id):
166 + if curr_offset < 0:
167 + curr_offset = i
168 + curr_length += 1
169 + elif prompt_token_ids[i] == image_end_id:
170 + curr_length += 1
171 + placeholder_ranges.append(
172 + PlaceholderRange(offset=curr_offset, length=curr_length))
173 + curr_offset = -1
174 + curr_length = 0
175 + else:
176 + pass
177 + return token_inputs(prompt=prompt,
178 + prompt_token_ids=prompt_token_ids,
179 + multi_modal_data=multi_modal_data,
180 + multi_modal_placeholders={"image": placeholder_ranges})
181
182
183 @MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_pixtral)
@@ -192,11 +225,29 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
225 return get_sampler()
226
227 def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
195 - image_input = self._parse_and_validate_image_input(**kwargs)
228 + image_input, image_tokens = self._parse_and_validate_image_input(
229 + **kwargs)
230 if image_input is None:
231 return None
232 +
233 vision_embeddings = self._process_image_input(image_input)
199 - return vision_embeddings
234 +
235 + # NOTE: We patch the outputs of the vision encoder with embeddings
236 + # from `[IMG_BREAK]` and `[IMG_END]` tokens.
237 + image_embeds = self.language_model.get_input_embeddings(image_tokens)
238 + image_token_mask = image_tokens == self.vision_args.image_token_id
239 + image_embeds[image_token_mask] = vision_embeddings
240 +
241 + # NOTE: Image embeddings are split into separate tensors for each image
242 + # by the indices of `[IMG_END]` token.
243 + split_indices = torch.where(
244 + image_tokens == PIXTRAL_IMAGE_END_ID)[0] + 1
245 + if len(split_indices) <= 1:
246 + # Do not split, return as tensor of shape [1, fs, hs]
247 + return image_embeds.unsqueeze(0)
248 +
249 + image_embeds = image_embeds.tensor_split(split_indices.cpu())
250 + return image_embeds
251
252 def get_input_embeddings(
253 self,
@@ -206,8 +257,10 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
257 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
258 if multimodal_embeddings is not None:
259 inputs_embeds = merge_multimodal_embeddings(
209 - input_ids, inputs_embeds, multimodal_embeddings,
210 - self.vision_args.image_token_id)
260 + input_ids, inputs_embeds, multimodal_embeddings, [
261 + self.vision_args.image_token_id, PIXTRAL_IMAGE_END_ID,
262 + PIXTRAL_IMAGE_BREAK_ID
263 + ])
264 return inputs_embeds
265
266 def forward(
@@ -245,10 +298,11 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
298 def _parse_and_validate_image_input(
299 self,
300 images: Optional[Union[List[List[torch.Tensor]], List[torch.Tensor],
248 - torch.Tensor]] = None
301 + torch.Tensor]] = None,
302 + image_tokens: Optional[torch.Tensor] = None,
303 ) -> Optional[List[torch.Tensor]]:
304 if images is None:
251 - return None
305 + return None, None
306
307 if isinstance(images, torch.Tensor):
308 # if passed as batch take all images
@@ -267,7 +321,16 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
321
322 images = flatten_images
323
270 - return images
324 + if isinstance(image_tokens, torch.Tensor):
325 + # image_tokens are batched
326 + image_tokens = image_tokens.flatten()
327 + elif isinstance(image_tokens, list):
328 + # image_tokens are of different lengths thus passed as a list
329 + image_tokens = torch.cat(image_tokens)
330 +
331 + assert image_tokens.dim() == 1
332 +
333 + return images, image_tokens
334
335 def _process_image_input(self,
336 image_input: List[torch.Tensor]) -> torch.Tensor:
vllm/model_executor/models/utils.py
+27 -1
@@ -409,16 +409,42 @@ def merge_multimodal_embeddings(
409 input_ids: torch.Tensor,
410 inputs_embeds: torch.Tensor,
411 multimodal_embeddings: NestedTensors,
412 - placeholder_token_id: int,
412 + placeholder_token_id: Union[int, List[int]],
413 ) -> torch.Tensor:
414 """
415 Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
416 positions in ``inputs_embeds`` corresponding to placeholder tokens in
417 ``input_ids``.
418 +
419 + ``placeholder_token_id`` can be a list of token ids (e.g, token ids
420 + of img_start, img_break, and img_end tokens) when needed: This means
421 + the order of these tokens in the ``input_ids`` MUST MATCH the order of
422 + their embeddings in ``multimodal_embeddings`` since we need to
423 + slice-merge instead of individually scattering.
424 +
425 + For example, if input_ids is "TTTTTSIIIBIIIBIIIETTT", where
426 + - T is text token
427 + - S is image start token
428 + - I is image embedding token
429 + - B is image break token
430 + - E is image end token.
431 +
432 + Then the image embeddings (that correspond to I's) from vision encoder
433 + must be padded with embeddings of S, B, and E in the same order of
434 + input_ids for a correct embedding merge.
435
436 Note:
437 This updates ``inputs_embeds`` in place.
438 """
439 + if isinstance(placeholder_token_id, list):
440 + placeholder_token_id = torch.tensor(placeholder_token_id,
441 + device=input_ids.device)
442 + return _merge_multimodal_embeddings(
443 + inputs_embeds,
444 + torch.isin(input_ids, placeholder_token_id),
445 + multimodal_embeddings,
446 + )
447 +
448 return _merge_multimodal_embeddings(
449 inputs_embeds,
450 (input_ids == placeholder_token_id),
vllm/multimodal/inputs.py
+2 -1
@@ -96,7 +96,8 @@ class PlaceholderRange(TypedDict):
96 """The length of the placeholder."""
97
98
99 -NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
99 +NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor,
100 + Tuple[torch.Tensor, ...]]
101 """
102 Uses a list instead of a tensor if the dimensions of each element do not match.
103 """
vllm/multimodal/utils.py
+6 -4
@@ -535,11 +535,13 @@ def repeat_and_pad_placeholder_tokens(
535 return new_prompt, new_token_ids, placeholder_ranges
536
537
538 -def consecutive_placeholder_ranges(num_items: int,
539 - item_size: int) -> List[PlaceholderRange]:
538 +def consecutive_placeholder_ranges(
539 + num_items: int,
540 + item_size: int,
541 + initial_offset: int = 0) -> List[PlaceholderRange]:
542 """Returns a list of consecutive PlaceholderRanges of a fixed size"""
543
544 return [
543 - PlaceholderRange(offset=i * item_size, length=item_size)
544 - for i in range(num_items)
545 + PlaceholderRange(offset=initial_offset + i * item_size,
546 + length=item_size) for i in range(num_items)
547 ]
vllm/v1/core/scheduler.py
+2 -2
@@ -73,12 +73,12 @@ class Scheduler:
73 # has the Transformer architecture (e.g., ViT).
74 # FIXME(woosuk): Below are placeholder values. We need to calculate the
75 # actual values from the configurations.
76 - self.max_num_encoder_input_tokens = 2048
76 + self.max_num_encoder_input_tokens = 16384
77 # NOTE(woosuk): For the models without encoder (e.g., text-only models),
78 # the encoder cache will not be initialized and used, regardless of
79 # the cache size. This is because the memory space for the encoder cache
80 # is preallocated in the profiling run.
81 - self.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
81 + self.encoder_cache_manager = EncoderCacheManager(cache_size=16384)
82
83 def schedule(self) -> "SchedulerOutput":
84 # NOTE(woosuk) on the scheduling algorithm:
vllm/v1/engine/llm_engine.py
+21 -3
@@ -1,5 +1,7 @@
1 from typing import Dict, List, Mapping, Optional, Type, Union
2
3 +from typing_extensions import TypeVar
4 +
5 from vllm.config import VllmConfig
6 from vllm.engine.arg_utils import EngineArgs
7 from vllm.engine.metrics_types import StatLoggerBase
@@ -12,7 +14,8 @@ from vllm.outputs import RequestOutput
14 from vllm.pooling_params import PoolingParams
15 from vllm.prompt_adapter.request import PromptAdapterRequest
16 from vllm.sampling_params import SamplingParams
15 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
17 +from vllm.transformers_utils.tokenizer_group import (
18 + BaseTokenizerGroup, init_tokenizer_from_configs)
19 from vllm.usage.usage_lib import UsageContext
20 from vllm.v1.engine.core_client import EngineCoreClient
21 from vllm.v1.engine.detokenizer import Detokenizer
@@ -21,6 +24,8 @@ from vllm.v1.executor.gpu_executor import GPUExecutor
24
25 logger = init_logger(__name__)
26
27 +_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
28 +
29
30 class LLMEngine:
31 """Legacy LLMEngine for backwards compatibility."""
@@ -169,5 +174,18 @@ class LLMEngine:
174 def stop_profile(self):
175 self.engine_core.profile(False)
176
172 - def get_tokenizer_group(self, group_type):
173 - pass
177 + def get_tokenizer_group(
178 + self,
179 + group_type: Type[_G] = BaseTokenizerGroup,
180 + ) -> _G:
181 + tokenizer_group = self.tokenizer
182 +
183 + if tokenizer_group is None:
184 + raise ValueError("Unable to get tokenizer because "
185 + "skip_tokenizer_init is True")
186 + if not isinstance(tokenizer_group, group_type):
187 + raise TypeError("Invalid type of tokenizer group. "
188 + f"Expected type: {group_type}, but "
189 + f"found type: {type(tokenizer_group)}")
190 +
191 + return tokenizer_group
vllm/v1/engine/mm_input_mapper.py
+1 -1
@@ -33,7 +33,7 @@ class MMInputMapper:
33 num_images = len(image_inputs)
34 for i in range(num_images):
35 mm_input = self.multi_modal_input_mapper(
36 - {"image": [image_inputs[i]]},
36 + {"image": image_inputs[i]},
37 mm_processor_kwargs=mm_processor_kwargs,
38 )
39 mm_inputs.append(mm_input)