A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Update interface for mistral-format Pixtral (#10703)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Nov 27, 2024 at 04:26 UTC
9e0a147d502758ed31b35df1361e37ea6bacd4a0
1 file changed
+28
-19
vllm/model_executor/models/pixtral.py
+28
-19
@@ -31,7 +31,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
31
from vllm.model_executor.models.utils import merge_multimodal_embeddings
32
from vllm.model_executor.sampling_metadata import SamplingMetadata
33
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
34
-from vllm.multimodal.inputs import PlaceholderRange
34
+from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
35
from vllm.multimodal.utils import (cached_get_tokenizer,
36
consecutive_placeholder_ranges,
37
resolve_visual_encoder_outputs)
@@ -190,6 +190,25 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
190
191
return get_sampler()
192
193
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
194
+ image_input = self._parse_and_validate_image_input(**kwargs)
195
+ if image_input is None:
196
+ return None
197
+ vision_embeddings = self._process_image_input(image_input)
198
+ return vision_embeddings
199
+
200
+ def get_input_embeddings(
201
+ self,
202
+ input_ids: torch.Tensor,
203
+ multimodal_embeddings: Optional[NestedTensors] = None,
204
+ ) -> torch.Tensor:
205
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
206
+ if multimodal_embeddings is not None:
207
+ inputs_embeds = merge_multimodal_embeddings(
208
+ input_ids, inputs_embeds, multimodal_embeddings,
209
+ self.vision_args.image_token_id)
210
+ return inputs_embeds
211
+
212
def forward(
213
self,
214
input_ids: torch.Tensor,
@@ -197,31 +216,21 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
216
kv_caches: List[torch.Tensor],
217
attn_metadata: AttentionMetadata,
218
intermediate_tensors: Optional[IntermediateTensors] = None,
219
+ inputs_embeds: Optional[torch.Tensor] = None,
220
**kwargs: object,
221
) -> Union[torch.Tensor, IntermediateTensors]:
222
"""Run forward pass for pixtral.
203
-
204
- TODO
205
-
223
"""
224
if intermediate_tensors is not None:
208
- input_ids = None
225
inputs_embeds = None
210
- else:
211
- image_input = self._parse_and_validate_image_input(**kwargs)
212
-
213
- if image_input is not None:
214
- vision_embeddings = self._process_image_input(image_input)
215
- inputs_embeds = self.language_model.model.get_input_embeddings(
216
- input_ids)
226
218
- inputs_embeds = merge_multimodal_embeddings(
219
- input_ids, inputs_embeds, vision_embeddings,
220
- self.vision_args.image_token_id)
221
-
222
- input_ids = None
223
- else:
224
- inputs_embeds = None
227
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
228
+ # condition is for v0 compatibility.
229
+ elif inputs_embeds is None:
230
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
231
+ inputs_embeds = self.get_input_embeddings(input_ids,
232
+ vision_embeddings)
233
+ input_ids = None
234
235
hidden_states = self.language_model.model(input_ids,
236
positions,