A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Misc] Make MM embedding merge interface explicit in model runner (#21147)
Signed-off-by: Roger Wang <hey@rogerw.me> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Roger Wang committed
Jul 18, 2025 at 00:13 UTC
ba2dfbb0c27d8a8d224e41cebf83cfd6fcfd9293
2 files changed
+8
-10
vllm/v1/worker/gpu_model_runner.py
+4
-5
@@ -1328,11 +1328,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1328
# embeddings), we always use embeddings (rather than token ids)
1329
# as input to the multimodal model, even when the input is text.
1330
input_ids = self.input_ids[:num_scheduled_tokens]
1331
- if mm_embeds:
1332
- inputs_embeds = self.model.get_input_embeddings(
1333
- input_ids, mm_embeds)
1334
- else:
1335
- inputs_embeds = self.model.get_input_embeddings(input_ids)
1331
+ inputs_embeds = self.model.get_input_embeddings(
1332
+ input_ids=input_ids,
1333
+ multimodal_embeddings=mm_embeds or None,
1334
+ )
1335
# TODO(woosuk): Avoid the copy. Optimize.
1336
self.inputs_embeds[:num_scheduled_tokens].copy_(inputs_embeds)
1337
inputs_embeds = self.inputs_embeds[:num_input_tokens]
vllm/v1/worker/tpu_model_runner.py
+4
-5
@@ -937,11 +937,10 @@ class TPUModelRunner(LoRAModelRunnerMixin):
937
# NOTE(woosuk): To unify token ids and soft tokens (vision
938
# embeddings), we always use embeddings (rather than token ids)
939
# as input to the multimodal model, even when the input is text.
940
- if mm_embeds:
941
- inputs_embeds = self.model.get_input_embeddings(
942
- input_ids, mm_embeds)
943
- else:
944
- inputs_embeds = self.model.get_input_embeddings(input_ids)
940
+ inputs_embeds = self.model.get_input_embeddings(
941
+ input_ids=input_ids,
942
+ multimodal_embeddings=mm_embeds,
943
+ )
944
return None, inputs_embeds
945
else:
946
# For text-only models, we use token ids as input.