A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 09:00 UTC
[Model Runner V2] Fix inputs_embeds=None bug for MM models (#35917)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Woosuk Kwon committed
Mar 3, 2026 at 13:47 UTC
467886a0c48b37552c8a2f3bdea99e96f2e98f8c
1 file changed
+3
-1
vllm/v1/worker/gpu/model_runner.py
+3
-1
@@ -907,9 +907,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
907
)
908
909
inputs_embeds = None
910
- if self.supports_mm_inputs and self.is_first_pp_rank and not dummy_run:
910
+ if self.supports_mm_inputs and self.is_first_pp_rank:
911
# Run MM encoder (if needed) and get multimodal embeddings.
912
# Only first PP rank prepares multimodal embeddings.
913
+ # NOTE(woosuk): We must call get_mm_embeddings even during dummy runs
914
+ # to obtain inputs_embeds, because the compiled model expects this input.
915
inputs_embeds = self.model_state.get_mm_embeddings(
916
scheduler_output.scheduled_encoder_inputs,
917
input_batch,