A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 23:00 UTC
[Model Runner V2] Fix mm input embeddings lookup (#36588)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Mar 10, 2026 at 00:24 UTC
ddbb0d230a3592106ac9f5f7f4e9a861863fcbee
1 file changed
+4
-1
vllm/v1/worker/gpu/model_states/default.py
+4
-1
@@ -98,8 +98,11 @@ class DefaultModelState(ModelState):
98
req_states.prefill_len.np[input_batch.idx_mapping_np],
99
req_states.num_computed_prefill_tokens[input_batch.idx_mapping_np],
100
)
101
+ # Use unpadded input_ids to match is_mm_embed size (num_tokens).
102
+ # input_batch.input_ids may be padded for CUDA graphs.
103
+ input_ids_unpadded = input_batch.input_ids[: input_batch.num_tokens]
104
inputs_embeds = self.encoder_runner.get_inputs_embeds(
102
- input_batch.input_ids, mm_embeds, is_mm_embed
105
+ input_ids_unpadded, mm_embeds, is_mm_embed
106
)
107
return inputs_embeds[: input_batch.num_tokens_after_padding]
108