A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 13:30 UTC
[Bugfix] Fix overallocation in MM profiling (#29386)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Nov 25, 2025 at 04:38 UTC
c2c661af9be413fb22adc59fc17fe5f5a680b313
1 file changed
+8
-4
vllm/v1/worker/gpu_model_runner.py
+8
-4
@@ -4245,14 +4245,18 @@ class GPUModelRunner(
4245
# NOTE: This happens when encoder cache needs to store
4246
# the embeddings that encoder outputs are scattered onto.
4247
# In this case we create dummy embeddings of size
4248
- # (encode_budget, hidden_size) and scatter encoder
4249
- # output into it.
4248
+ # (max_tokens_for_modality, hidden_size) and scatter
4249
+ # encoder output into it.
4250
encoder_output_shape = dummy_encoder_outputs[0].shape
4251
- if encoder_output_shape[0] < encoder_budget:
4251
+ max_mm_tokens_per_item = mm_budget.max_tokens_by_modality[
4252
+ dummy_modality
4253
+ ]
4254
+ if encoder_output_shape[0] < max_mm_tokens_per_item:
4255
+ encoder_hidden_size = encoder_output_shape[-1]
4256
expanded_outputs = []
4257
for output in dummy_encoder_outputs:
4258
expanded = output.new_zeros(
4255
- (encoder_budget, encoder_output_shape[-1])
4259
+ (max_mm_tokens_per_item, encoder_hidden_size)
4260
)
4261
num_tokens = output.shape[0]
4262
expanded[:num_tokens].copy_(output)