A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Bugfix] Correct max tokens for non-contiguous embeds (#21798)
Signed-off-by: Alexandre Milesi <30204471+milesial@users.noreply.github.com> Co-authored-by: Alexandre Milesi <30204471+milesial@users.noreply.github.com>
milesial committed
Jul 29, 2025 at 18:16 UTC
0e36abf9931baa070609376debb4fb3772f4a3fe
2 files changed
+29
-4
vllm/multimodal/profiling.py
+28
-3
@@ -180,11 +180,14 @@ class MultiModalProfiler(Generic[_I]):
180
def _get_mm_num_tokens(
181
self,
182
mm_inputs: MultiModalInputs,
183
+ mm_embeddings_only: bool = True,
184
) -> Mapping[str, int]:
185
placeholders_by_modality = mm_inputs["mm_placeholders"]
186
187
return {
187
- modality: sum(item.get_num_embeds() for item in placeholders)
188
+ modality:
189
+ sum(item.get_num_embeds() if mm_embeddings_only else item.length
190
+ for item in placeholders)
191
for modality, placeholders in placeholders_by_modality.items()
192
}
193
@@ -253,10 +256,11 @@ class MultiModalProfiler(Generic[_I]):
256
multi_modal_placeholders=mm_inputs["mm_placeholders"],
257
)
258
256
- def get_mm_max_tokens(
259
+ def _get_mm_max_tokens(
260
self,
261
seq_len: int,
262
mm_counts: Optional[Mapping[str, int]] = None,
263
+ mm_embeddings_only: bool = True,
264
) -> Mapping[str, int]:
265
if mm_counts is None:
266
mm_counts = self.get_mm_limits()
@@ -285,4 +289,25 @@ class MultiModalProfiler(Generic[_I]):
289
return max_tokens_per_item
290
291
mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
288
- return self._get_mm_num_tokens(mm_inputs)
292
+ return self._get_mm_num_tokens(mm_inputs,
293
+ mm_embeddings_only=mm_embeddings_only)
294
+
295
+ def get_mm_max_contiguous_tokens(
296
+ self,
297
+ seq_len: int,
298
+ mm_counts: Optional[Mapping[str, int]] = None,
299
+ ):
300
+ """
301
+ Returns the maximum length of the multimodal (image placeholders+text)
302
+ tokens, including any break/text tokens in-between image embeddings.
303
+
304
+ <im_start> [IMG] [IMG] [IMG] <row_break> [IMG] [IMG] [IMG] <im_end>
305
+ Returns 9, even when the number of image embeddings is 6.
306
+
307
+ This is important to take into account when profiling and
308
+ initializing the encoder cache size.
309
+ """
310
+
311
+ return self._get_mm_max_tokens(seq_len,
312
+ mm_counts,
313
+ mm_embeddings_only=False)
vllm/multimodal/registry.py
+1
-1
@@ -129,7 +129,7 @@ class MultiModalRegistry:
129
seq_len = model_config.max_model_len
130
mm_limits = self.get_mm_limits_per_prompt(model_config)
131
132
- return profiler.get_mm_max_tokens(
132
+ return profiler.get_mm_max_contiguous_tokens(
133
seq_len,
134
{
135
modality: 1