A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix mm_limits access for merged multi-modal processor (#12252)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 21, 2025 at 18:09 UTC
a94eee4456b05458bafacc17377de4701ac598a0
2 files changed
+16
-7
vllm/multimodal/profiling.py
+2
-2
@@ -106,7 +106,7 @@ class MultiModalProfiler(Generic[_I]):
106
def dummy_inputs(self) -> BaseDummyInputsBuilder[_I]:
107
return self.processor.dummy_inputs
108
109
- def _get_mm_limits(self) -> Mapping[str, int]:
109
+ def get_mm_limits(self) -> Mapping[str, int]:
110
mm_config = self.processing_info.ctx.get_mm_config()
111
mm_limit_per_prompt = mm_config.limit_per_prompt
112
@@ -146,7 +146,7 @@ class MultiModalProfiler(Generic[_I]):
146
# Avoid circular import
147
from vllm.sequence import SequenceData
148
149
- mm_counts = self._get_mm_limits()
149
+ mm_counts = self.get_mm_limits()
150
151
info = self.processing_info
152
mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(seq_len)
vllm/multimodal/registry.py
+14
-5
@@ -17,7 +17,7 @@ from .image import ImagePlugin
17
from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
18
from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
19
ProcessingCache)
20
-from .profiling import BaseDummyInputsBuilder
20
+from .profiling import BaseDummyInputsBuilder, MultiModalProfiler
21
from .utils import cached_get_tokenizer
22
from .video import VideoPlugin
23
@@ -282,13 +282,13 @@ class MultiModalRegistry:
282
This is currently directly used only in V1 for profiling the memory
283
usage of a model.
284
"""
285
- limits_per_plugin = self._limits_by_model[model_config]
285
+ mm_limits = self.get_mm_limits_per_prompt(model_config)
286
287
return {
288
key: max_tokens_per_mm_item
289
for key, max_tokens_per_mm_item in
290
self.get_max_tokens_per_item_by_modality(model_config).items()
291
- if limits_per_plugin[key] > 0
291
+ if mm_limits[key] > 0
292
}
293
294
def get_max_tokens_by_modality(
@@ -304,10 +304,10 @@ class MultiModalRegistry:
304
Note:
305
This should be called after :meth:`init_mm_limits_per_prompt`.
306
"""
307
- limits_per_plugin = self._limits_by_model[model_config]
307
+ mm_limits = self.get_mm_limits_per_prompt(model_config)
308
309
return {
310
- key: limits_per_plugin[key] * max_tokens_per_mm_item
310
+ key: mm_limits[key] * max_tokens_per_mm_item
311
for key, max_tokens_per_mm_item in
312
self.get_max_tokens_per_item_by_modality(model_config).items()
313
}
@@ -371,6 +371,15 @@ class MultiModalRegistry:
371
Note:
372
This should be called after :meth:`init_mm_limits_per_prompt`.
373
"""
374
+ if self.has_processor(model_config):
375
+ tokenizer = cached_get_tokenizer(
376
+ model_config.tokenizer,
377
+ trust_remote_code=model_config.trust_remote_code,
378
+ )
379
+ processor = self.create_processor(model_config, tokenizer)
380
+ profiler = MultiModalProfiler(processor)
381
+ return profiler.get_mm_limits()
382
+
383
return self._limits_by_model[model_config]
384
385
def register_processor(