@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a94eee4456
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix mm_limits access for merged multi-modal processor (#12252)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 21, 2025 at 18:09 UTC a94eee4456b05458bafacc17377de4701ac598a0
2 files changed +16 -7
vllm/multimodal/profiling.py
+2 -2
@@ -106,7 +106,7 @@ class MultiModalProfiler(Generic[_I]):
106 def dummy_inputs(self) -> BaseDummyInputsBuilder[_I]:
107 return self.processor.dummy_inputs
108
109 - def _get_mm_limits(self) -> Mapping[str, int]:
109 + def get_mm_limits(self) -> Mapping[str, int]:
110 mm_config = self.processing_info.ctx.get_mm_config()
111 mm_limit_per_prompt = mm_config.limit_per_prompt
112
@@ -146,7 +146,7 @@ class MultiModalProfiler(Generic[_I]):
146 # Avoid circular import
147 from vllm.sequence import SequenceData
148
149 - mm_counts = self._get_mm_limits()
149 + mm_counts = self.get_mm_limits()
150
151 info = self.processing_info
152 mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(seq_len)
vllm/multimodal/registry.py
+14 -5
@@ -17,7 +17,7 @@ from .image import ImagePlugin
17 from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
18 from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
19 ProcessingCache)
20 -from .profiling import BaseDummyInputsBuilder
20 +from .profiling import BaseDummyInputsBuilder, MultiModalProfiler
21 from .utils import cached_get_tokenizer
22 from .video import VideoPlugin
23
@@ -282,13 +282,13 @@ class MultiModalRegistry:
282 This is currently directly used only in V1 for profiling the memory
283 usage of a model.
284 """
285 - limits_per_plugin = self._limits_by_model[model_config]
285 + mm_limits = self.get_mm_limits_per_prompt(model_config)
286
287 return {
288 key: max_tokens_per_mm_item
289 for key, max_tokens_per_mm_item in
290 self.get_max_tokens_per_item_by_modality(model_config).items()
291 - if limits_per_plugin[key] > 0
291 + if mm_limits[key] > 0
292 }
293
294 def get_max_tokens_by_modality(
@@ -304,10 +304,10 @@ class MultiModalRegistry:
304 Note:
305 This should be called after :meth:`init_mm_limits_per_prompt`.
306 """
307 - limits_per_plugin = self._limits_by_model[model_config]
307 + mm_limits = self.get_mm_limits_per_prompt(model_config)
308
309 return {
310 - key: limits_per_plugin[key] * max_tokens_per_mm_item
310 + key: mm_limits[key] * max_tokens_per_mm_item
311 for key, max_tokens_per_mm_item in
312 self.get_max_tokens_per_item_by_modality(model_config).items()
313 }
@@ -371,6 +371,15 @@ class MultiModalRegistry:
371 Note:
372 This should be called after :meth:`init_mm_limits_per_prompt`.
373 """
374 + if self.has_processor(model_config):
375 + tokenizer = cached_get_tokenizer(
376 + model_config.tokenizer,
377 + trust_remote_code=model_config.trust_remote_code,
378 + )
379 + processor = self.create_processor(model_config, tokenizer)
380 + profiler = MultiModalProfiler(processor)
381 + return profiler.get_mm_limits()
382 +
383 return self._limits_by_model[model_config]
384
385 def register_processor(