A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Core] Don't use cache during multi-modal profiling (#14336)
Cyrus Leung committed
Mar 7, 2025 at 00:03 UTC
82551ad616507de1cda0afb6babeec3f56f189eb
2 files changed
+15
-5
vllm/inputs/registry.py
+3
-1
@@ -331,7 +331,9 @@ class InputRegistry:
331
332
if mm_registry.has_processor(model_config):
333
tokenizer = cached_tokenizer_from_config(model_config)
334
- processor = mm_registry.create_processor(model_config, tokenizer)
334
+ processor = mm_registry.create_processor(model_config,
335
+ tokenizer,
336
+ disable_cache=True)
337
profiler = MultiModalProfiler(processor)
338
dummy_data = profiler.get_dummy_data(
339
seq_len, is_encoder_data=is_encoder_data)
vllm/multimodal/registry.py
+12
-4
@@ -257,7 +257,9 @@ class MultiModalRegistry:
257
"""
258
if self.has_processor(model_config):
259
tokenizer = cached_tokenizer_from_config(model_config)
260
- processor = self.create_processor(model_config, tokenizer)
260
+ processor = self.create_processor(model_config,
261
+ tokenizer,
262
+ disable_cache=True)
263
seq_len = model_config.max_model_len
264
mm_limits = self.get_mm_limits_per_prompt(model_config)
265
return processor.info.get_mm_max_tokens_per_item(
@@ -372,7 +374,9 @@ class MultiModalRegistry:
374
"""
375
if self.has_processor(model_config):
376
tokenizer = cached_tokenizer_from_config(model_config)
375
- processor = self.create_processor(model_config, tokenizer)
377
+ processor = self.create_processor(model_config,
378
+ tokenizer,
379
+ disable_cache=True)
380
profiler = MultiModalProfiler(processor)
381
return profiler.get_mm_limits()
382
@@ -433,6 +437,8 @@ class MultiModalRegistry:
437
self,
438
model_config: "ModelConfig",
439
tokenizer: AnyTokenizer,
440
+ *,
441
+ disable_cache: Optional[bool] = None,
442
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
443
"""
444
Create a multi-modal processor for a specific model and tokenizer.
@@ -440,11 +446,13 @@ class MultiModalRegistry:
446
See also:
447
:ref:`mm-processing`
448
"""
449
+ if disable_cache is None:
450
+ disable_cache = model_config.disable_mm_preprocessor_cache
451
+
452
model_cls = self._get_model_cls(model_config)
453
factories = self._processor_factories[model_cls]
454
455
ctx = InputProcessingContext(model_config, tokenizer)
447
- cache = (None if model_config.disable_mm_preprocessor_cache else
448
- self._processing_cache)
456
+ cache = None if disable_cache else self._processing_cache
457
458
return factories.build_processor(ctx, cache=cache)