A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[Bugfix] Disable multi-modal preprocessor cache for DP (#21896)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 1, 2025 at 15:03 UTC
b4e081cb150797b12039cc1232205dbb25ca0206
3 files changed
+21
-2
vllm/config.py
+6
@@ -871,6 +871,12 @@ class ModelConfig:
871
872
return None
873
874
+ def set_disable_mm_preprocessor_cache(self, value: bool) -> None:
875
+ mm_config = self.get_multimodal_config()
876
+
877
+ self.disable_mm_preprocessor_cache = value
878
+ mm_config.disable_mm_preprocessor_cache = value
879
+
880
def _get_encoder_config(self):
881
return get_sentence_transformer_tokenizer_config(
882
self.model, self.revision)
vllm/engine/arg_utils.py
+12
@@ -1197,6 +1197,18 @@ class EngineArgs:
1197
enable_multimodal_encoder_data_parallel,
1198
)
1199
1200
+ supports_mm_preprocessor_cache = (self.data_parallel_size == 1
1201
+ or data_parallel_external_lb)
1202
+ if (not supports_mm_preprocessor_cache
1203
+ and model_config.is_multimodal_model
1204
+ and not model_config.disable_mm_preprocessor_cache):
1205
+ logger.warning(
1206
+ "Multi-modal preprocessor cache is not compatible "
1207
+ "with data parallelism when there does not exist a "
1208
+ "one-to-one correspondance between API process and "
1209
+ "EngineCore process, so the cache will be disabled.")
1210
+ model_config.set_disable_mm_preprocessor_cache(True)
1211
+
1212
speculative_config = self.create_speculative_config(
1213
target_model_config=model_config,
1214
target_parallel_config=parallel_config,
vllm/entrypoints/cli/serve.py
+3
-2
@@ -167,8 +167,9 @@ def run_multi_api_server(args: argparse.Namespace):
167
168
if model_config.is_multimodal_model and not (
169
orig_disable_mm_preprocessor_cache):
170
- logger.warning("Multi-model preprocessor cache will be disabled "
171
- "for api_server_count > 1")
170
+ logger.warning(
171
+ "Multi-modal preprocessor cache is not compatible "
172
+ "with api_server_count > 1, so the cache will be disabled.")
173
174
executor_class = Executor.get_class(vllm_config)
175
log_stats = not engine_args.disable_log_stats