A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[Bugfix] Actually disable processing cache when API server is scaled out (#21839)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jul 30, 2025 at 11:36 UTC
44bc46da6008c04d351d8fd0bf026bff8ab57dab
1 file changed
+8
-5
vllm/entrypoints/cli/serve.py
+8
-5
@@ -140,11 +140,16 @@ def run_multi_api_server(args: argparse.Namespace):
140
num_api_servers = args.api_server_count
141
assert num_api_servers > 0
142
143
+ orig_disable_mm_preprocessor_cache = args.disable_mm_preprocessor_cache
144
+
145
# set_process_title("ProcManager")
146
147
if num_api_servers > 1:
148
setup_multiprocess_prometheus()
149
150
+ # Not compatible with API server scale-out
151
+ args.disable_mm_preprocessor_cache = True
152
+
153
listen_address, sock = setup_server(args)
154
155
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
@@ -161,11 +166,9 @@ def run_multi_api_server(args: argparse.Namespace):
166
"with api_server_count > 1")
167
168
if model_config.is_multimodal_model and not (
164
- model_config.disable_mm_preprocessor_cache):
165
- logger.warning(
166
- "Multi-model preprocessor cache will be disabled for"
167
- " api_server_count > 1")
168
- model_config.disable_mm_preprocessor_cache = True
169
+ orig_disable_mm_preprocessor_cache):
170
+ logger.warning("Multi-model preprocessor cache will be disabled "
171
+ "for api_server_count > 1")
172
173
executor_class = Executor.get_class(vllm_config)
174
log_stats = not engine_args.disable_log_stats