A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Bugfix] Fix entrypoints metrics tests (#18063)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 13, 2025 at 21:42 UTC
b922c2ebd27159a5f6247aa557ed5fc12d864660
2 files changed
+5
-5
vllm/entrypoints/openai/api_server.py
+4
-4
@@ -150,10 +150,6 @@ async def build_async_engine_client(
150
151
async with build_async_engine_client_from_engine_args(
152
engine_args, args.disable_frontend_multiprocessing) as engine:
153
-
154
- # Don't keep the dummy data in memory
155
- await engine.reset_mm_cache()
156
-
153
yield engine
154
155
@@ -189,6 +185,10 @@ async def build_async_engine_client_from_engine_args(
185
usage_context=usage_context,
186
disable_log_requests=engine_args.disable_log_requests,
187
disable_log_stats=engine_args.disable_log_stats)
188
+
189
+ # Don't keep the dummy data in memory
190
+ await async_llm.reset_mm_cache()
191
+
192
yield async_llm
193
finally:
194
if async_llm:
vllm/v1/engine/core.py
+1
-1
@@ -289,7 +289,7 @@ class EngineCore:
289
def reset_mm_cache(self):
290
# NOTE: Since this is mainly for debugging, we don't attempt to
291
# re-sync the internal caches (P0 processor, P0 mirror, P1 mirror)
292
- if self.scheduler.get_num_unfinished_requests():
292
+ if self.scheduler.has_unfinished_requests():
293
logger.warning("Resetting the multi-modal cache when requests are "
294
"in progress may lead to desynced internal caches.")
295