A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Add reset prefix cache for online serving (#22726)
Signed-off-by: iAmir97 <Amir.balwel@embeddedllm.com> Signed-off-by: iAmir97 <71513472+iAmir97@users.noreply.github.com> Co-authored-by: iAmir97 <Amir.balwel@embeddedllm.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
iAmir97 committed
Aug 14, 2025 at 18:04 UTC
7655dc3e45e65f39eee9755cda5298e7319240f2
2 files changed
+2
vllm/engine/async_llm_engine.py
+1
@@ -1092,6 +1092,7 @@ class AsyncLLMEngine(EngineClient):
1092
self.engine.reset_prefix_cache(device)
1093
1094
async def sleep(self, level: int = 1) -> None:
1095
+ await self.reset_prefix_cache()
1096
self.engine.sleep(level)
1097
1098
async def wake_up(self, tags: Optional[list[str]] = None) -> None:
vllm/v1/engine/async_llm.py
+1
@@ -576,6 +576,7 @@ class AsyncLLM(EngineClient):
576
await self.engine_core.reset_prefix_cache_async()
577
578
async def sleep(self, level: int = 1) -> None:
579
+ await self.reset_prefix_cache()
580
await self.engine_core.sleep_async(level)
581
582
async def wake_up(self, tags: Optional[list[str]] = None) -> None: