A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 05:00 UTC
[Bugfix] Fix inconsistent handling of cache reset (#33481)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 1, 2026 at 12:23 UTC
79b6ec6aab4b3d93e421a15ab41ba6d09faadd8f
7 files changed
+35
-11
docs/benchmarking/sweeps.md
+1
-1
@@ -82,7 +82,7 @@ vllm bench sweep serve \
82
You can use `--dry-run` to preview the commands to be run.
83
84
We only start the server once for each `--serve-params`, and keep it running for multiple `--bench-params`.
85
- Between each benchmark run, we call the `/reset_prefix_cache` and `/reset_mm_cache` endpoints to get a clean slate for the next run.
85
+ Between each benchmark run, we call all `/reset_*_cache` endpoints to get a clean slate for the next run.
86
In case you are using a custom `--serve-cmd`, you can override the commands used for resetting the state by setting `--after-bench-cmd`.
87
88
!!! note
vllm/benchmarks/sweep/server.py
+9
-5
@@ -12,6 +12,12 @@ from typing_extensions import Self
12
13
14
class ServerProcess:
15
+ VLLM_RESET_CACHE_ENDPOINTS = [
16
+ "/reset_prefix_cache",
17
+ "/reset_mm_cache",
18
+ "/reset_encoder_cache",
19
+ ]
20
+
21
def __init__(
22
self,
23
server_cmd: list[str],
@@ -120,11 +126,9 @@ class ServerProcess:
126
server_address = self._get_vllm_server_address()
127
print(f"Resetting caches at {server_address}")
128
123
- res = requests.post(f"{server_address}/reset_prefix_cache")
124
- res.raise_for_status()
125
-
126
- res = requests.post(f"{server_address}/reset_mm_cache")
127
- res.raise_for_status()
129
+ for endpoint in self.VLLM_RESET_CACHE_ENDPOINTS:
130
+ res = requests.post(server_address + endpoint)
131
+ res.raise_for_status()
132
elif server_cmd[0].endswith("infinity_emb"):
133
if "--vector-disk-cache" in server_cmd:
134
raise NotImplementedError(
vllm/entrypoints/openai/engine/serving.py
-4
@@ -286,10 +286,6 @@ class OpenAIServing:
286
raise TypeError(f"{reasoning_parser_name=} has not been registered") from e
287
return parser
288
289
- async def reset_mm_cache(self) -> None:
290
- self.input_processor.clear_mm_cache()
291
- await self.engine_client.reset_mm_cache()
292
-
289
async def beam_search(
290
self,
291
prompt: PromptType,
vllm/v1/engine/async_llm.py
+1
@@ -741,6 +741,7 @@ class AsyncLLM(EngineClient):
741
if clear_cache:
742
await self.reset_prefix_cache()
743
await self.reset_mm_cache()
744
+ await self.reset_encoder_cache()
745
746
async def resume_generation(self) -> None:
747
"""Resume generation after :meth:`pause_generation`."""
vllm/v1/worker/gpu/mm/encoder_runner.py
+16
@@ -31,6 +31,22 @@ class EncoderRunner:
31
self.req_id_to_mm_features: dict[str, list[MultiModalFeatureSpec]] = {}
32
self.encoder_cache: dict[str, torch.Tensor] = {}
33
34
+ def reset_mm_cache(self) -> None:
35
+ """
36
+ Clear the multi-modal cache that was used during profiling,
37
+ but no longer needed during inference.
38
+ """
39
+ # TODO: Implement MM budget for encoder dummy run
40
+ pass
41
+
42
+ def reset_encoder_cache(self) -> None:
43
+ """Clear the GPU-side encoder cache storing vision embeddings.
44
+
45
+ This should be called when model weights are updated to ensure
46
+ stale embeddings computed with old weights are not reused.
47
+ """
48
+ self.encoder_cache.clear()
49
+
50
def add_request(self, req_id: str, mm_features: list[MultiModalFeatureSpec]):
51
self.req_id_to_mm_features[req_id] = mm_features
52
vllm/v1/worker/gpu/model_runner.py
+4
-1
@@ -339,7 +339,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
339
gc.collect()
340
341
def reset_mm_cache(self) -> None:
342
- pass
342
+ self.encoder_runner.reset_mm_cache()
343
+
344
+ def reset_encoder_cache(self) -> None:
345
+ self.encoder_runner.reset_encoder_cache()
346
347
def _get_num_input_tokens(self, num_scheduled_tokens: int) -> int:
348
# SP is not supported yet.
vllm/v1/worker/gpu_model_runner.py
+4
@@ -717,6 +717,10 @@ class GPUModelRunner(
717
self.effective_drafter_max_model_len = self.max_model_len
718
719
def reset_mm_cache(self) -> None:
720
+ """
721
+ Clear the multi-modal cache that was used during profiling,
722
+ but no longer needed during inference.
723
+ """
724
if self.mm_budget:
725
self.mm_budget.reset_cache()
726