Fix Stale Encoder Cache After Weight Update (#45093)
Signed-off-by: littlecircle0730 <littlecircle0730@gmail.com>
littlecircle0730 committed
Jun 18, 2026 at 02:32 UTC
08985351f369d3dd6b80bc54ce143ede268e2846
2 files changed
+12
vllm/entrypoints/llm.py
+6
@@ -898,6 +898,12 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
898
def finish_weight_update(self) -> None:
899
"""Finish the current weight update."""
900
self.llm_engine.collective_rpc("finish_weight_update")
901
+ # Invalidate cached state computed with the old weights so it isn't
902
+ # reused for subsequent requests:
903
+ # - prefix cache: KV blocks computed with the old weights
904
+ # - encoder cache: multimodal embeddings keyed only by mm_hash
905
+ self.llm_engine.reset_prefix_cache()
906
+ self.llm_engine.reset_encoder_cache()
907
908
def __repr__(self) -> str:
909
"""Return a transformers-style hierarchical view of the model."""
vllm/v1/engine/async_llm.py
+6
@@ -1109,3 +1109,9 @@ class AsyncLLM(EngineClient):
1109
async def finish_weight_update(self) -> None:
1110
"""Finish the current weight update."""
1111
await self.collective_rpc("finish_weight_update")
1112
+ # Invalidate cached state computed with the old weights so it isn't
1113
+ # reused for subsequent requests:
1114
+ # - prefix cache: KV blocks computed with the old weights
1115
+ # - encoder cache: multimodal embeddings keyed only by mm_hash
1116
+ await self.reset_prefix_cache()
1117
+ await self.reset_encoder_cache()