A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[Bugfix] Cuda Clean up scales Kvcache fp8/int8_per_token_head (#39224)
Signed-off-by: JartX <sagformas@epdcenter.es> Co-authored-by: Michael Goin <mgoin64@gmail.com>
JartX committed
Apr 8, 2026 at 13:08 UTC
140cbb118698b0f685f2a1ce2c348962636755c6
1 file changed
+7
vllm/v1/worker/gpu_model_runner.py
+7
@@ -5859,6 +5859,13 @@ class GPUModelRunner(
5859
layer.kv_cache = (
5860
torch.tensor([]) if isinstance(kv_cache, torch.Tensor) else []
5861
)
5862
+ # Clean up quantized KV cache scale views
5863
+ # (int8_per_token_head, fp8_per_token_head)
5864
+ if hasattr(layer, "impl"):
5865
+ if hasattr(layer.impl, "_k_scale_cache"):
5866
+ layer.impl._k_scale_cache = None
5867
+ if hasattr(layer.impl, "_v_scale_cache"):
5868
+ layer.impl._v_scale_cache = None
5869
5870
gc.collect()
5871
torch.accelerator.empty_cache()