A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 02:45 UTC
mla: don't update kv cache on dummy forwards (#36282)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
Itay Alroy committed
Mar 7, 2026 at 02:36 UTC
24a03915f525b88ebc4c36127c3e9ccf56dc21ee
1 file changed
+4
vllm/model_executor/layers/attention/mla_attention.py
+4
@@ -905,6 +905,10 @@ def unified_mla_kv_cache_update(
905
the data dependency between them to ensure torch.compile preserves ordering.
906
"""
907
forward_context = get_forward_context()
908
+ if forward_context.attn_metadata is None:
909
+ # Dummy/profile forwards should not update live KV cache pages.
910
+ return torch.empty(0, device=kv_c_normed.device, dtype=kv_c_normed.dtype)
911
+
912
attn_layer = forward_context.no_compile_layers[layer_name]
913
kv_cache = attn_layer.kv_cache[forward_context.virtual_engine]
914