A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 11:45 UTC
[CPU] Specify required KV cache layout for CPU attention backend (#42740)
Signed-off-by: Tony Lin <tony.lin@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
Tony Lin committed
May 18, 2026 at 17:38 UTC
965d076148326f4511b6b832cbe7d974db74dbe9
1 file changed
+5
vllm/v1/attention/backends/cpu_attn.py
+5
@@ -25,6 +25,7 @@ from vllm.v1.attention.backend import (
25
MultipleOf,
26
)
27
from vllm.v1.attention.backends.utils import (
28
+ KVCacheLayoutType,
29
split_decodes_and_prefills,
30
)
31
from vllm.v1.kv_cache_interface import AttentionSpec, CrossAttentionSpec
@@ -94,6 +95,10 @@ class CPUAttentionBackend(AttentionBackend):
95
) -> tuple[int, ...]:
96
return 2, num_blocks, num_kv_heads, block_size, head_size
97
98
+ @classmethod
99
+ def get_required_kv_cache_layout(cls) -> "KVCacheLayoutType | None":
100
+ return "HND"
101
+
102
@staticmethod
103
def use_cascade_attention(*args, **kwargs) -> bool:
104
return False