A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Add kv_scale input parameter to CPU backend (#3840)
Woosuk Kwon committed
Apr 3, 2024 at 21:33 UTC
498eb5cfa3b482c715a710c548a430ad140d109c
4 files changed
+12
-5
csrc/cpu/attention.cpp
+4
-2
@@ -419,7 +419,8 @@ void paged_attention_v1(torch::Tensor &out, torch::Tensor &query,
419
torch::Tensor &context_lens, int block_size,
420
int max_context_len,
421
const c10::optional<torch::Tensor> &alibi_slopes,
422
- const std::string &kv_cache_dtype) {
422
+ const std::string &kv_cache_dtype, float kv_scale) {
423
+ TORCH_CHECK(kv_scale == 1.0f);
424
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
425
[&] {
426
CPU_KERNEL_GUARD_IN(paged_attention_v1_impl)
@@ -734,7 +735,8 @@ void paged_attention_v2(torch::Tensor &out, torch::Tensor &exp_sums,
735
torch::Tensor &context_lens, int block_size,
736
int max_context_len,
737
const c10::optional<torch::Tensor> &alibi_slopes,
737
- const std::string &kv_cache_dtype) {
738
+ const std::string &kv_cache_dtype, float kv_scale) {
739
+ TORCH_CHECK(kv_scale == 1.0f);
740
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
741
[&] {
742
CPU_KERNEL_GUARD_IN(paged_attention_v2_impl)
csrc/cpu/cache.cpp
+3
-1
@@ -111,7 +111,9 @@ void copy_blocks(std::vector<torch::Tensor> &key_caches,
111
void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
112
torch::Tensor &key_cache, torch::Tensor &value_cache,
113
torch::Tensor &slot_mapping,
114
- const std::string &kv_cache_dtype) {
114
+ const std::string &kv_cache_dtype, float kv_scale) {
115
+ TORCH_CHECK(kv_scale == 1.0f);
116
+
117
int num_tokens = key.size(0);
118
int num_heads = key.size(1);
119
int head_size = key.size(2);
vllm/attention/backends/torch_sdpa.py
+4
-1
@@ -114,6 +114,7 @@ class TorchSDPABackendImpl(AttentionImpl):
114
value: torch.Tensor,
115
kv_cache: Optional[torch.Tensor],
116
attn_metadata: TorchSDPAMetadata,
117
+ kv_scale: float,
118
) -> torch.Tensor:
119
"""Forward pass with torch SDPA and PagedAttention.
120
@@ -138,7 +139,8 @@ class TorchSDPABackendImpl(AttentionImpl):
139
PagedAttention.write_to_paged_cache(key, value, key_cache,
140
value_cache,
141
attn_metadata.slot_mapping,
141
- attn_metadata.kv_cache_dtype)
142
+ attn_metadata.kv_cache_dtype,
143
+ kv_scale)
144
145
if attn_metadata.is_prompt:
146
if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
@@ -199,6 +201,7 @@ class TorchSDPABackendImpl(AttentionImpl):
201
self.num_kv_heads,
202
self.scale,
203
self.alibi_slopes,
204
+ kv_scale,
205
)
206
207
# Reshape the output tensor.
vllm/attention/ops/paged_attn.py
+1
-1
@@ -97,7 +97,7 @@ class PagedAttention:
97
num_kv_heads: int,
98
scale: float,
99
alibi_slopes: Optional[torch.Tensor],
100
- kv_scale,
100
+ kv_scale: float,
101
) -> torch.Tensor:
102
output = torch.empty_like(query)
103