@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 498eb5cfa3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Add kv_scale input parameter to CPU backend (#3840)

Woosuk Kwon committed Apr 3, 2024 at 21:33 UTC 498eb5cfa3b482c715a710c548a430ad140d109c
4 files changed +12 -5
csrc/cpu/attention.cpp
+4 -2
@@ -419,7 +419,8 @@ void paged_attention_v1(torch::Tensor &out, torch::Tensor &query,
419 torch::Tensor &context_lens, int block_size,
420 int max_context_len,
421 const c10::optional<torch::Tensor> &alibi_slopes,
422 - const std::string &kv_cache_dtype) {
422 + const std::string &kv_cache_dtype, float kv_scale) {
423 + TORCH_CHECK(kv_scale == 1.0f);
424 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
425 [&] {
426 CPU_KERNEL_GUARD_IN(paged_attention_v1_impl)
@@ -734,7 +735,8 @@ void paged_attention_v2(torch::Tensor &out, torch::Tensor &exp_sums,
735 torch::Tensor &context_lens, int block_size,
736 int max_context_len,
737 const c10::optional<torch::Tensor> &alibi_slopes,
737 - const std::string &kv_cache_dtype) {
738 + const std::string &kv_cache_dtype, float kv_scale) {
739 + TORCH_CHECK(kv_scale == 1.0f);
740 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
741 [&] {
742 CPU_KERNEL_GUARD_IN(paged_attention_v2_impl)
csrc/cpu/cache.cpp
+3 -1
@@ -111,7 +111,9 @@ void copy_blocks(std::vector<torch::Tensor> &key_caches,
111 void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
112 torch::Tensor &key_cache, torch::Tensor &value_cache,
113 torch::Tensor &slot_mapping,
114 - const std::string &kv_cache_dtype) {
114 + const std::string &kv_cache_dtype, float kv_scale) {
115 + TORCH_CHECK(kv_scale == 1.0f);
116 +
117 int num_tokens = key.size(0);
118 int num_heads = key.size(1);
119 int head_size = key.size(2);
vllm/attention/backends/torch_sdpa.py
+4 -1
@@ -114,6 +114,7 @@ class TorchSDPABackendImpl(AttentionImpl):
114 value: torch.Tensor,
115 kv_cache: Optional[torch.Tensor],
116 attn_metadata: TorchSDPAMetadata,
117 + kv_scale: float,
118 ) -> torch.Tensor:
119 """Forward pass with torch SDPA and PagedAttention.
120
@@ -138,7 +139,8 @@ class TorchSDPABackendImpl(AttentionImpl):
139 PagedAttention.write_to_paged_cache(key, value, key_cache,
140 value_cache,
141 attn_metadata.slot_mapping,
141 - attn_metadata.kv_cache_dtype)
142 + attn_metadata.kv_cache_dtype,
143 + kv_scale)
144
145 if attn_metadata.is_prompt:
146 if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
@@ -199,6 +201,7 @@ class TorchSDPABackendImpl(AttentionImpl):
201 self.num_kv_heads,
202 self.scale,
203 self.alibi_slopes,
204 + kv_scale,
205 )
206
207 # Reshape the output tensor.
vllm/attention/ops/paged_attn.py
+1 -1
@@ -97,7 +97,7 @@ class PagedAttention:
97 num_kv_heads: int,
98 scale: float,
99 alibi_slopes: Optional[torch.Tensor],
100 - kv_scale,
100 + kv_scale: float,
101 ) -> torch.Tensor:
102 output = torch.empty_like(query)
103