A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix][TPU] Fix CPU cache allocation (#5869)
Woosuk Kwon committed
Jun 26, 2024 at 13:42 UTC
f5c8628fdc78bf9ca70206ef41175030fb67e870
2 files changed
+8
-5
vllm/attention/backends/pallas.py
+2
-3
@@ -37,11 +37,10 @@ class PallasAttentionBackend(AttentionBackend):
37
) -> None:
38
src_k_cache, src_v_cache = src_kv_cache
39
dst_k_cache, dst_v_cache = dst_kv_cache
40
+ src_indices, dst_indices = src_to_dst
41
+ device = dst_k_cache.device
42
torch.ops.xla.dynamo_set_buffer_donor_(dst_k_cache, True)
43
torch.ops.xla.dynamo_set_buffer_donor_(dst_v_cache, True)
42
-
43
- device = dst_k_cache.device
44
- src_indices, dst_indices = src_to_dst
44
dst_k_cache[:, dst_indices] = src_k_cache[:, src_indices].to(device)
45
dst_v_cache[:, dst_indices] = src_v_cache[:, src_indices].to(device)
46
vllm/worker/tpu_worker.py
+6
-2
@@ -156,14 +156,18 @@ class TPUWorker(LoraNotSupportedWorkerBase):
156
self.tpu_cache = []
157
tpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
158
num_gpu_blocks, self.block_size, num_kv_heads, head_size)
159
+ cpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
160
+ num_cpu_blocks, self.block_size, num_kv_heads, head_size)
161
for _ in range(num_layers):
162
tpu_k_cache = torch.zeros(tpu_cache_shape,
163
dtype=dtype,
164
device=self.device)
165
tpu_v_cache = torch.zeros_like(tpu_k_cache)
166
self.tpu_cache.append((tpu_k_cache, tpu_v_cache))
165
- cpu_k_cache = torch.zeros_like(tpu_k_cache, device="cpu")
166
- cpu_v_cache = torch.zeros_like(tpu_v_cache, device="cpu")
167
+ cpu_k_cache = torch.zeros(cpu_cache_shape,
168
+ dtype=dtype,
169
+ device="cpu")
170
+ cpu_v_cache = torch.zeros_like(cpu_k_cache)
171
self.cpu_cache.append((cpu_k_cache, cpu_v_cache))
172
self._warmup_model()
173