@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f5c8628fdc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix][TPU] Fix CPU cache allocation (#5869)

Woosuk Kwon committed Jun 26, 2024 at 13:42 UTC f5c8628fdc78bf9ca70206ef41175030fb67e870
2 files changed +8 -5
vllm/attention/backends/pallas.py
+2 -3
@@ -37,11 +37,10 @@ class PallasAttentionBackend(AttentionBackend):
37 ) -> None:
38 src_k_cache, src_v_cache = src_kv_cache
39 dst_k_cache, dst_v_cache = dst_kv_cache
40 + src_indices, dst_indices = src_to_dst
41 + device = dst_k_cache.device
42 torch.ops.xla.dynamo_set_buffer_donor_(dst_k_cache, True)
43 torch.ops.xla.dynamo_set_buffer_donor_(dst_v_cache, True)
42 -
43 - device = dst_k_cache.device
44 - src_indices, dst_indices = src_to_dst
44 dst_k_cache[:, dst_indices] = src_k_cache[:, src_indices].to(device)
45 dst_v_cache[:, dst_indices] = src_v_cache[:, src_indices].to(device)
46
vllm/worker/tpu_worker.py
+6 -2
@@ -156,14 +156,18 @@ class TPUWorker(LoraNotSupportedWorkerBase):
156 self.tpu_cache = []
157 tpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
158 num_gpu_blocks, self.block_size, num_kv_heads, head_size)
159 + cpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
160 + num_cpu_blocks, self.block_size, num_kv_heads, head_size)
161 for _ in range(num_layers):
162 tpu_k_cache = torch.zeros(tpu_cache_shape,
163 dtype=dtype,
164 device=self.device)
165 tpu_v_cache = torch.zeros_like(tpu_k_cache)
166 self.tpu_cache.append((tpu_k_cache, tpu_v_cache))
165 - cpu_k_cache = torch.zeros_like(tpu_k_cache, device="cpu")
166 - cpu_v_cache = torch.zeros_like(tpu_v_cache, device="cpu")
167 + cpu_k_cache = torch.zeros(cpu_cache_shape,
168 + dtype=dtype,
169 + device="cpu")
170 + cpu_v_cache = torch.zeros_like(cpu_k_cache)
171 self.cpu_cache.append((cpu_k_cache, cpu_v_cache))
172 self._warmup_model()
173