@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f136da15e1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Hardware][TPU] Optimize KV cache swapping (#5878)

Woosuk Kwon committed Jun 27, 2024 at 21:12 UTC f136da15e154b25c7eb3221772a85a15811f0318
2 files changed +36 -22
vllm/attention/backends/pallas.py
+4 -12
@@ -28,21 +28,13 @@ class PallasAttentionBackend(AttentionBackend):
28 ) -> Tuple[int, ...]:
29 return (num_kv_heads, num_blocks, block_size, head_size)
30
31 - @torch.compile(backend="openxla")
31 @staticmethod
32 def swap_blocks(
34 - src_kv_cache: Tuple[torch.Tensor, torch.Tensor],
35 - dst_kv_cache: Tuple[torch.Tensor, torch.Tensor],
36 - src_to_dst: Tuple[torch.Tensor, torch.Tensor],
33 + src_kv_cache: torch.Tensor,
34 + dst_kv_cache: torch.Tensor,
35 + src_to_dst: torch.Tensor,
36 ) -> None:
38 - src_k_cache, src_v_cache = src_kv_cache
39 - dst_k_cache, dst_v_cache = dst_kv_cache
40 - src_indices, dst_indices = src_to_dst
41 - device = dst_k_cache.device
42 - torch.ops.xla.dynamo_set_buffer_donor_(dst_k_cache, True)
43 - torch.ops.xla.dynamo_set_buffer_donor_(dst_v_cache, True)
44 - dst_k_cache[:, dst_indices] = src_k_cache[:, src_indices].to(device)
45 - dst_v_cache[:, dst_indices] = src_v_cache[:, src_indices].to(device)
37 + raise RuntimeError("swap_blocks is not used for the TPU backend.")
38
39 @torch.compile(backend="openxla")
40 @staticmethod
vllm/worker/tpu_worker.py
+32 -10
@@ -3,6 +3,7 @@ from typing import List, Optional, Tuple, Union
3
4 import torch
5 import torch_xla.core.xla_model as xm
6 +import torch_xla.experimental.dynamo_set_buffer_donor # noqa: F401
7 import torch_xla.runtime as xr
8
9 import vllm.envs as envs
@@ -152,8 +153,8 @@ class TPUWorker(LoraNotSupportedWorkerBase):
153 num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
154 head_size = self.model_config.get_head_size()
155
155 - self.cpu_cache = []
156 - self.tpu_cache = []
156 + self.cpu_cache: List[Tuple[torch.Tensor, torch.Tensor]] = []
157 + self.tpu_cache: List[Tuple[torch.Tensor, torch.Tensor]] = []
158 tpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
159 num_gpu_blocks, self.block_size, num_kv_heads, head_size)
160 cpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
@@ -227,18 +228,25 @@ class TPUWorker(LoraNotSupportedWorkerBase):
228
229 if blocks_to_swap_in:
230 # Swap from CPU to TPU.
230 - src_to_dst = _make_src_to_dst(blocks_to_swap_in, "cpu",
231 - self.device)
231 + src_indices, dst_indices = _make_src_to_dst(
232 + blocks_to_swap_in, "cpu", self.device)
233 for i in range(num_layers):
233 - attn_backend.swap_blocks(self.cpu_cache[i], self.tpu_cache[i],
234 - src_to_dst)
234 + tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
235 + cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
236 + k = cpu_k_cache[:, src_indices].to(self.device)
237 + v = cpu_v_cache[:, src_indices].to(self.device)
238 + _insert_kv(k, v, dst_indices, tpu_k_cache, tpu_v_cache)
239 +
240 if blocks_to_swap_out:
241 # Swap from TPU to CPU.
237 - src_to_dst = _make_src_to_dst(blocks_to_swap_out, self.device,
238 - "cpu")
242 + src_indices, dst_indices = _make_src_to_dst(
243 + blocks_to_swap_out, self.device, "cpu")
244 for i in range(num_layers):
240 - attn_backend.swap_blocks(self.tpu_cache[i], self.cpu_cache[i],
241 - src_to_dst)
245 + tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
246 + cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
247 + cpu_k_cache[:, dst_indices] = tpu_k_cache[:, src_indices].cpu()
248 + cpu_v_cache[:, dst_indices] = tpu_v_cache[:, src_indices].cpu()
249 +
250 if blocks_to_copy:
251 src_to_dst = _make_src_to_dst(blocks_to_copy, self.device,
252 self.device)
@@ -267,3 +275,17 @@ def _make_src_to_dst(
275 device=dst_device,
276 dtype=torch.int64)
277 return src_indices, dst_indices
278 +
279 +
280 +@torch.compile(backend="openxla")
281 +def _insert_kv(
282 + k: torch.Tensor,
283 + v: torch.Tensor,
284 + indices: torch.Tensor,
285 + tpu_k_cache: torch.Tensor,
286 + tpu_v_cache: torch.Tensor,
287 +) -> None:
288 + torch.ops.xla.dynamo_set_buffer_donor_(tpu_k_cache, True)
289 + torch.ops.xla.dynamo_set_buffer_donor_(tpu_v_cache, True)
290 + tpu_k_cache[:, indices] = k
291 + tpu_v_cache[:, indices] = v