@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f4887de77
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix KVCache shape

Woosuk Kwon committed Feb 16, 2023 at 01:24 UTC 2f4887de77fc36ec27f6b2e8d4cd52c9cf02efed
1 file changed +43 -32
cacheflow/worker/cache_engine.py
+43 -32
@@ -39,42 +39,53 @@ class CacheEngine:
39 # Initialize the events for stream synchronization.
40 self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
41
42 - def allocate_gpu_cache(self) -> List[List[KVCache]]:
43 - gpu_cache: List[List[KVCache]] = []
42 + def get_key_block_shape(self) -> Tuple[int, int, int, int, int]:
43 + element_size = torch.tensor([], dtype=self.dtype).element_size()
44 + x = 16 // element_size
45 + return (
46 + self.num_heads,
47 + self.head_size // x,
48 + self.block_size,
49 + x,
50 + )
51 +
52 + def get_value_block_shape(self) -> Tuple[int, int, int, int]:
53 + return (
54 + self.num_heads,
55 + self.block_size,
56 + self.head_size,
57 + )
58 +
59 + def allocate_gpu_cache(self) -> List[KVCache]:
60 + gpu_cache: List[KVCache] = []
61 for _ in range(self.num_layers):
45 - layer_cache: List[KVCache] = []
46 - for _ in range(self.num_heads):
47 - key_blocks = torch.empty(
48 - (self.num_gpu_blocks, self.block_size * self.head_size),
49 - dtype=self.dtype,
50 - device=self.gpu_id,
51 - )
52 - value_blocks = torch.empty(
53 - (self.num_gpu_blocks, self.block_size * self.head_size),
54 - dtype=self.dtype,
55 - device=self.gpu_id,
56 - )
57 - layer_cache.append((key_blocks, value_blocks))
58 - gpu_cache.append(layer_cache)
62 + key_blocks = torch.empty(
63 + size=(self.num_gpu_blocks, *self.get_key_block_shape()),
64 + dtype=self.dtype,
65 + device=self.gpu_id,
66 + )
67 + value_blocks = torch.empty(
68 + size=(self.num_gpu_blocks, *self.get_value_block_shape()),
69 + dtype=self.dtype,
70 + device=self.gpu_id,
71 + )
72 + gpu_cache.append((key_blocks, value_blocks))
73 return gpu_cache
74
61 - def allocate_cpu_cache(self) -> List[List[KVCache]]:
62 - cpu_cache: List[List[KVCache]] = []
75 + def allocate_cpu_cache(self) -> List[KVCache]:
76 + cpu_cache: List[KVCache] = []
77 for _ in range(self.num_layers):
64 - layer_cache: List[KVCache] = []
65 - for _ in range(self.num_heads):
66 - key_blocks = torch.empty(
67 - (self.num_cpu_blocks, self.block_size * self.head_size),
68 - dtype=self.dtype,
69 - pin_memory=True,
70 - )
71 - value_blocks = torch.empty(
72 - (self.num_cpu_blocks, self.block_size * self.head_size),
73 - dtype=self.dtype,
74 - pin_memory=True,
75 - )
76 - layer_cache.append((key_blocks, value_blocks))
77 - cpu_cache.append(layer_cache)
78 + key_blocks = torch.empty(
79 + size=(self.num_cpu_blocks, *self.get_key_block_shape()),
80 + dtype=self.dtype,
81 + pin_memory=True,
82 + )
83 + value_blocks = torch.empty(
84 + size=(self.num_cpu_blocks, *self.get_value_block_shape()),
85 + dtype=self.dtype,
86 + pin_memory=True,
87 + )
88 + cpu_cache.append((key_blocks, value_blocks))
89 return cpu_cache
90
91 def copy(self, src_to_dst: Dict[int, int]) -> None: