A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix KVCache shape
Woosuk Kwon committed
Feb 16, 2023 at 01:24 UTC
2f4887de77fc36ec27f6b2e8d4cd52c9cf02efed
1 file changed
+43
-32
cacheflow/worker/cache_engine.py
+43
-32
@@ -39,42 +39,53 @@ class CacheEngine:
39
# Initialize the events for stream synchronization.
40
self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
41
42
- def allocate_gpu_cache(self) -> List[List[KVCache]]:
43
- gpu_cache: List[List[KVCache]] = []
42
+ def get_key_block_shape(self) -> Tuple[int, int, int, int, int]:
43
+ element_size = torch.tensor([], dtype=self.dtype).element_size()
44
+ x = 16 // element_size
45
+ return (
46
+ self.num_heads,
47
+ self.head_size // x,
48
+ self.block_size,
49
+ x,
50
+ )
51
+
52
+ def get_value_block_shape(self) -> Tuple[int, int, int, int]:
53
+ return (
54
+ self.num_heads,
55
+ self.block_size,
56
+ self.head_size,
57
+ )
58
+
59
+ def allocate_gpu_cache(self) -> List[KVCache]:
60
+ gpu_cache: List[KVCache] = []
61
for _ in range(self.num_layers):
45
- layer_cache: List[KVCache] = []
46
- for _ in range(self.num_heads):
47
- key_blocks = torch.empty(
48
- (self.num_gpu_blocks, self.block_size * self.head_size),
49
- dtype=self.dtype,
50
- device=self.gpu_id,
51
- )
52
- value_blocks = torch.empty(
53
- (self.num_gpu_blocks, self.block_size * self.head_size),
54
- dtype=self.dtype,
55
- device=self.gpu_id,
56
- )
57
- layer_cache.append((key_blocks, value_blocks))
58
- gpu_cache.append(layer_cache)
62
+ key_blocks = torch.empty(
63
+ size=(self.num_gpu_blocks, *self.get_key_block_shape()),
64
+ dtype=self.dtype,
65
+ device=self.gpu_id,
66
+ )
67
+ value_blocks = torch.empty(
68
+ size=(self.num_gpu_blocks, *self.get_value_block_shape()),
69
+ dtype=self.dtype,
70
+ device=self.gpu_id,
71
+ )
72
+ gpu_cache.append((key_blocks, value_blocks))
73
return gpu_cache
74
61
- def allocate_cpu_cache(self) -> List[List[KVCache]]:
62
- cpu_cache: List[List[KVCache]] = []
75
+ def allocate_cpu_cache(self) -> List[KVCache]:
76
+ cpu_cache: List[KVCache] = []
77
for _ in range(self.num_layers):
64
- layer_cache: List[KVCache] = []
65
- for _ in range(self.num_heads):
66
- key_blocks = torch.empty(
67
- (self.num_cpu_blocks, self.block_size * self.head_size),
68
- dtype=self.dtype,
69
- pin_memory=True,
70
- )
71
- value_blocks = torch.empty(
72
- (self.num_cpu_blocks, self.block_size * self.head_size),
73
- dtype=self.dtype,
74
- pin_memory=True,
75
- )
76
- layer_cache.append((key_blocks, value_blocks))
77
- cpu_cache.append(layer_cache)
78
+ key_blocks = torch.empty(
79
+ size=(self.num_cpu_blocks, *self.get_key_block_shape()),
80
+ dtype=self.dtype,
81
+ pin_memory=True,
82
+ )
83
+ value_blocks = torch.empty(
84
+ size=(self.num_cpu_blocks, *self.get_value_block_shape()),
85
+ dtype=self.dtype,
86
+ pin_memory=True,
87
+ )
88
+ cpu_cache.append((key_blocks, value_blocks))
89
return cpu_cache
90
91
def copy(self, src_to_dst: Dict[int, int]) -> None: