A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix cache engine
Woosuk Kwon committed
Feb 13, 2023 at 09:35 UTC
bb59a3e7302ad6892e097eee4040e3f516e9f4ea
1 file changed
+15
-34
cacheflow/worker/cache_engine.py
+15
-34
@@ -1,4 +1,4 @@
1
-from typing import List, Tuple
1
+from typing import Dict, List, Tuple
2
3
import torch
4
@@ -14,34 +14,30 @@ class CacheEngine:
14
num_layers: int,
15
num_heads: int,
16
head_size: int,
17
+ block_size: int,
18
num_gpu_blocks: int,
19
num_cpu_blocks: int,
19
- block_size: int,
20
- dtype: torch.dtype = torch.float16,
20
+ dtype: torch.dtype,
21
) -> None:
22
self.worker_id = worker_id
23
self.gpu_id = gpu_id
24
self.num_layers = num_layers
25
self.num_heads = num_heads
26
self.head_size = head_size
27
+ self.block_size = block_size
28
self.num_gpu_blocks = num_gpu_blocks
29
self.num_cpu_blocks = num_cpu_blocks
29
- self.block_size = block_size
30
self.dtype = dtype
31
32
# Initialize the cache.
33
self.gpu_cache = self.allocate_gpu_cache()
34
self.cpu_cache = self.allocate_cpu_cache()
35
36
- # Initialize the streams.
37
- self.copy_stream = torch.cuda.Stream(device=gpu_id)
38
- self.swap_stream = torch.cuda.Stream(device=gpu_id)
39
- assert self.copy_stream != self.swap_stream
40
- current_stream = torch.cuda.current_stream(device=gpu_id)
41
- assert self.copy_stream != current_stream
42
- assert self.swap_stream != current_stream
43
-
44
- # Initialize the events for synchronization.
36
+ # Initialize the stream for caching operations.
37
+ self.cache_stream = torch.cuda.Stream(device=gpu_id)
38
+ assert self.cache_stream != torch.cuda.current_stream(device=gpu_id)
39
+ # Initialize the events for stream synchronization.
40
+ self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
41
42
def allocate_gpu_cache(self) -> List[List[KVCache]]:
43
gpu_cache: List[List[KVCache]] = []
@@ -81,29 +77,14 @@ class CacheEngine:
77
cpu_cache.append(layer_cache)
78
return cpu_cache
79
84
- def copy(
85
- self,
86
- src_block_numbers: List[int],
87
- dst_block_numbers: List[int],
88
- ) -> None:
89
- for layer in range(self.num_layers):
90
- # TODO: Call the COPY op.
80
+ def copy(self, src_to_dst: Dict[int, int]) -> None:
81
+ for event in self.events:
82
pass
83
93
- def swap_out(
94
- self,
95
- gpu_block_numbers: List[int],
96
- cpu_block_numbers: List[int],
97
- ) -> None:
98
- for layer in range(self.num_layers):
99
- # TODO: Call the SWAP_OUT op on the swap stream.
84
+ def swap_in(self, src_to_dst: Dict[int, int]) -> None:
85
+ for event in self.events:
86
pass
87
102
- def swap_in(
103
- self,
104
- gpu_block_numbers: List[int],
105
- cpu_block_numbers: List[int],
106
- ) -> None:
107
- for layer in range(self.num_layers):
108
- # TODO: Call the SWAP_IN op on the swap stream.
88
+ def swap_out(self, src_to_dst: Dict[int, int]) -> None:
89
+ for event in self.events:
90
pass