@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb59a3e730
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix cache engine

Woosuk Kwon committed Feb 13, 2023 at 09:35 UTC bb59a3e7302ad6892e097eee4040e3f516e9f4ea
1 file changed +15 -34
cacheflow/worker/cache_engine.py
+15 -34
@@ -1,4 +1,4 @@
1 -from typing import List, Tuple
1 +from typing import Dict, List, Tuple
2
3 import torch
4
@@ -14,34 +14,30 @@ class CacheEngine:
14 num_layers: int,
15 num_heads: int,
16 head_size: int,
17 + block_size: int,
18 num_gpu_blocks: int,
19 num_cpu_blocks: int,
19 - block_size: int,
20 - dtype: torch.dtype = torch.float16,
20 + dtype: torch.dtype,
21 ) -> None:
22 self.worker_id = worker_id
23 self.gpu_id = gpu_id
24 self.num_layers = num_layers
25 self.num_heads = num_heads
26 self.head_size = head_size
27 + self.block_size = block_size
28 self.num_gpu_blocks = num_gpu_blocks
29 self.num_cpu_blocks = num_cpu_blocks
29 - self.block_size = block_size
30 self.dtype = dtype
31
32 # Initialize the cache.
33 self.gpu_cache = self.allocate_gpu_cache()
34 self.cpu_cache = self.allocate_cpu_cache()
35
36 - # Initialize the streams.
37 - self.copy_stream = torch.cuda.Stream(device=gpu_id)
38 - self.swap_stream = torch.cuda.Stream(device=gpu_id)
39 - assert self.copy_stream != self.swap_stream
40 - current_stream = torch.cuda.current_stream(device=gpu_id)
41 - assert self.copy_stream != current_stream
42 - assert self.swap_stream != current_stream
43 -
44 - # Initialize the events for synchronization.
36 + # Initialize the stream for caching operations.
37 + self.cache_stream = torch.cuda.Stream(device=gpu_id)
38 + assert self.cache_stream != torch.cuda.current_stream(device=gpu_id)
39 + # Initialize the events for stream synchronization.
40 + self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
41
42 def allocate_gpu_cache(self) -> List[List[KVCache]]:
43 gpu_cache: List[List[KVCache]] = []
@@ -81,29 +77,14 @@ class CacheEngine:
77 cpu_cache.append(layer_cache)
78 return cpu_cache
79
84 - def copy(
85 - self,
86 - src_block_numbers: List[int],
87 - dst_block_numbers: List[int],
88 - ) -> None:
89 - for layer in range(self.num_layers):
90 - # TODO: Call the COPY op.
80 + def copy(self, src_to_dst: Dict[int, int]) -> None:
81 + for event in self.events:
82 pass
83
93 - def swap_out(
94 - self,
95 - gpu_block_numbers: List[int],
96 - cpu_block_numbers: List[int],
97 - ) -> None:
98 - for layer in range(self.num_layers):
99 - # TODO: Call the SWAP_OUT op on the swap stream.
84 + def swap_in(self, src_to_dst: Dict[int, int]) -> None:
85 + for event in self.events:
86 pass
87
102 - def swap_in(
103 - self,
104 - gpu_block_numbers: List[int],
105 - cpu_block_numbers: List[int],
106 - ) -> None:
107 - for layer in range(self.num_layers):
108 - # TODO: Call the SWAP_IN op on the swap stream.
88 + def swap_out(self, src_to_dst: Dict[int, int]) -> None:
89 + for event in self.events:
90 pass