@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7bee2aa81
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add cache engine

Woosuk Kwon committed Feb 9, 2023 at 11:28 UTC e7bee2aa811963b8c5ce352a427595749f6bfca1
1 file changed +109
cacheflow/worker/cache_engine.py new
+109
@@ -0,0 +1,109 @@
1 +from typing import List, Tuple
2 +
3 +import torch
4 +
5 +KVCache = Tuple[torch.Tensor, torch.Tensor]
6 +
7 +
8 +class CacheEngine:
9 +
10 + def __init__(
11 + self,
12 + worker_id: int,
13 + gpu_id: int,
14 + num_layers: int,
15 + num_heads: int,
16 + head_size: int,
17 + num_gpu_blocks: int,
18 + num_cpu_blocks: int,
19 + block_size: int,
20 + dtype: torch.dtype = torch.float16,
21 + ) -> None:
22 + self.worker_id = worker_id
23 + self.gpu_id = gpu_id
24 + self.num_layers = num_layers
25 + self.num_heads = num_heads
26 + self.head_size = head_size
27 + self.num_gpu_blocks = num_gpu_blocks
28 + self.num_cpu_blocks = num_cpu_blocks
29 + self.block_size = block_size
30 + self.dtype = dtype
31 +
32 + # Initialize the cache.
33 + self.gpu_cache = self.allocate_gpu_cache()
34 + self.cpu_cache = self.allocate_cpu_cache()
35 +
36 + # Initialize the streams.
37 + self.copy_stream = torch.cuda.Stream(device=gpu_id)
38 + self.swap_stream = torch.cuda.Stream(device=gpu_id)
39 + assert self.copy_stream != self.swap_stream
40 + current_stream = torch.cuda.current_stream(device=gpu_id)
41 + assert self.copy_stream != current_stream
42 + assert self.swap_stream != current_stream
43 +
44 + # Initialize the events for synchronization.
45 +
46 + def allocate_gpu_cache(self) -> List[List[KVCache]]:
47 + gpu_cache: List[List[KVCache]] = []
48 + for _ in range(self.num_layers):
49 + layer_cache: List[KVCache] = []
50 + for _ in range(self.num_heads):
51 + key_blocks = torch.empty(
52 + (self.num_gpu_blocks, self.block_size * self.head_size),
53 + dtype=self.dtype,
54 + device=self.gpu_id,
55 + )
56 + value_blocks = torch.empty(
57 + (self.num_gpu_blocks, self.block_size * self.head_size),
58 + dtype=self.dtype,
59 + device=self.gpu_id,
60 + )
61 + layer_cache.append((key_blocks, value_blocks))
62 + gpu_cache.append(layer_cache)
63 + return gpu_cache
64 +
65 + def allocate_cpu_cache(self) -> List[List[KVCache]]:
66 + cpu_cache: List[List[KVCache]] = []
67 + for _ in range(self.num_layers):
68 + layer_cache: List[KVCache] = []
69 + for _ in range(self.num_heads):
70 + key_blocks = torch.empty(
71 + (self.num_cpu_blocks, self.block_size * self.head_size),
72 + dtype=self.dtype,
73 + pin_memory=True,
74 + )
75 + value_blocks = torch.empty(
76 + (self.num_cpu_blocks, self.block_size * self.head_size),
77 + dtype=self.dtype,
78 + pin_memory=True,
79 + )
80 + layer_cache.append((key_blocks, value_blocks))
81 + cpu_cache.append(layer_cache)
82 + return cpu_cache
83 +
84 + def copy(
85 + self,
86 + src_block_numbers: List[int],
87 + dst_block_numbers: List[int],
88 + ) -> None:
89 + for layer in range(self.num_layers):
90 + # TODO: Call the COPY op.
91 + pass
92 +
93 + def swap_out(
94 + self,
95 + gpu_block_numbers: List[int],
96 + cpu_block_numbers: List[int],
97 + ) -> None:
98 + for layer in range(self.num_layers):
99 + # TODO: Call the SWAP_OUT op on the swap stream.
100 + pass
101 +
102 + def swap_in(
103 + self,
104 + gpu_block_numbers: List[int],
105 + cpu_block_numbers: List[int],
106 + ) -> None:
107 + for layer in range(self.num_layers):
108 + # TODO: Call the SWAP_IN op on the swap stream.
109 + pass