A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Misc] Remove cache stream and cache events (#3461)
Woosuk Kwon committed
Mar 20, 2024 at 00:38 UTC
5ee14494e4c78769fa10af8b58c3e7808053da0d
3 files changed
+86
-32
tests/worker/test_swap.py
new
+77
@@ -0,0 +1,77 @@
1
+import torch
2
+
3
+from vllm.engine.arg_utils import EngineArgs
4
+from vllm.worker.worker import Worker
5
+from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6
+
7
+
8
+def test_swap() -> None:
9
+ # Configure the engine.
10
+ engine_args = EngineArgs(model="facebook/opt-125m",
11
+ dtype="half",
12
+ load_format="dummy")
13
+ (model_config, cache_config, parallel_config, scheduler_config,
14
+ device_config, _) = engine_args.create_engine_configs()
15
+ cache_config.num_gpu_blocks = 100
16
+ cache_config.num_cpu_blocks = 100
17
+
18
+ # Create the worker.
19
+ distributed_init_method = get_distributed_init_method(
20
+ get_ip(), get_open_port())
21
+ worker = Worker(
22
+ model_config=model_config,
23
+ parallel_config=parallel_config,
24
+ scheduler_config=scheduler_config,
25
+ device_config=device_config,
26
+ local_rank=0,
27
+ rank=0,
28
+ distributed_init_method=distributed_init_method,
29
+ is_driver_worker=True,
30
+ )
31
+
32
+ # Initialize the worker.
33
+ worker.init_model()
34
+ worker.load_model()
35
+ worker.init_cache_engine(cache_config)
36
+ worker.warm_up_model()
37
+
38
+ # Randomly initialize the cache.
39
+ gpu_cache = worker.cache_engine.gpu_cache
40
+ cpu_cache = worker.cache_engine.cpu_cache
41
+ num_layers = len(gpu_cache)
42
+ for i in range(num_layers):
43
+ gpu_key_cache, gpu_value_cache = gpu_cache[i]
44
+ gpu_key_cache.random_()
45
+ gpu_value_cache.random_()
46
+ cpu_key_cache, cpu_value_cache = cpu_cache[i]
47
+ cpu_key_cache.random_()
48
+ cpu_value_cache.random_()
49
+
50
+ allclose = lambda a, b: torch.allclose(
51
+ a.cuda(), b.cuda(), rtol=0.0, atol=0.0)
52
+
53
+ # Test swap out.
54
+ blocks_to_swap_out = {3: 72, 56: 35, 84: 34}
55
+ worker.execute_model(seq_group_metadata_list=[],
56
+ blocks_to_swap_in={},
57
+ blocks_to_swap_out=blocks_to_swap_out,
58
+ blocks_to_copy={})
59
+ for i in range(num_layers):
60
+ gpu_key_cache, gpu_value_cache = gpu_cache[i]
61
+ cpu_key_cache, cpu_value_cache = cpu_cache[i]
62
+ for src, dst in blocks_to_swap_out.items():
63
+ assert allclose(gpu_key_cache[src], cpu_key_cache[dst])
64
+ assert allclose(gpu_value_cache[src], cpu_value_cache[dst])
65
+
66
+ # Test swap in.
67
+ blocks_to_swap_in = {19: 45, 67: 23, 12: 78, 40: 99, 1: 71}
68
+ worker.execute_model(seq_group_metadata_list=[],
69
+ blocks_to_swap_in=blocks_to_swap_in,
70
+ blocks_to_swap_out={},
71
+ blocks_to_copy={})
72
+ for i in range(num_layers):
73
+ gpu_key_cache, gpu_value_cache = gpu_cache[i]
74
+ cpu_key_cache, cpu_value_cache = cpu_cache[i]
75
+ for src, dst in blocks_to_swap_in.items():
76
+ assert allclose(gpu_key_cache[dst], cpu_key_cache[src])
77
+ assert allclose(gpu_value_cache[dst], cpu_value_cache[src])
vllm/worker/cache_engine.py
+8
-18
@@ -38,7 +38,7 @@ class CacheEngine:
38
self.num_gpu_blocks = cache_config.num_gpu_blocks
39
self.num_cpu_blocks = cache_config.num_cpu_blocks
40
41
- # Skip initializing CUDA stream and buffer for Neuron backend.
41
+ # Skip initializing KV cache for Neuron backend.
42
if is_neuron():
43
return
44
@@ -51,12 +51,6 @@ class CacheEngine:
51
self.gpu_cache = self.allocate_gpu_cache()
52
self.cpu_cache = self.allocate_cpu_cache()
53
54
- # Initialize the stream for caching operations.
55
- self.cache_stream = torch.cuda.Stream()
56
- assert self.cache_stream != torch.cuda.current_stream()
57
- # Initialize the events for stream synchronization.
58
- self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
59
-
54
def get_key_block_shape(self) -> Tuple[int, int, int, int]:
55
element_size = torch.tensor([], dtype=self.dtype).element_size()
56
x = 16 // element_size
@@ -126,17 +120,13 @@ class CacheEngine:
120
) -> None:
121
from vllm._C import cache_ops
122
129
- with torch.cuda.stream(self.cache_stream):
130
- for i in range(self.num_layers):
131
- src_key_cache, src_value_cache = src[i]
132
- dst_key_cache, dst_value_cache = dst[i]
133
- # Copy the key blocks.
134
- cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
135
- # Copy the value blocks.
136
- cache_ops.swap_blocks(src_value_cache, dst_value_cache,
137
- src_to_dst)
138
- event = self.events[i]
139
- event.record(stream=self.cache_stream)
123
+ for i in range(self.num_layers):
124
+ src_key_cache, src_value_cache = src[i]
125
+ dst_key_cache, dst_value_cache = dst[i]
126
+ # Copy the key blocks.
127
+ cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
128
+ # Copy the value blocks.
129
+ cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
130
131
def swap_in(self, src_to_dst: Dict[int, int]) -> None:
132
self._swap(self.cpu_cache, self.gpu_cache, src_to_dst)
vllm/worker/worker.py
+1
-14
@@ -65,7 +65,6 @@ class Worker:
65
# self.init_cache_engine().
66
self.cache_config = None
67
self.cache_engine = None
68
- self.cache_events = None
68
self.gpu_cache = None
69
70
def init_model(self, cupy_port: Optional[int] = None) -> None:
@@ -148,7 +147,6 @@ class Worker:
147
self.cache_config = cache_config
148
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
149
self.parallel_config)
151
- self.cache_events = self.cache_engine.events
150
self.gpu_cache = self.cache_engine.gpu_cache
151
self.model_runner.set_block_size(self.cache_engine.block_size)
152
@@ -166,24 +164,13 @@ class Worker:
164
blocks_to_copy: Dict[int, List[int]],
165
) -> None:
166
# Issue cache operations.
169
- issued_cache_op = False
167
+ # TODO(woosuk): Profile swapping overhead and optimize if needed.
168
if blocks_to_swap_in:
169
self.cache_engine.swap_in(blocks_to_swap_in)
172
- issued_cache_op = True
170
if blocks_to_swap_out:
171
self.cache_engine.swap_out(blocks_to_swap_out)
175
- issued_cache_op = True
172
if blocks_to_copy:
173
self.cache_engine.copy(blocks_to_copy)
178
- issued_cache_op = True
179
-
180
- cache_events = self.cache_events if issued_cache_op else None
181
-
182
- # Wait for cache operations to finish.
183
- # TODO(woosuk): Profile swapping overhead and optimize if needed.
184
- if cache_events is not None:
185
- for event in cache_events:
186
- event.wait()
174
175
@torch.inference_mode()
176
def execute_model(