@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5ee14494e4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc] Remove cache stream and cache events (#3461)

Woosuk Kwon committed Mar 20, 2024 at 00:38 UTC 5ee14494e4c78769fa10af8b58c3e7808053da0d
3 files changed +86 -32
tests/worker/test_swap.py new
+77
@@ -0,0 +1,77 @@
1 +import torch
2 +
3 +from vllm.engine.arg_utils import EngineArgs
4 +from vllm.worker.worker import Worker
5 +from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6 +
7 +
8 +def test_swap() -> None:
9 + # Configure the engine.
10 + engine_args = EngineArgs(model="facebook/opt-125m",
11 + dtype="half",
12 + load_format="dummy")
13 + (model_config, cache_config, parallel_config, scheduler_config,
14 + device_config, _) = engine_args.create_engine_configs()
15 + cache_config.num_gpu_blocks = 100
16 + cache_config.num_cpu_blocks = 100
17 +
18 + # Create the worker.
19 + distributed_init_method = get_distributed_init_method(
20 + get_ip(), get_open_port())
21 + worker = Worker(
22 + model_config=model_config,
23 + parallel_config=parallel_config,
24 + scheduler_config=scheduler_config,
25 + device_config=device_config,
26 + local_rank=0,
27 + rank=0,
28 + distributed_init_method=distributed_init_method,
29 + is_driver_worker=True,
30 + )
31 +
32 + # Initialize the worker.
33 + worker.init_model()
34 + worker.load_model()
35 + worker.init_cache_engine(cache_config)
36 + worker.warm_up_model()
37 +
38 + # Randomly initialize the cache.
39 + gpu_cache = worker.cache_engine.gpu_cache
40 + cpu_cache = worker.cache_engine.cpu_cache
41 + num_layers = len(gpu_cache)
42 + for i in range(num_layers):
43 + gpu_key_cache, gpu_value_cache = gpu_cache[i]
44 + gpu_key_cache.random_()
45 + gpu_value_cache.random_()
46 + cpu_key_cache, cpu_value_cache = cpu_cache[i]
47 + cpu_key_cache.random_()
48 + cpu_value_cache.random_()
49 +
50 + allclose = lambda a, b: torch.allclose(
51 + a.cuda(), b.cuda(), rtol=0.0, atol=0.0)
52 +
53 + # Test swap out.
54 + blocks_to_swap_out = {3: 72, 56: 35, 84: 34}
55 + worker.execute_model(seq_group_metadata_list=[],
56 + blocks_to_swap_in={},
57 + blocks_to_swap_out=blocks_to_swap_out,
58 + blocks_to_copy={})
59 + for i in range(num_layers):
60 + gpu_key_cache, gpu_value_cache = gpu_cache[i]
61 + cpu_key_cache, cpu_value_cache = cpu_cache[i]
62 + for src, dst in blocks_to_swap_out.items():
63 + assert allclose(gpu_key_cache[src], cpu_key_cache[dst])
64 + assert allclose(gpu_value_cache[src], cpu_value_cache[dst])
65 +
66 + # Test swap in.
67 + blocks_to_swap_in = {19: 45, 67: 23, 12: 78, 40: 99, 1: 71}
68 + worker.execute_model(seq_group_metadata_list=[],
69 + blocks_to_swap_in=blocks_to_swap_in,
70 + blocks_to_swap_out={},
71 + blocks_to_copy={})
72 + for i in range(num_layers):
73 + gpu_key_cache, gpu_value_cache = gpu_cache[i]
74 + cpu_key_cache, cpu_value_cache = cpu_cache[i]
75 + for src, dst in blocks_to_swap_in.items():
76 + assert allclose(gpu_key_cache[dst], cpu_key_cache[src])
77 + assert allclose(gpu_value_cache[dst], cpu_value_cache[src])
vllm/worker/cache_engine.py
+8 -18
@@ -38,7 +38,7 @@ class CacheEngine:
38 self.num_gpu_blocks = cache_config.num_gpu_blocks
39 self.num_cpu_blocks = cache_config.num_cpu_blocks
40
41 - # Skip initializing CUDA stream and buffer for Neuron backend.
41 + # Skip initializing KV cache for Neuron backend.
42 if is_neuron():
43 return
44
@@ -51,12 +51,6 @@ class CacheEngine:
51 self.gpu_cache = self.allocate_gpu_cache()
52 self.cpu_cache = self.allocate_cpu_cache()
53
54 - # Initialize the stream for caching operations.
55 - self.cache_stream = torch.cuda.Stream()
56 - assert self.cache_stream != torch.cuda.current_stream()
57 - # Initialize the events for stream synchronization.
58 - self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
59 -
54 def get_key_block_shape(self) -> Tuple[int, int, int, int]:
55 element_size = torch.tensor([], dtype=self.dtype).element_size()
56 x = 16 // element_size
@@ -126,17 +120,13 @@ class CacheEngine:
120 ) -> None:
121 from vllm._C import cache_ops
122
129 - with torch.cuda.stream(self.cache_stream):
130 - for i in range(self.num_layers):
131 - src_key_cache, src_value_cache = src[i]
132 - dst_key_cache, dst_value_cache = dst[i]
133 - # Copy the key blocks.
134 - cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
135 - # Copy the value blocks.
136 - cache_ops.swap_blocks(src_value_cache, dst_value_cache,
137 - src_to_dst)
138 - event = self.events[i]
139 - event.record(stream=self.cache_stream)
123 + for i in range(self.num_layers):
124 + src_key_cache, src_value_cache = src[i]
125 + dst_key_cache, dst_value_cache = dst[i]
126 + # Copy the key blocks.
127 + cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
128 + # Copy the value blocks.
129 + cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
130
131 def swap_in(self, src_to_dst: Dict[int, int]) -> None:
132 self._swap(self.cpu_cache, self.gpu_cache, src_to_dst)
vllm/worker/worker.py
+1 -14
@@ -65,7 +65,6 @@ class Worker:
65 # self.init_cache_engine().
66 self.cache_config = None
67 self.cache_engine = None
68 - self.cache_events = None
68 self.gpu_cache = None
69
70 def init_model(self, cupy_port: Optional[int] = None) -> None:
@@ -148,7 +147,6 @@ class Worker:
147 self.cache_config = cache_config
148 self.cache_engine = CacheEngine(self.cache_config, self.model_config,
149 self.parallel_config)
151 - self.cache_events = self.cache_engine.events
150 self.gpu_cache = self.cache_engine.gpu_cache
151 self.model_runner.set_block_size(self.cache_engine.block_size)
152
@@ -166,24 +164,13 @@ class Worker:
164 blocks_to_copy: Dict[int, List[int]],
165 ) -> None:
166 # Issue cache operations.
169 - issued_cache_op = False
167 + # TODO(woosuk): Profile swapping overhead and optimize if needed.
168 if blocks_to_swap_in:
169 self.cache_engine.swap_in(blocks_to_swap_in)
172 - issued_cache_op = True
170 if blocks_to_swap_out:
171 self.cache_engine.swap_out(blocks_to_swap_out)
175 - issued_cache_op = True
172 if blocks_to_copy:
173 self.cache_engine.copy(blocks_to_copy)
178 - issued_cache_op = True
179 -
180 - cache_events = self.cache_events if issued_cache_op else None
181 -
182 - # Wait for cache operations to finish.
183 - # TODO(woosuk): Profile swapping overhead and optimize if needed.
184 - if cache_events is not None:
185 - for event in cache_events:
186 - event.wait()
174
175 @torch.inference_mode()
176 def execute_model(