@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a79b122400
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Do not allocate beyond the max_model_len (#10730)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 28, 2024 at 00:13 UTC a79b1224005836bdf0ab6d3bab807d2f5d8a5ef1
3 files changed +41 -15
tests/v1/core/test_prefix_caching.py
+16 -8
@@ -23,7 +23,8 @@ def test_prefill():
23 manager = KVCacheManager(
24 block_size=16,
25 num_gpu_blocks=10,
26 - sliding_window=False,
26 + max_model_len=8192,
27 + sliding_window=None,
28 enable_caching=True,
29 num_preallocate_tokens=16,
30 )
@@ -121,7 +122,8 @@ def test_decode():
122 manager = KVCacheManager(
123 block_size=16,
124 num_gpu_blocks=10,
124 - sliding_window=False,
125 + max_model_len=8192,
126 + sliding_window=None,
127 enable_caching=True,
128 num_preallocate_tokens=16,
129 )
@@ -172,7 +174,8 @@ def test_evict():
174 manager = KVCacheManager(
175 block_size=16,
176 num_gpu_blocks=10,
175 - sliding_window=False,
177 + max_model_len=8192,
178 + sliding_window=None,
179 enable_caching=True,
180 num_preallocate_tokens=16,
181 )
@@ -220,7 +223,8 @@ def test_hash_block_correct_reuse():
223 manager = KVCacheManager(
224 block_size=block_size,
225 num_gpu_blocks=1,
223 - sliding_window=False,
226 + max_model_len=8192,
227 + sliding_window=None,
228 enable_caching=True,
229 num_preallocate_tokens=0,
230 )
@@ -256,7 +260,8 @@ def test_computed_blocks_not_evicted():
260 manager = KVCacheManager(
261 block_size=block_size,
262 num_gpu_blocks=2,
259 - sliding_window=False,
263 + max_model_len=8192,
264 + sliding_window=None,
265 enable_caching=True,
266 num_preallocate_tokens=0,
267 )
@@ -303,7 +308,8 @@ def test_basic_prefix_caching_disabled():
308 manager = KVCacheManager(
309 block_size=block_size,
310 num_gpu_blocks=4,
306 - sliding_window=False,
311 + max_model_len=8192,
312 + sliding_window=None,
313 enable_caching=False,
314 num_preallocate_tokens=0,
315 )
@@ -342,7 +348,8 @@ def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
348 manager = KVCacheManager(
349 block_size=block_size,
350 num_gpu_blocks=10,
345 - sliding_window=False,
351 + max_model_len=8192,
352 + sliding_window=None,
353 enable_caching=True,
354 num_preallocate_tokens=num_preallocate_tokens,
355 )
@@ -370,7 +377,8 @@ def test_cache_blocks():
377 manager = KVCacheManager(
378 block_size=block_size,
379 num_gpu_blocks=5,
373 - sliding_window=False,
380 + max_model_len=8192,
381 + sliding_window=None,
382 enable_caching=True,
383 num_preallocate_tokens=0,
384 )
vllm/v1/core/kv_cache_manager.py
+17
@@ -17,12 +17,15 @@ class KVCacheManager:
17 self,
18 block_size: int,
19 num_gpu_blocks: int,
20 + max_model_len: int,
21 sliding_window: Optional[int] = None,
22 enable_caching: bool = True,
23 num_preallocate_tokens: int = 64,
24 ) -> None:
25 self.block_size = block_size
26 self.num_gpu_blocks = num_gpu_blocks
27 + self.max_model_len = max_model_len
28 + self.max_num_blocks_per_req = cdiv(max_model_len, block_size)
29 self.sliding_window = sliding_window
30 self.enable_caching = enable_caching
31 # NOTE(woosuk): To avoid frequent block allocation, we preallocate some
@@ -132,7 +135,14 @@ class KVCacheManager:
135 num_new_blocks = min(
136 num_new_blocks + self.num_preallocate_blocks,
137 self.free_block_queue.num_free_blocks,
138 + # Should not exceed the maximum number of blocks per request.
139 + # This is especially because the block table has the shape
140 + # [..., max_num_blocks_per_req].
141 + # TODO(woosuk): Check and reject requests if
142 + # num_prompt_tokens + max_tokens > max_model_len.
143 + self.max_num_blocks_per_req - len(req_blocks),
144 )
145 + assert num_new_blocks > 0
146
147 new_blocks = self._get_new_blocks(num_new_blocks)
148 req_blocks.extend(new_blocks)
@@ -212,7 +222,14 @@ class KVCacheManager:
222 num_required_blocks + self.num_preallocate_blocks,
223 self.free_block_queue.num_free_blocks -
224 num_evictable_computed_blocks,
225 + # Should not exceed the maximum number of blocks per request.
226 + # This is especially because the block table has the shape
227 + # [..., max_num_blocks_per_req].
228 + # TODO(woosuk): Check and reject requests if
229 + # num_prompt_tokens + max_tokens > max_model_len.
230 + self.max_num_blocks_per_req - len(computed_blocks),
231 )
232 + assert num_new_blocks > 0
233
234 # Concatenate the computed block IDs and the new block IDs.
235 new_blocks = self._get_new_blocks(num_new_blocks)
vllm/v1/core/scheduler.py
+8 -7
@@ -33,22 +33,23 @@ class Scheduler:
33 # TODO: Support LoRA.
34 assert lora_config is None, "V1 does not support LoRA yet."
35
36 + # Scheduling constraints.
37 + self.max_num_running_reqs = self.scheduler_config.max_num_seqs
38 + self.max_num_scheduled_tokens = \
39 + self.scheduler_config.max_num_batched_tokens
40 + self.max_model_len = self.scheduler_config.max_model_len
41 +
42 num_gpu_blocks = cache_config.num_gpu_blocks
43 assert isinstance(num_gpu_blocks, int) and num_gpu_blocks > 0
38 - # Create the block space manager.
44 + # Create the KV cache manager.
45 self.kv_cache_manager = KVCacheManager(
46 block_size=self.cache_config.block_size,
47 num_gpu_blocks=num_gpu_blocks,
48 + max_model_len=self.max_model_len,
49 sliding_window=self.cache_config.sliding_window,
50 enable_caching=self.cache_config.enable_prefix_caching)
51 self.block_size = self.cache_config.block_size
52
46 - # Scheduling constraints.
47 - self.max_num_running_reqs = self.scheduler_config.max_num_seqs
48 - self.max_num_scheduled_tokens = \
49 - self.scheduler_config.max_num_batched_tokens
50 - self.max_model_len = self.scheduler_config.max_model_len
51 -
53 # req_id -> Request
54 self.requests: Dict[str, Request] = {}
55 # Priority queues for requests.