A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Do not allocate beyond the max_model_len (#10730)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 28, 2024 at 00:13 UTC
a79b1224005836bdf0ab6d3bab807d2f5d8a5ef1
3 files changed
+41
-15
tests/v1/core/test_prefix_caching.py
+16
-8
@@ -23,7 +23,8 @@ def test_prefill():
23
manager = KVCacheManager(
24
block_size=16,
25
num_gpu_blocks=10,
26
- sliding_window=False,
26
+ max_model_len=8192,
27
+ sliding_window=None,
28
enable_caching=True,
29
num_preallocate_tokens=16,
30
)
@@ -121,7 +122,8 @@ def test_decode():
122
manager = KVCacheManager(
123
block_size=16,
124
num_gpu_blocks=10,
124
- sliding_window=False,
125
+ max_model_len=8192,
126
+ sliding_window=None,
127
enable_caching=True,
128
num_preallocate_tokens=16,
129
)
@@ -172,7 +174,8 @@ def test_evict():
174
manager = KVCacheManager(
175
block_size=16,
176
num_gpu_blocks=10,
175
- sliding_window=False,
177
+ max_model_len=8192,
178
+ sliding_window=None,
179
enable_caching=True,
180
num_preallocate_tokens=16,
181
)
@@ -220,7 +223,8 @@ def test_hash_block_correct_reuse():
223
manager = KVCacheManager(
224
block_size=block_size,
225
num_gpu_blocks=1,
223
- sliding_window=False,
226
+ max_model_len=8192,
227
+ sliding_window=None,
228
enable_caching=True,
229
num_preallocate_tokens=0,
230
)
@@ -256,7 +260,8 @@ def test_computed_blocks_not_evicted():
260
manager = KVCacheManager(
261
block_size=block_size,
262
num_gpu_blocks=2,
259
- sliding_window=False,
263
+ max_model_len=8192,
264
+ sliding_window=None,
265
enable_caching=True,
266
num_preallocate_tokens=0,
267
)
@@ -303,7 +308,8 @@ def test_basic_prefix_caching_disabled():
308
manager = KVCacheManager(
309
block_size=block_size,
310
num_gpu_blocks=4,
306
- sliding_window=False,
311
+ max_model_len=8192,
312
+ sliding_window=None,
313
enable_caching=False,
314
num_preallocate_tokens=0,
315
)
@@ -342,7 +348,8 @@ def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
348
manager = KVCacheManager(
349
block_size=block_size,
350
num_gpu_blocks=10,
345
- sliding_window=False,
351
+ max_model_len=8192,
352
+ sliding_window=None,
353
enable_caching=True,
354
num_preallocate_tokens=num_preallocate_tokens,
355
)
@@ -370,7 +377,8 @@ def test_cache_blocks():
377
manager = KVCacheManager(
378
block_size=block_size,
379
num_gpu_blocks=5,
373
- sliding_window=False,
380
+ max_model_len=8192,
381
+ sliding_window=None,
382
enable_caching=True,
383
num_preallocate_tokens=0,
384
)
vllm/v1/core/kv_cache_manager.py
+17
@@ -17,12 +17,15 @@ class KVCacheManager:
17
self,
18
block_size: int,
19
num_gpu_blocks: int,
20
+ max_model_len: int,
21
sliding_window: Optional[int] = None,
22
enable_caching: bool = True,
23
num_preallocate_tokens: int = 64,
24
) -> None:
25
self.block_size = block_size
26
self.num_gpu_blocks = num_gpu_blocks
27
+ self.max_model_len = max_model_len
28
+ self.max_num_blocks_per_req = cdiv(max_model_len, block_size)
29
self.sliding_window = sliding_window
30
self.enable_caching = enable_caching
31
# NOTE(woosuk): To avoid frequent block allocation, we preallocate some
@@ -132,7 +135,14 @@ class KVCacheManager:
135
num_new_blocks = min(
136
num_new_blocks + self.num_preallocate_blocks,
137
self.free_block_queue.num_free_blocks,
138
+ # Should not exceed the maximum number of blocks per request.
139
+ # This is especially because the block table has the shape
140
+ # [..., max_num_blocks_per_req].
141
+ # TODO(woosuk): Check and reject requests if
142
+ # num_prompt_tokens + max_tokens > max_model_len.
143
+ self.max_num_blocks_per_req - len(req_blocks),
144
)
145
+ assert num_new_blocks > 0
146
147
new_blocks = self._get_new_blocks(num_new_blocks)
148
req_blocks.extend(new_blocks)
@@ -212,7 +222,14 @@ class KVCacheManager:
222
num_required_blocks + self.num_preallocate_blocks,
223
self.free_block_queue.num_free_blocks -
224
num_evictable_computed_blocks,
225
+ # Should not exceed the maximum number of blocks per request.
226
+ # This is especially because the block table has the shape
227
+ # [..., max_num_blocks_per_req].
228
+ # TODO(woosuk): Check and reject requests if
229
+ # num_prompt_tokens + max_tokens > max_model_len.
230
+ self.max_num_blocks_per_req - len(computed_blocks),
231
)
232
+ assert num_new_blocks > 0
233
234
# Concatenate the computed block IDs and the new block IDs.
235
new_blocks = self._get_new_blocks(num_new_blocks)
vllm/v1/core/scheduler.py
+8
-7
@@ -33,22 +33,23 @@ class Scheduler:
33
# TODO: Support LoRA.
34
assert lora_config is None, "V1 does not support LoRA yet."
35
36
+ # Scheduling constraints.
37
+ self.max_num_running_reqs = self.scheduler_config.max_num_seqs
38
+ self.max_num_scheduled_tokens = \
39
+ self.scheduler_config.max_num_batched_tokens
40
+ self.max_model_len = self.scheduler_config.max_model_len
41
+
42
num_gpu_blocks = cache_config.num_gpu_blocks
43
assert isinstance(num_gpu_blocks, int) and num_gpu_blocks > 0
38
- # Create the block space manager.
44
+ # Create the KV cache manager.
45
self.kv_cache_manager = KVCacheManager(
46
block_size=self.cache_config.block_size,
47
num_gpu_blocks=num_gpu_blocks,
48
+ max_model_len=self.max_model_len,
49
sliding_window=self.cache_config.sliding_window,
50
enable_caching=self.cache_config.enable_prefix_caching)
51
self.block_size = self.cache_config.block_size
52
46
- # Scheduling constraints.
47
- self.max_num_running_reqs = self.scheduler_config.max_num_seqs
48
- self.max_num_scheduled_tokens = \
49
- self.scheduler_config.max_num_batched_tokens
50
- self.max_model_len = self.scheduler_config.max_model_len
51
-
53
# req_id -> Request
54
self.requests: Dict[str, Request] = {}
55
# Priority queues for requests.