A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Core] Reduce TTFT with concurrent partial prefills (#10235)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Joe Runde committed
Feb 14, 2025 at 16:36 UTC
3bcb8c75da56e39cfd6e440402cf043a3d470b33
6 files changed
+701
-108
tests/basic_correctness/test_chunked_prefill.py
+10
-20
@@ -8,7 +8,6 @@ prefill requests are chunked.
8
Run `pytest tests/models/test_chunked_prefill.py`.
9
"""
10
import os
11
-from contextlib import nullcontext
11
12
import pytest
13
@@ -233,7 +232,6 @@ def test_with_prefix_caching(
232
233
max_num_batched_tokens = max_num_seqs = chunk_size
234
outputs = {} # type: ignore
236
- check_result = True
235
for enable in (True, False):
236
with vllm_runner(
237
model,
@@ -245,25 +243,17 @@ def test_with_prefix_caching(
243
enforce_eager=enforce_eager,
244
max_num_seqs=max_num_seqs,
245
) as vllm_model:
248
- # It should fail when prefix caching is enable and chunk
249
- # size is not a multiple of block size (16).
250
- should_fail = chunk_size % 16 != 0 and enable
251
- check_result &= not should_fail
246
outputs[enable] = []
253
- # Send the request one-by-one to ensure the cache is populated.
254
- with pytest.raises(ValueError) if should_fail else nullcontext():
255
- for prompt in full_prompts:
256
- outputs[enable] += vllm_model.generate_greedy([prompt],
257
- max_tokens)
258
-
259
- # Check results only if we did not expect a failure.
260
- if check_result:
261
- check_outputs_equal(
262
- outputs_0_lst=outputs[False],
263
- outputs_1_lst=outputs[True],
264
- name_0="w/o prefix caching",
265
- name_1="with prefix caching",
266
- )
247
+ for prompt in full_prompts:
248
+ outputs[enable] += vllm_model.generate_greedy([prompt],
249
+ max_tokens)
250
+
251
+ check_outputs_equal(
252
+ outputs_0_lst=outputs[False],
253
+ outputs_1_lst=outputs[True],
254
+ name_0="w/o prefix caching",
255
+ name_1="with prefix caching",
256
+ )
257
258
259
@pytest.mark.parametrize("model", ["facebook/opt-125m"])
tests/core/test_chunked_prefill_scheduler.py
+314
-2
@@ -7,6 +7,9 @@ import pytest # noqa
7
8
from vllm.config import CacheConfig, SchedulerConfig
9
from vllm.core.scheduler import Scheduler
10
+from vllm.engine.arg_utils import EngineArgs
11
+from vllm.engine.llm_engine import LLMEngine
12
+from vllm.sampling_params import SamplingParams
13
from vllm.sequence import Logprob, SequenceGroup
14
15
from .utils import create_dummy_prompt
@@ -16,7 +19,7 @@ def get_sequence_groups(scheduler_output):
19
return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
20
21
19
-def append_new_token(seq_group, token_id: int):
22
+def append_new_token(seq_group: SequenceGroup, token_id: int):
23
for seq in seq_group.get_seqs():
24
seq.append_token_id(token_id, {token_id: Logprob(token_id)})
25
@@ -123,6 +126,232 @@ def test_chunk():
126
assert out.num_batched_tokens == 57
127
128
129
+def test_concurrent_chunking():
130
+ """Verify prefills are chunked properly when
131
+ --max-num-partial-prefills is > 1"""
132
+ block_size = 4
133
+ max_seqs = 60
134
+ max_model_len = 2000
135
+ max_num_batched_tokens = 64
136
+ scheduler_config = SchedulerConfig(
137
+ "generate",
138
+ max_num_batched_tokens,
139
+ max_seqs,
140
+ max_model_len,
141
+ enable_chunked_prefill=True,
142
+ max_num_partial_prefills=2, # Up to 2 partial prefills at a time
143
+ )
144
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
145
+ cache_config.num_cpu_blocks = 32
146
+ cache_config.num_gpu_blocks = 32
147
+ scheduler = Scheduler(scheduler_config, cache_config, None)
148
+ running: List[SequenceGroup] = []
149
+
150
+ # Add seq groups to scheduler.
151
+ for i in range(2):
152
+ _, seq_group = create_dummy_prompt(str(i),
153
+ prompt_length=60,
154
+ block_size=block_size)
155
+ scheduler.add_seq_group(seq_group)
156
+ running.append(seq_group)
157
+
158
+ # Verify both requests are chunked with half of max_num_batched_tokens each
159
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
160
+ assert set(get_sequence_groups(out)) == set(running)
161
+ assert seq_group_meta[0].token_chunk_size == 32
162
+ assert seq_group_meta[1].token_chunk_size == 32
163
+ assert out.num_prefill_groups == 2
164
+ assert out.num_batched_tokens == 64
165
+
166
+ # After one iteration, both should have 60 - 32 = 28 tokens left to prefill
167
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
168
+ assert set(get_sequence_groups(out)) == set(running)
169
+ assert seq_group_meta[0].token_chunk_size == 28
170
+ assert seq_group_meta[1].token_chunk_size == 28
171
+ assert out.num_prefill_groups == 2
172
+ assert out.num_batched_tokens == 56
173
+
174
+
175
+def test_concurrent_chunking_large_requests():
176
+ """Verify large prefill requests are run one at a time"""
177
+ block_size = 4
178
+ max_seqs = 60
179
+ max_model_len = 2000
180
+ max_num_batched_tokens = 64
181
+ scheduler_config = SchedulerConfig(
182
+ "generate",
183
+ max_num_batched_tokens,
184
+ max_seqs,
185
+ max_model_len,
186
+ enable_chunked_prefill=True,
187
+ max_num_partial_prefills=2, # Up to 2 partial prefills at a time
188
+ )
189
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
190
+ cache_config.num_cpu_blocks = 3200 # large KV cache size for large requests
191
+ cache_config.num_gpu_blocks = 3200
192
+ scheduler = Scheduler(scheduler_config, cache_config, None)
193
+
194
+ # Add seq groups to scheduler.
195
+ for i in range(2):
196
+ _, seq_group = create_dummy_prompt(
197
+ str(i),
198
+ prompt_length=1200, # Very large prompt
199
+ block_size=block_size)
200
+ scheduler.add_seq_group(seq_group)
201
+
202
+ # Verify only a single request is chunked, and it gets all 64 tokens
203
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
204
+ assert len(get_sequence_groups(out)) == 1
205
+ assert seq_group_meta[0].token_chunk_size == 64
206
+ assert out.num_prefill_groups == 1
207
+ assert out.num_batched_tokens == 64
208
+
209
+
210
+def test_short_prompts_jump_long_prompts_in_queue():
211
+ """Verify large prefill requests are punted behind smaller ones if
212
+ another large prefill request is already running"""
213
+ block_size = 4
214
+ max_seqs = 60
215
+ max_model_len = 2000
216
+ max_num_batched_tokens = 64
217
+ scheduler_config = SchedulerConfig(
218
+ "generate",
219
+ max_num_batched_tokens,
220
+ max_seqs,
221
+ max_model_len,
222
+ enable_chunked_prefill=True,
223
+ max_num_partial_prefills=2, # Up to 2 partial prefills at a time
224
+ )
225
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
226
+ cache_config.num_cpu_blocks = 3200 # large KV cache size for large requests
227
+ cache_config.num_gpu_blocks = 3200
228
+ scheduler = Scheduler(scheduler_config, cache_config, None)
229
+ long_seqs: List[SequenceGroup] = []
230
+ short_seqs: List[SequenceGroup] = []
231
+
232
+ # Add 2 large seq groups to scheduler.
233
+ for i in range(2):
234
+ _, seq_group = create_dummy_prompt(
235
+ str(i),
236
+ prompt_length=1200, # Very large prompt
237
+ block_size=block_size)
238
+ scheduler.add_seq_group(seq_group)
239
+ long_seqs.append(seq_group)
240
+ assert seq_group.is_prefill()
241
+
242
+ # Add 2 small seq groups behind them
243
+ for i in range(2):
244
+ _, seq_group = create_dummy_prompt(
245
+ str(i + 2),
246
+ prompt_length=40, # Very small prompt
247
+ block_size=block_size)
248
+ scheduler.add_seq_group(seq_group)
249
+ short_seqs.append(seq_group)
250
+ assert seq_group.is_prefill()
251
+
252
+ # Verify one large req and 1 small req chunked
253
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
254
+ assert seq_group_meta[0].token_chunk_size == 32 # large req gets 32 tokens
255
+ assert seq_group_meta[1].token_chunk_size == 32 # small req gets 32 tokens
256
+
257
+ # all 4 are prefilling
258
+ assert long_seqs[0].is_prefill()
259
+ assert long_seqs[1].is_prefill()
260
+ assert short_seqs[0].is_prefill()
261
+ assert short_seqs[1].is_prefill()
262
+ # First short and first long sequences have been scheduled
263
+ assert long_seqs[0].first_seq.get_num_computed_tokens() == 32
264
+ assert long_seqs[1].first_seq.get_num_computed_tokens() == 0
265
+ assert short_seqs[0].first_seq.get_num_computed_tokens() == 32
266
+ assert short_seqs[1].first_seq.get_num_computed_tokens() == 0
267
+
268
+ assert out.num_prefill_groups == 2
269
+ assert out.num_batched_tokens == 64
270
+
271
+ # in the second iteration,
272
+ # the first small request had only 8 tokens left
273
+ # so it went to decode
274
+ # The other small req is scheduled
275
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
276
+ # the new small req got 64 - (32+8) tokens
277
+ assert seq_group_meta[0].token_chunk_size == 24
278
+ assert seq_group_meta[1].token_chunk_size == 32 # large req still got 32
279
+ # the other small request had only 8 tokens left
280
+ assert seq_group_meta[2].token_chunk_size == 8 # 40-32
281
+
282
+ # The first small request got to decode now
283
+ assert long_seqs[0].is_prefill()
284
+ assert long_seqs[1].is_prefill()
285
+ assert not short_seqs[0].is_prefill()
286
+ assert short_seqs[1].is_prefill()
287
+ # Both small requests have started in front of the second long request
288
+ assert long_seqs[0].first_seq.get_num_computed_tokens() == 64
289
+ assert long_seqs[1].first_seq.get_num_computed_tokens() == 0
290
+ assert short_seqs[0].first_seq.get_num_computed_tokens() == 40
291
+ assert short_seqs[1].first_seq.get_num_computed_tokens() == 24
292
+
293
+ assert out.num_prefill_groups == 3
294
+ assert out.num_batched_tokens == 64
295
+ # the first small seq group has a new token appended.
296
+ append_new_token(short_seqs[0], 1)
297
+
298
+ # in the third iteration,
299
+ # the first small request is already decoding
300
+ # the second small request only has 16 tokens left and will enter decoding
301
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
302
+ assert seq_group_meta[0].token_chunk_size == 32 # large still got 32
303
+ # small req finished prefilling 40-24=16 tokens
304
+ assert seq_group_meta[1].token_chunk_size == 16
305
+ assert seq_group_meta[2].token_chunk_size == 1 # decode
306
+ assert out.num_prefill_groups == 2
307
+ assert out.num_batched_tokens == 49 # (32+16+1 decode)
308
+
309
+ # both small requests have now reached decode
310
+ assert long_seqs[0].is_prefill()
311
+ assert long_seqs[1].is_prefill()
312
+ assert not short_seqs[0].is_prefill()
313
+ assert not short_seqs[1].is_prefill()
314
+ assert long_seqs[0].first_seq.get_num_computed_tokens() == 96
315
+ assert long_seqs[1].first_seq.get_num_computed_tokens() == 0
316
+ assert short_seqs[0].first_seq.get_num_computed_tokens() == 41
317
+ assert short_seqs[1].first_seq.get_num_computed_tokens() == 40
318
+
319
+ # both the small seq groups have a new token appended
320
+ append_new_token(short_seqs[0], 1)
321
+ append_new_token(short_seqs[1], 1)
322
+
323
+ # in the fourth iteration, both small requests are decoding
324
+ # so large request gets all the budget
325
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
326
+
327
+ # large req gets 62 tokens (minus 2 for decode)
328
+ assert seq_group_meta[0].token_chunk_size == 62
329
+ assert seq_group_meta[1].token_chunk_size == 1 # decode
330
+ assert seq_group_meta[2].token_chunk_size == 1 # decode
331
+ assert out.num_prefill_groups == 1
332
+ assert out.num_batched_tokens == 64
333
+
334
+ assert long_seqs[0].first_seq.get_num_computed_tokens() == 158
335
+
336
+ # assert long_seqs[0].is_prefill()
337
+ # assert long_seqs[1].is_prefill()
338
+ # assert not short_seqs[0].is_prefill()
339
+ # assert not short_seqs[1].is_prefill()
340
+
341
+ # # both the small seq groups have a new token appended
342
+ # append_new_token(short_seqs[0], 1)
343
+ # append_new_token(short_seqs[1], 1)
344
+
345
+ # # in the fifth iteration, large request gets all the budget
346
+ # # while both small requests are decoding
347
+ # seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
348
+ # assert seq_group_meta[0].token_chunk_size == 62
349
+ # assert seq_group_meta[1].token_chunk_size == 1 # decode
350
+ # assert seq_group_meta[2].token_chunk_size == 1 # decode
351
+ # assert out.num_prefill_groups == 1
352
+ # assert out.num_batched_tokens == 64
353
+
354
+
355
def test_complex():
356
block_size = 4
357
max_seqs = 60
@@ -508,7 +737,7 @@ def test_chunked_prefill_max_seqs():
737
assert not running[1].is_prefill()
738
739
511
-def test_perfix_caching():
740
+def test_prefix_caching():
741
"""Verify allocating full blocks when prefix caching is enabled."""
742
block_size = 4
743
max_seqs = 10
@@ -548,3 +777,86 @@ def test_perfix_caching():
777
assert seq_group_meta[1].token_chunk_size == 12
778
assert out.num_prefill_groups == 2
779
assert out.num_batched_tokens == 62
780
+
781
+
782
+def test_prefix_caching_with_concurrent_partial_prefills():
783
+ """Verify allocating full blocks when prefix caching is enabled with
784
+ --max-num-partial-prefills > 1."""
785
+ block_size = 4
786
+ max_seqs = 10
787
+ max_model_len = 8000
788
+ max_num_batched_tokens = 60 # With two slots, each slot will get 30 tokens
789
+ scheduler_config = SchedulerConfig("generate",
790
+ max_num_batched_tokens,
791
+ max_seqs,
792
+ max_model_len,
793
+ enable_chunked_prefill=True,
794
+ max_num_partial_prefills=2)
795
+ cache_config = CacheConfig(block_size,
796
+ 1.0,
797
+ 1,
798
+ "auto",
799
+ enable_prefix_caching=True)
800
+ cache_config.num_cpu_blocks = 0
801
+ cache_config.num_gpu_blocks = 32
802
+ scheduler = Scheduler(scheduler_config, cache_config, None)
803
+ running: List[SequenceGroup] = []
804
+
805
+ # Add seq groups to scheduler.
806
+ for i in range(2):
807
+ _, seq_group = create_dummy_prompt(str(i),
808
+ block_size=block_size,
809
+ prompt_length=50)
810
+ scheduler.add_seq_group(seq_group)
811
+ running.append(seq_group)
812
+
813
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
814
+ assert set(get_sequence_groups(out)) == set(running)
815
+ # To partially prefill both sequences, both can chunk up to 30 tokens
816
+ # But the next lowest multiple of the block size (4) is 28
817
+ assert seq_group_meta[0].token_chunk_size == 28
818
+ assert seq_group_meta[1].token_chunk_size == 28
819
+ assert out.num_prefill_groups == 2
820
+ assert out.num_batched_tokens == 56
821
+
822
+ # On the next iteration, both sequences should finish prefill
823
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
824
+ assert set(get_sequence_groups(out)) == set(running)
825
+ # Both sequences have 50 - 28 = 22 tokens left to prefill.
826
+ # This is not a multiple of the block size, but we don't care since we don't
827
+ # cache the final partial block of prefix sequences
828
+ assert seq_group_meta[0].token_chunk_size == 22
829
+ assert seq_group_meta[1].token_chunk_size == 22
830
+ assert out.num_prefill_groups == 2
831
+ assert out.num_batched_tokens == 44
832
+
833
+
834
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
835
+@pytest.mark.parametrize("max_num_partial_prefills", [2, 4, 8])
836
+def test_chunked_prefill_with_actual_engine(model: str,
837
+ max_num_partial_prefills: int):
838
+ """Make sure the model can actually sample with concurrent
839
+ partial prefills
840
+ """
841
+
842
+ prompt = "hello" * 40
843
+
844
+ engine_args = EngineArgs(
845
+ model=model,
846
+ max_num_partial_prefills=max_num_partial_prefills,
847
+ max_num_batched_tokens=40,
848
+ max_num_seqs=8,
849
+ enable_chunked_prefill=True,
850
+ gpu_memory_utilization=0.8,
851
+ )
852
+
853
+ engine = LLMEngine.from_engine_args(engine_args)
854
+ sampling_params = SamplingParams(temperature=0)
855
+
856
+ for req_num in range(max_num_partial_prefills):
857
+ engine.add_request(f"{req_num}", prompt, sampling_params)
858
+ # first step
859
+ request_outputs = engine.step()
860
+ # means all are prefilling
861
+ assert len(request_outputs) == 0
862
+ assert len(engine.scheduler[0].running) == max_num_partial_prefills
vllm/config.py
+46
@@ -1430,6 +1430,17 @@ class SchedulerConfig:
1430
# Maximum length of a sequence (including prompt and generated text).
1431
max_model_len: int = 8192
1432
1433
+ # Maximum number of sequences that can be partially prefilled concurrently
1434
+ max_num_partial_prefills: int = 1
1435
+
1436
+ # Maximum number of "very long prompt" sequences that can be prefilled
1437
+ # concurrently (long is defined by long_prefill_threshold)
1438
+ max_long_partial_prefills: int = 1
1439
+
1440
+ # calculate context length that determines which sequences are
1441
+ # considered "long"
1442
+ long_prefill_token_threshold: int = 0
1443
+
1444
# The number of slots to allocate per sequence per
1445
# step, beyond the known token ids. This is used in speculative
1446
# decoding to store KV activations of tokens which may or may not be
@@ -1537,6 +1548,18 @@ class SchedulerConfig:
1548
self.max_num_batched_tokens)
1549
1550
self.chunked_prefill_enabled = self.enable_chunked_prefill
1551
+ if self.max_num_partial_prefills > 1:
1552
+ if self.long_prefill_token_threshold == 0:
1553
+ self.long_prefill_token_threshold = int(self.max_model_len *
1554
+ 0.04)
1555
+
1556
+ logger.info(
1557
+ "Concurrent partial prefills enabled with "
1558
+ "max_num_partial_prefills=%d, max_long_partial_prefills=%d, "
1559
+ "long_prefill_token_threshold=%d",
1560
+ self.max_num_partial_prefills, self.max_long_partial_prefills,
1561
+ self.long_prefill_token_threshold)
1562
+
1563
self._verify_args()
1564
1565
def _verify_args(self) -> None:
@@ -1568,6 +1591,29 @@ class SchedulerConfig:
1591
f"({self.num_scheduler_steps}) must be greater than or "
1592
"equal to 1.")
1593
1594
+ if self.max_num_partial_prefills < 1:
1595
+ raise ValueError(
1596
+ f"max_num_partial_prefills ({self.max_num_partial_prefills}) "
1597
+ "must be greater than or equal to 1.")
1598
+ elif self.max_num_partial_prefills > 1:
1599
+ if not self.chunked_prefill_enabled:
1600
+ raise ValueError("Chunked prefill must be enabled to set "
1601
+ "max_num_partial_prefills > 1.")
1602
+
1603
+ if self.long_prefill_token_threshold > self.max_model_len:
1604
+ raise ValueError(
1605
+ "long_prefill_token_threshold "
1606
+ f"({self.long_prefill_token_threshold}) cannot be greater "
1607
+ f"than the max_model_len ({self.max_model_len}).")
1608
+
1609
+ if (self.max_long_partial_prefills
1610
+ < 1) or (self.max_long_partial_prefills
1611
+ > self.max_num_partial_prefills):
1612
+ raise ValueError(
1613
+ f"max_long_partial_prefills ({self.max_long_partial_prefills}) "
1614
+ "must be greater than or equal to 1 and less than or equal to "
1615
+ f"max_num_partial_prefills ({self.max_num_partial_prefills}).")
1616
+
1617
@property
1618
def is_multi_step(self) -> bool:
1619
return self.num_scheduler_steps > 1
vllm/core/scheduler.py
+291
-84
@@ -17,7 +17,7 @@ from vllm.lora.request import LoRARequest
17
from vllm.prompt_adapter.request import PromptAdapterRequest
18
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
19
SequenceGroupMetadata, SequenceGroupMetadataDelta,
20
- SequenceStatus)
20
+ SequenceStage, SequenceStatus)
21
from vllm.utils import Device, PyObjectCache
22
23
logger = init_logger(__name__)
@@ -39,6 +39,7 @@ class PreemptionMode(enum.Enum):
39
recompute them when the sequences are resumed, treating the sequences as
40
new prompts.
41
"""
42
+
43
SWAP = enum.auto()
44
RECOMPUTE = enum.auto()
45
@@ -54,6 +55,7 @@ class SchedulingBudget:
55
happen if we only have chunked prefill scheduling, we can remove this
56
feature from the API when chunked prefill is enabled by default.
57
"""
58
+
59
token_budget: int
60
max_num_seqs: int
61
_request_ids_num_batched_tokens: Set[str] = field(default_factory=set)
@@ -132,6 +134,7 @@ class ScheduledSequenceGroup:
134
@dataclass
135
class SchedulerOutputs:
136
"""The scheduling decision made from a scheduler."""
137
+
138
# Scheduled sequence groups.
139
scheduled_seq_groups: GenericSequence[ScheduledSequenceGroup]
140
# Number of prefill groups scheduled.
@@ -205,6 +208,7 @@ class SchedulerRunningOutputs:
208
Could contain prefill (prefill that's chunked) or decodes. If there's not
209
enough memory, it can be preempted (for recompute) or swapped out.
210
"""
211
+
212
# Selected sequences that are running and in a decoding phase.
213
decode_seq_groups: List[ScheduledSequenceGroup]
214
# Selected sequences that are running and in a prefill phase.
@@ -246,6 +250,7 @@ class SchedulerSwappedInOutputs:
250
251
Could contain prefill (prefill that's chunked) or decodes.
252
"""
253
+
254
# Selected sequences that are going to be swapped in and is in a
255
# decoding phase.
256
decode_seq_groups: List[ScheduledSequenceGroup]
@@ -280,6 +285,7 @@ class SchedulerPrefillOutputs:
285
Could contain a fresh prefill requests or preempted requests that need
286
to be recomputed from scratch.
287
"""
288
+
289
# Selected sequences for prefill.
290
seq_groups: List[ScheduledSequenceGroup]
291
# Ignored sequence groups.
@@ -321,6 +327,100 @@ def scheduled_seq_group_builder():
327
# return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
328
329
330
+@dataclass
331
+class PartialPrefillMetadata:
332
+ """Holds information about the partial prefills that are currently running
333
+ during a single iteration of the Scheduler.
334
+ When chunked prefill is enabled, we allow a certain number of seqs to be
335
+ partially prefilled during each iteration. Having multiple partial prefills
336
+ in flight allows us to minimize TTFT and avoid decode starvation in cases
337
+ where a single sequence group with a very large prompt blocks the queue for
338
+ too many iterations.
339
+ The number of long prefill requests is limited so that smaller
340
+ requests may jump the queue in front of them and get to the decode
341
+ phase faster.
342
+ """
343
+
344
+ # A minimum bound on the total number of prefills to be scheduled during
345
+ # this iteration
346
+ schedulable_prefills: int
347
+
348
+ # The number of long prefill requests currently running
349
+ long_prefills: int
350
+
351
+ scheduler_config: SchedulerConfig
352
+
353
+ def can_schedule(self, seq_group: SequenceGroup) -> bool:
354
+ """When concurrent partial prefills are enabled,
355
+ we limit the number of long requests and only accept
356
+ shorter requests from the queue while running them
357
+ concurrently"""
358
+ return not (seq_group.first_seq.get_num_new_tokens()
359
+ > self.scheduler_config.long_prefill_token_threshold
360
+ and self.long_prefills
361
+ >= self.scheduler_config.max_long_partial_prefills
362
+ and self.scheduler_config.max_num_partial_prefills > 1)
363
+
364
+ def maybe_increment_partial_prefills(self,
365
+ seq_group: SequenceGroup) -> None:
366
+ # When a new prefill is scheduled, we need to know if it is a
367
+ # long request
368
+ if (seq_group.first_seq.get_num_new_tokens()
369
+ > self.scheduler_config.long_prefill_token_threshold):
370
+ self.long_prefills += 1
371
+
372
+ @classmethod
373
+ def from_queues(
374
+ cls,
375
+ running: Deque[SequenceGroup],
376
+ waiting: Deque[SequenceGroup],
377
+ scheduler_config: SchedulerConfig,
378
+ ) -> "PartialPrefillMetadata":
379
+ """Create a PartialPrefillMetadata object from the current state of
380
+ the scheduler's queues.
381
+ This accounts for the currently running prefill requests, and peeks into
382
+ the waiting queue to see if there are more prefills to potentially be
383
+ scheduled during this iteration."""
384
+ prefills = 0
385
+ long_prefills = 0
386
+
387
+ waiting_long_prefills = 0
388
+
389
+ for sg in running:
390
+ if sg.first_seq.data.stage == SequenceStage.PREFILL:
391
+ prefills += 1
392
+ if (sg.first_seq.get_num_new_tokens()
393
+ > scheduler_config.long_prefill_token_threshold):
394
+ long_prefills += 1
395
+
396
+ for sg in waiting:
397
+ # Don't bother looping through the rest of the queue if we know
398
+ # there are already at
399
+ # least max_partial_prefills requests to fill
400
+ if prefills >= scheduler_config.max_num_partial_prefills:
401
+ break
402
+
403
+ # Don't count long requests from the waiting queue if we aren't
404
+ # going to schedule them anyway
405
+ if (sg.first_seq.get_num_new_tokens()
406
+ > scheduler_config.long_prefill_token_threshold):
407
+ if (long_prefills + waiting_long_prefills
408
+ >= scheduler_config.max_long_partial_prefills):
409
+ continue
410
+ waiting_long_prefills += 1
411
+ prefills += 1
412
+
413
+ # NB: long_prefills and waiting_long_prefills are tracked separately.
414
+ # We don't account for the waiting requests here because we need to use
415
+ # this metadata to track how many have actually been scheduled.
416
+ return PartialPrefillMetadata(
417
+ schedulable_prefills=min(
418
+ prefills, scheduler_config.max_num_partial_prefills),
419
+ long_prefills=long_prefills,
420
+ scheduler_config=scheduler_config,
421
+ )
422
+
423
+
424
class Scheduler:
425
426
def __init__(
@@ -360,7 +460,8 @@ class Scheduler:
460
num_gpu_blocks=num_gpu_blocks,
461
num_cpu_blocks=num_cpu_blocks,
462
sliding_window=self.cache_config.sliding_window,
363
- enable_caching=self.cache_config.enable_prefix_caching)
463
+ enable_caching=self.cache_config.enable_prefix_caching,
464
+ )
465
466
# Sequence groups in the WAITING state.
467
# Contain new prefill or preempted requests.
@@ -421,6 +522,18 @@ class Scheduler:
522
# for processing and deallocation by the free_finished_seq_groups()
523
self._async_stopped: List[SequenceGroup] = []
524
525
+ # List with the chunk sizes to hand out to each sequence depending
526
+ # on how many partial prefills are running. This is slightly faster than
527
+ # running an integer division every time a prefill is scheduled.
528
+ # This splits the budget evenly among all prefills.
529
+ self.partial_prefill_budget_lookup_list = [0] * (
530
+ self.scheduler_config.max_num_partial_prefills + 1)
531
+ self.partial_prefill_budget_lookup_list[0] = (
532
+ scheduler_config.max_num_batched_tokens)
533
+ for i in range(1, self.scheduler_config.max_num_partial_prefills + 1):
534
+ self.partial_prefill_budget_lookup_list[i] = (
535
+ scheduler_config.max_num_batched_tokens // i)
536
+
537
@property
538
def next_cache_id(self):
539
return (self.cache_id + 1) % self.num_cache_iters
@@ -500,8 +613,8 @@ class Scheduler:
613
self.block_manager.free_cross(seq_group)
614
615
def has_unfinished_seqs(self) -> bool:
503
- return len(self.waiting) != 0 or len(self.running) != 0 or len(
504
- self.swapped) != 0
616
+ return (len(self.waiting) != 0 or len(self.running) != 0
617
+ or len(self.swapped) != 0)
618
619
def get_prefix_cache_hit_rate(self, device: Device) -> float:
620
return self.block_manager.get_prefix_cache_hit_rate(device)
@@ -523,6 +636,7 @@ class Scheduler:
636
budget: SchedulingBudget,
637
curr_loras: Optional[Set[int]],
638
enable_chunking: bool = False,
639
+ partial_prefill_metadata: Optional[PartialPrefillMetadata] = None,
640
) -> SchedulerRunningOutputs:
641
"""Schedule sequence groups that are running.
642
@@ -537,12 +651,14 @@ class Scheduler:
651
chunked number of tokens are scheduled if
652
`budget.num_batched_tokens` has not enough capacity to schedule
653
all tokens.
540
-
654
+ partial_prefill_metadata: information about the partial prefills
655
+ that are currently running
656
+
657
Returns:
658
SchedulerRunningOutputs.
659
"""
544
- ret: SchedulerRunningOutputs = \
545
- self._scheduler_running_outputs_cache[self.cache_id].get_object()
660
+ ret: SchedulerRunningOutputs = self._scheduler_running_outputs_cache[
661
+ self.cache_id].get_object()
662
ret.blocks_to_swap_out.clear()
663
ret.blocks_to_copy.clear()
664
ret.decode_seq_groups.clear()
@@ -577,10 +693,14 @@ class Scheduler:
693
# 2. If a sequence is running with non-chunked prefill, then
694
# there it's a decoding sequence, and the cached tokens info is
695
# irrelevant.
580
- num_uncached_new_tokens, _ = (
696
+ num_uncached_new_tokens, _ = \
697
self._get_num_new_uncached_and_cached_tokens(
582
- seq_group, SequenceStatus.RUNNING, enable_chunking,
583
- budget))
698
+ seq_group,
699
+ SequenceStatus.RUNNING,
700
+ enable_chunking,
701
+ budget,
702
+ partial_prefill_metadata,
703
+ )
704
705
num_running_tokens = num_uncached_new_tokens
706
if num_running_tokens == 0:
@@ -593,8 +713,8 @@ class Scheduler:
713
# to process the final tokens. The check below avoids this extra
714
# decode run when the model max len is reached, in order to avoid
715
# a memory overflow.
596
- if self.use_async_output_proc and seq_group.seqs[0].get_len(
597
- ) > self.scheduler_config.max_model_len:
716
+ if (self.use_async_output_proc and seq_group.seqs[0].get_len()
717
+ > self.scheduler_config.max_model_len):
718
self._async_stopped.append(seq_group)
719
continue
720
@@ -653,8 +773,9 @@ class Scheduler:
773
self._append_slots(seq_group, blocks_to_copy, enable_chunking)
774
is_prefill = seq_group.is_prefill()
775
656
- scheduled_seq_group: ScheduledSequenceGroup = \
657
- self._scheduled_seq_group_cache[self.cache_id].get_object()
776
+ scheduled_seq_group: ScheduledSequenceGroup = (
777
+ self._scheduled_seq_group_cache[
778
+ self.cache_id].get_object())
779
scheduled_seq_group.seq_group = seq_group
780
if is_prefill:
781
scheduled_seq_group.token_chunk_size = num_running_tokens
@@ -731,7 +852,8 @@ class Scheduler:
852
logger.warning(
853
"Failing the request %s because there's not enough kv "
854
"cache blocks to run the entire sequence.",
734
- seq_group.request_id)
855
+ seq_group.request_id,
856
+ )
857
for seq in seq_group.get_seqs():
858
seq.status = SequenceStatus.FINISHED_IGNORED
859
infeasible_seq_groups.append(seq_group)
@@ -800,16 +922,17 @@ class Scheduler:
922
)
923
924
def _get_prompt_limit(self, seq_group: SequenceGroup) -> int:
803
- if self.scheduler_config.chunked_prefill_enabled and \
804
- not self.scheduler_config.is_multi_step:
925
+ if (self.scheduler_config.chunked_prefill_enabled
926
+ and not self.scheduler_config.is_multi_step):
927
prompt_limit = self.scheduler_config.max_model_len
928
else:
807
- prompt_limit = min(self.scheduler_config.max_model_len,
808
- self.scheduler_config.max_num_batched_tokens)
929
+ prompt_limit = min(
930
+ self.scheduler_config.max_model_len,
931
+ self.scheduler_config.max_num_batched_tokens,
932
+ )
933
934
# Model is fine tuned with long context. Return the fine tuned max_len.
811
- if (seq_group.lora_request
812
- and seq_group.lora_request.long_lora_max_len):
935
+ if seq_group.lora_request and seq_group.lora_request.long_lora_max_len:
936
assert prompt_limit <= seq_group.lora_request.long_lora_max_len
937
return seq_group.lora_request.long_lora_max_len
938
else:
@@ -817,7 +940,7 @@ class Scheduler:
940
941
def _get_priority(self,
942
seq_group: SequenceGroup) -> Tuple[Optional[int], float]:
820
- """ Get the priority of the sequence group.
943
+ """Get the priority of the sequence group.
944
Highest preference to user-defined priority, followed by arrival time.
945
Args:
946
seq_group: The sequence group input.
@@ -850,14 +973,14 @@ class Scheduler:
973
if waiting_queue:
974
seq_group = waiting_queue.popleft()
975
num_new_seqs = seq_group.get_max_num_running_seqs()
853
- num_new_tokens_uncached, _ = (
976
+ num_new_tokens_uncached, _ = \
977
self._get_num_new_uncached_and_cached_tokens(
855
- seq_group, SequenceStatus.WAITING, False, budget))
978
+ seq_group, SequenceStatus.WAITING, False, budget)
979
857
- #Only preempt if priority inversion exists
980
+ # Only preempt if priority inversion exists
981
while running_queue and self._get_priority(
982
running_queue[-1]) > self._get_priority(seq_group):
860
- #Only preempt if waiting sequence cannot be allocated
983
+ # Only preempt if waiting sequence cannot be allocated
984
can_allocate = self.block_manager.can_allocate(seq_group)
985
if (num_new_tokens_uncached > 0
986
and can_allocate == AllocStatus.OK
@@ -867,7 +990,7 @@ class Scheduler:
990
)):
991
break
992
870
- #Adjust budget to remove the victim sequence group
993
+ # Adjust budget to remove the victim sequence group
994
vseq_group = running_queue.pop()
995
num_running_tokens_uncached, _ = (
996
self._get_num_new_uncached_and_cached_tokens(
@@ -878,11 +1001,11 @@ class Scheduler:
1001
budget.subtract_num_seqs(vseq_group.request_id,
1002
num_running_seqs)
1003
881
- #Preempt out the victim sequence group
1004
+ # Preempt out the victim sequence group
1005
self._preempt(vseq_group, blocks_to_swap_out)
1006
waiting_queue.appendleft(vseq_group)
1007
force_preemption_count += 1
885
- #Put the sequence back into the waiting queue
1008
+ # Put the sequence back into the waiting queue
1009
waiting_queue.appendleft(seq_group)
1010
1011
waiting_queue = deque(sorted(waiting_queue, key=self._get_priority))
@@ -896,6 +1019,7 @@ class Scheduler:
1019
budget: SchedulingBudget,
1020
curr_loras: Optional[Set[int]],
1021
enable_chunking: bool = False,
1022
+ partial_prefill_metadata: Optional[PartialPrefillMetadata] = None,
1023
) -> SchedulerPrefillOutputs:
1024
"""Schedule sequence groups that are in prefill stage.
1025
@@ -916,10 +1040,20 @@ class Scheduler:
1040
chunked number of tokens are scheduled if
1041
`budget.num_batched_tokens` has not enough capacity to schedule
1042
all tokens.
1043
+ partial_prefill_metadata: information about the partial prefills
1044
+ that are currently running
1045
1046
Returns:
1047
SchedulerPrefillOutputs.
1048
"""
1049
+ if budget.remaining_token_budget() == 0:
1050
+ # Do nothing: Can't add any more prefill anyway
1051
+ return SchedulerPrefillOutputs(
1052
+ seq_groups=[],
1053
+ ignored_seq_groups=[],
1054
+ num_lookahead_slots=self._get_num_lookahead_slots(
1055
+ is_prefill=True, enable_chunking=enable_chunking),
1056
+ )
1057
ignored_seq_groups: List[SequenceGroup] = []
1058
seq_groups: List[ScheduledSequenceGroup] = []
1059
@@ -933,10 +1067,19 @@ class Scheduler:
1067
assert len(waiting_seqs) == 1, (
1068
"Waiting sequence group should have only one prompt "
1069
"sequence.")
1070
+ if (partial_prefill_metadata is not None
1071
+ and not partial_prefill_metadata.can_schedule(seq_group)):
1072
+ leftover_waiting_sequences.appendleft(seq_group)
1073
+ waiting_queue.popleft()
1074
+ continue
1075
num_new_tokens_uncached, num_new_tokens_cached = (
1076
self._get_num_new_uncached_and_cached_tokens(
938
- seq_group, SequenceStatus.WAITING, enable_chunking,
939
- budget))
1077
+ seq_group,
1078
+ SequenceStatus.WAITING,
1079
+ enable_chunking,
1080
+ budget,
1081
+ partial_prefill_metadata=partial_prefill_metadata,
1082
+ ))
1083
num_new_tokens = num_new_tokens_uncached + num_new_tokens_cached
1084
1085
if not enable_chunking:
@@ -947,7 +1090,10 @@ class Scheduler:
1090
if num_new_tokens > prompt_limit:
1091
logger.warning(
1092
"Input prompt (%d tokens) is too long"
950
- " and exceeds limit of %d", num_new_tokens, prompt_limit)
1093
+ " and exceeds limit of %d",
1094
+ num_new_tokens,
1095
+ prompt_limit,
1096
+ )
1097
for seq in waiting_seqs:
1098
seq.status = SequenceStatus.FINISHED_IGNORED
1099
ignored_seq_groups.append(seq_group)
@@ -968,7 +1114,9 @@ class Scheduler:
1114
logger.warning(
1115
"Input prompt (%d tokens) + lookahead slots (%d) is "
1116
"too long and exceeds the capacity of block_manager",
971
- num_new_tokens, num_lookahead_slots)
1117
+ num_new_tokens,
1118
+ num_lookahead_slots,
1119
+ )
1120
for seq in waiting_seqs:
1121
seq.status = SequenceStatus.FINISHED_IGNORED
1122
ignored_seq_groups.append(seq_group)
@@ -1009,6 +1157,10 @@ class Scheduler:
1157
waiting_queue.popleft()
1158
self._allocate_and_set_running(seq_group)
1159
1160
+ if partial_prefill_metadata is not None:
1161
+ partial_prefill_metadata.maybe_increment_partial_prefills(
1162
+ seq_group)
1163
+
1164
if enable_chunking and self.scheduler_config.is_multi_step:
1165
blocks_to_copy: List[Tuple[int, int]] = []
1166
# init_multi_step_from_lookahead_slots happens in append_slots
@@ -1024,7 +1176,8 @@ class Scheduler:
1176
num_scheduler_steps=self.scheduler_config.
1177
num_scheduler_steps,
1178
is_multi_step=self.scheduler_config.is_multi_step,
1027
- enable_chunking=enable_chunking)
1179
+ enable_chunking=enable_chunking,
1180
+ )
1181
1182
seq_groups.append(
1183
ScheduledSequenceGroup(seq_group=seq_group,
@@ -1045,11 +1198,12 @@ class Scheduler:
1198
seq_groups=seq_groups,
1199
ignored_seq_groups=ignored_seq_groups,
1200
num_lookahead_slots=self._get_num_lookahead_slots(
1048
- is_prefill=True, enable_chunking=enable_chunking))
1201
+ is_prefill=True, enable_chunking=enable_chunking),
1202
+ )
1203
1204
def _schedule_default(self) -> SchedulerOutputs:
1205
"""Schedule queued requests.
1052
-
1206
+
1207
The current policy is designed to optimize the throughput. First,
1208
it batches as many prefill requests as possible. And it schedules
1209
decodes. If there's a pressure on GPU memory, decode requests can
@@ -1065,9 +1219,9 @@ class Scheduler:
1219
for seq_group in self.running:
1220
budget.add_num_seqs(seq_group.request_id,
1221
seq_group.get_max_num_running_seqs())
1068
- curr_loras = set(
1222
+ curr_loras = (set(
1223
seq_group.lora_int_id for seq_group in self.running
1070
- if seq_group.lora_int_id > 0) if self.lora_enabled else None
1224
+ if seq_group.lora_int_id > 0) if self.lora_enabled else None)
1225
1226
prefills = SchedulerPrefillOutputs.create_empty()
1227
running_scheduled = SchedulerRunningOutputs.create_empty()
@@ -1093,9 +1247,10 @@ class Scheduler:
1247
1248
# If any sequence group is preempted, do not swap in any sequence
1249
# group. because it means there's no slot for new running requests.
1096
- if len(running_scheduled.preempted) + len(
1097
- running_scheduled.swapped_out) == 0:
1098
- swapped_in = self._schedule_swapped(budget, curr_loras)
1250
+ if (len(running_scheduled.preempted) +
1251
+ len(running_scheduled.swapped_out) == 0):
1252
+ swapped_in = \
1253
+ self._schedule_swapped(budget, curr_loras)
1254
1255
assert (budget.num_batched_tokens
1256
<= self.scheduler_config.max_num_batched_tokens)
@@ -1115,8 +1270,8 @@ class Scheduler:
1270
1271
# Update swapped requests.
1272
self.swapped.extend(running_scheduled.swapped_out)
1118
- preempted = (len(running_scheduled.preempted) +
1119
- len(running_scheduled.swapped_out))
1273
+ preempted = len(running_scheduled.preempted) + len(
1274
+ running_scheduled.swapped_out)
1275
1276
# There should be no prefill from running queue because this policy
1277
# doesn't allow chunked prefills.
@@ -1154,7 +1309,7 @@ class Scheduler:
1309
1310
def _schedule_chunked_prefill(self) -> SchedulerOutputs:
1311
"""Schedule queued requests.
1157
-
1312
+
1313
Chunked prefill allows to chunk prefill requests, batch them together
1314
with decode requests. This policy 1. schedule as many decoding requests
1315
as possible. 2. schedule chunked prefill requests that are not
@@ -1175,10 +1330,20 @@ class Scheduler:
1330
prefills = SchedulerPrefillOutputs.create_empty()
1331
swapped_in = SchedulerSwappedInOutputs.create_empty()
1332
1333
+ # Create partial prefill metadata
1334
+ partial_prefill_metadata = PartialPrefillMetadata.from_queues(
1335
+ running=self.running,
1336
+ waiting=self.waiting,
1337
+ scheduler_config=self.scheduler_config,
1338
+ )
1339
+
1340
# Decoding should be always scheduled first by fcfs.
1179
- running_scheduled = self._schedule_running(budget,
1180
- curr_loras,
1181
- enable_chunking=True)
1341
+ running_scheduled = self._schedule_running(
1342
+ budget,
1343
+ curr_loras,
1344
+ enable_chunking=True,
1345
+ partial_prefill_metadata=partial_prefill_metadata,
1346
+ )
1347
1348
# Schedule swapped out requests.
1349
# If preemption happens, it means we don't have space for swap-in.
@@ -1186,9 +1351,12 @@ class Scheduler:
1351
running_scheduled.swapped_out) == 0:
1352
swapped_in = self._schedule_swapped(budget, curr_loras)
1353
1189
- prefills = self._schedule_prefills(budget,
1190
- curr_loras,
1191
- enable_chunking=True)
1354
+ prefills = self._schedule_prefills(
1355
+ budget,
1356
+ curr_loras,
1357
+ enable_chunking=True,
1358
+ partial_prefill_metadata=partial_prefill_metadata,
1359
+ )
1360
1361
assert (budget.num_batched_tokens
1362
<= self.scheduler_config.max_num_batched_tokens)
@@ -1207,8 +1375,15 @@ class Scheduler:
1375
[s.seq_group for s in swapped_in.prefill_seq_groups])
1376
self.running.extend(
1377
[s.seq_group for s in running_scheduled.decode_seq_groups])
1378
+ # Because multiple prefills may be running concurrently, we need to
1379
+ # make sure that prefills which are scheduled to finish are listed
1380
+ # before those that won't. This is so that on the next scheduling
1381
+ # iteration when they have transitioned to the decode stage, they are
1382
+ # properly prioritized over sequences that are still in the prefill
1383
+ # stage.
1384
self.running.extend(
1211
- [s.seq_group for s in running_scheduled.prefill_seq_groups])
1385
+ self._order_finishing_prefills_first(
1386
+ running_scheduled.prefill_seq_groups))
1387
self.running.extend([s.seq_group for s in prefills.seq_groups])
1388
1389
# Update swapped requests.
@@ -1225,7 +1400,7 @@ class Scheduler:
1400
# If all prompts, then we set num_lookahead_slots to 0
1401
# this allows us to go through the `no_spec` path in
1402
# `spec_decode_worker.py`
1228
- all_prefills = (len(scheduled_seq_groups) == num_prefill_groups)
1403
+ all_prefills = len(scheduled_seq_groups) == num_prefill_groups
1404
num_lookahead_slots = (0 if
1405
(all_prefills
1406
and not self.scheduler_config.is_multi_step)
@@ -1247,6 +1422,21 @@ class Scheduler:
1422
len(running_scheduled.swapped_out)),
1423
)
1424
1425
+ def _order_finishing_prefills_first(
1426
+ self, scheduled_prefill_seqs: List[ScheduledSequenceGroup]
1427
+ ) -> List[SequenceGroup]:
1428
+ """Returns a list of prefilling SequenceGroups where sequences that are
1429
+ scheduled to finish prefilling are listed first"""
1430
+ finishing = [
1431
+ s.seq_group for s in scheduled_prefill_seqs
1432
+ if s.seq_group.get_num_uncomputed_tokens() == s.token_chunk_size
1433
+ ]
1434
+ not_finishing = [
1435
+ s.seq_group for s in scheduled_prefill_seqs
1436
+ if s.seq_group.get_num_uncomputed_tokens() != s.token_chunk_size
1437
+ ]
1438
+ return finishing + not_finishing
1439
+
1440
def _schedule(self) -> SchedulerOutputs:
1441
"""Schedule queued requests."""
1442
if self.scheduler_config.chunked_prefill_enabled:
@@ -1385,10 +1575,12 @@ class Scheduler:
1575
# between engine and worker.
1576
# the subsequent comms can still use delta, but
1577
# `multi_modal_data` will be None.
1388
- multi_modal_data=seq_group.multi_modal_data
1389
- if scheduler_outputs.num_prefill_groups > 0 else None,
1390
- multi_modal_placeholders=seq_group.multi_modal_placeholders
1391
- if scheduler_outputs.num_prefill_groups > 0 else None,
1578
+ multi_modal_data=(seq_group.multi_modal_data
1579
+ if scheduler_outputs.num_prefill_groups
1580
+ > 0 else None),
1581
+ multi_modal_placeholders=(
1582
+ seq_group.multi_modal_placeholders
1583
+ if scheduler_outputs.num_prefill_groups > 0 else None),
1584
mm_processor_kwargs=seq_group.mm_processor_kwargs,
1585
prompt_adapter_request=seq_group.prompt_adapter_request,
1586
)
@@ -1494,10 +1686,12 @@ class Scheduler:
1686
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
1687
seq.status = SequenceStatus.RUNNING
1688
1497
- def _append_slots(self,
1498
- seq_group: SequenceGroup,
1499
- blocks_to_copy: List[Tuple[int, int]],
1500
- enable_chunking: bool = False) -> None:
1689
+ def _append_slots(
1690
+ self,
1691
+ seq_group: SequenceGroup,
1692
+ blocks_to_copy: List[Tuple[int, int]],
1693
+ enable_chunking: bool = False,
1694
+ ) -> None:
1695
"""Appends new slots to the sequences in the given sequence group.
1696
1697
Args:
@@ -1518,7 +1712,8 @@ class Scheduler:
1712
num_lookahead_slots,
1713
num_scheduler_steps=self.scheduler_config.num_scheduler_steps,
1714
is_multi_step=self.scheduler_config.is_multi_step,
1521
- enable_chunking=enable_chunking)
1715
+ enable_chunking=enable_chunking,
1716
+ )
1717
1718
seq_status: Optional[SequenceStatus] = SequenceStatus.RUNNING
1719
if self.scheduler_config.is_multi_step and enable_chunking:
@@ -1561,8 +1756,11 @@ class Scheduler:
1756
"not enough KV cache space. This can affect the end-to-end "
1757
"performance. Increase gpu_memory_utilization or "
1758
"tensor_parallel_size to provide more KV cache memory. "
1564
- "total_num_cumulative_preemption=%d", seq_group.request_id,
1565
- preemption_mode, self.num_cumulative_preemption + 1)
1759
+ "total_num_cumulative_preemption=%d",
1760
+ seq_group.request_id,
1761
+ preemption_mode,
1762
+ self.num_cumulative_preemption + 1,
1763
+ )
1764
self.num_cumulative_preemption += 1
1765
1766
if preemption_mode == PreemptionMode.RECOMPUTE:
@@ -1668,6 +1866,7 @@ class Scheduler:
1866
status: SequenceStatus,
1867
enable_chunking: bool,
1868
budget: SchedulingBudget,
1869
+ partial_prefill_metadata: Optional[PartialPrefillMetadata] = None,
1870
) -> Tuple[int, int]:
1871
"""
1872
Returns the number of new uncached and cached tokens to schedule for a
@@ -1691,6 +1890,8 @@ class Scheduler:
1890
to schedule.
1891
enable_chunking: Whether to chunk the number of tokens to compute.
1892
budget: The budget to chunk the number of tokens to compute.
1893
+ partial_prefill_metadata: information about the partial prefills
1894
+ that are currently running
1895
1896
1897
Returns:
@@ -1768,6 +1969,8 @@ class Scheduler:
1969
budget,
1970
self._get_prompt_limit(seq_group),
1971
num_uncached_new_tokens,
1972
+ self.partial_prefill_budget_lookup_list,
1973
+ partial_prefill_metadata,
1974
)
1975
1976
return num_uncached_new_tokens, num_cached_new_tokens
@@ -1779,6 +1982,8 @@ class Scheduler:
1982
budget: SchedulingBudget,
1983
prompt_limit: int,
1984
num_new_tokens: int,
1985
+ partial_prefill_budget_lookup_list: List[int],
1986
+ partial_prefill_metadata: Optional[PartialPrefillMetadata] = None,
1987
) -> int:
1988
"""
1989
Chunks the number of new tokens to schedule based on the budget when
@@ -1811,29 +2016,31 @@ class Scheduler:
2016
# the sequence.
2017
return num_new_tokens
2018
1814
- return (0 if num_new_tokens > remaining_token_budget else
1815
- num_new_tokens)
2019
+ return 0 if num_new_tokens > \
2020
+ remaining_token_budget else num_new_tokens
2021
1817
- if cache_config.enable_prefix_caching:
1818
- # Adjust the remaining token budget to be divisible by the block
1819
- # size when prefix caching is enabled.
2022
+ # Get the number of tokens to allocate to this prefill slot
2023
+ prefill_slot_budget = (
2024
+ remaining_token_budget if partial_prefill_metadata is None else
2025
+ partial_prefill_budget_lookup_list[
2026
+ partial_prefill_metadata.schedulable_prefills])
2027
1821
- # When prefix caching is enabled, we always allocate
1822
- # the number of new tokens that is dividable by the block
1823
- # size to avoid partial block matching.
2028
+ if cache_config.enable_prefix_caching:
2029
+ # When prefix caching is enabled and we're partially prefilling
2030
+ # a sequence, we always allocate a number of new tokens that is
2031
+ # divisible by the block size to avoid partial block matching.
2032
block_size = cache_config.block_size
1825
- remainder = budget.token_budget % block_size
1826
- if remainder != 0:
1827
- raise ValueError("When enabling chunked prefill and "
1828
- "prefix caching, max_num_batched_tokens "
1829
- "(chunk size) must be dividable by "
1830
- "block size, but got chunk_size "
1831
- f"({budget.token_budget}) % block_size "
1832
- f"({block_size}) = {remainder}")
1833
- # Round down to block size.
1834
- remaining_token_budget = (remaining_token_budget // block_size *
1835
- block_size)
1836
-
1837
- num_new_tokens = min(num_new_tokens, remaining_token_budget)
2033
+ # Don't exceed either the total budget or slot budget.
2034
+ # Take min of those and get the next lowest multiple of the
2035
+ # block size:
2036
+ remaining_token_budget = (
2037
+ min(remaining_token_budget, prefill_slot_budget) //
2038
+ block_size) * block_size
2039
+ # NB: In the case where num_new_tokens < budget, we are
2040
+ # finishing prefill for this sequence, so we do not need to
2041
+ # allocate a full block.
2042
+
2043
+ num_new_tokens = min(num_new_tokens, remaining_token_budget,
2044
+ prefill_slot_budget)
2045
2046
return num_new_tokens
vllm/engine/arg_utils.py
+33
-1
@@ -120,6 +120,9 @@ class EngineArgs:
120
cpu_offload_gb: float = 0 # GiB
121
gpu_memory_utilization: float = 0.90
122
max_num_batched_tokens: Optional[int] = None
123
+ max_num_partial_prefills: Optional[int] = 1
124
+ max_long_partial_prefills: Optional[int] = 1
125
+ long_prefill_token_threshold: Optional[int] = 0
126
max_num_seqs: Optional[int] = None
127
max_logprobs: int = 20 # Default value for OpenAI Chat Completions API
128
disable_log_stats: bool = False
@@ -515,6 +518,31 @@ class EngineArgs:
518
default=EngineArgs.max_num_batched_tokens,
519
help='Maximum number of batched tokens per '
520
'iteration.')
521
+ parser.add_argument(
522
+ "--max-num-partial-prefills",
523
+ type=int,
524
+ default=EngineArgs.max_num_partial_prefills,
525
+ help="For chunked prefill, the max number of concurrent \
526
+ partial prefills."
527
+ "Defaults to 1",
528
+ )
529
+ parser.add_argument(
530
+ "--max-long-partial-prefills",
531
+ type=int,
532
+ default=EngineArgs.max_long_partial_prefills,
533
+ help="For chunked prefill, the maximum number of prompts longer "
534
+ "than --long-prefill-token-threshold that will be prefilled "
535
+ "concurrently. Setting this less than --max-num-partial-prefills "
536
+ "will allow shorter prompts to jump the queue in front of longer "
537
+ "prompts in some cases, improving latency. Defaults to 1.")
538
+ parser.add_argument(
539
+ "--long-prefill-token-threshold",
540
+ type=float,
541
+ default=EngineArgs.long_prefill_token_threshold,
542
+ help="For chunked prefill, a request is considered long if the "
543
+ "prompt is longer than this number of tokens. Defaults to 4%% of "
544
+ "the model's context length.",
545
+ )
546
parser.add_argument('--max-num-seqs',
547
type=int,
548
default=EngineArgs.max_num_seqs,
@@ -1244,7 +1272,11 @@ class EngineArgs:
1272
multi_step_stream_outputs=self.multi_step_stream_outputs,
1273
send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
1274
and parallel_config.use_ray),
1247
- policy=self.scheduling_policy)
1275
+ policy=self.scheduling_policy,
1276
+ max_num_partial_prefills=self.max_num_partial_prefills,
1277
+ max_long_partial_prefills=self.max_long_partial_prefills,
1278
+ long_prefill_token_threshold=self.long_prefill_token_threshold,
1279
+ )
1280
lora_config = LoRAConfig(
1281
bias_enabled=self.enable_lora_bias,
1282
max_lora_rank=self.max_lora_rank,
vllm/model_executor/layers/sampler.py
+7
-1
@@ -958,7 +958,9 @@ def get_logprobs(
958
if len(query_indices) == 0:
959
empty_sampled_logprob: SampleLogprobs = []
960
empty_prompt_logprob: Optional[PromptLogprobs] = None
961
- return [empty_prompt_logprob], [empty_sampled_logprob]
961
+ num_seq_groups = len(sampling_metadata.seq_groups)
962
+ return [empty_prompt_logprob
963
+ ] * num_seq_groups, [empty_sampled_logprob] * num_seq_groups
964
965
selected_logprobs, ranks = None, None
966
top_logprobs, top_token_ids = None, None
@@ -1225,6 +1227,10 @@ def _build_sampler_output(
1227
assert sample_logprobs is not None
1228
assert not isinstance(maybe_deferred_sample_results,
1229
SampleResultArgsType)
1230
+ assert len(sampling_metadata.seq_groups) \
1231
+ == len(maybe_deferred_sample_results) \
1232
+ == len(prompt_logprobs) \
1233
+ == len(sample_logprobs)
1234
deferred_sample_results_args = None
1235
1236
for (seq_group, sample_result, group_prompt_logprobs,