@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3bcb8c75da
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Core] Reduce TTFT with concurrent partial prefills (#10235)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Joe Runde committed Feb 14, 2025 at 16:36 UTC 3bcb8c75da56e39cfd6e440402cf043a3d470b33
6 files changed +701 -108
tests/basic_correctness/test_chunked_prefill.py
+10 -20
@@ -8,7 +8,6 @@ prefill requests are chunked.
8 Run `pytest tests/models/test_chunked_prefill.py`.
9 """
10 import os
11 -from contextlib import nullcontext
11
12 import pytest
13
@@ -233,7 +232,6 @@ def test_with_prefix_caching(
232
233 max_num_batched_tokens = max_num_seqs = chunk_size
234 outputs = {} # type: ignore
236 - check_result = True
235 for enable in (True, False):
236 with vllm_runner(
237 model,
@@ -245,25 +243,17 @@ def test_with_prefix_caching(
243 enforce_eager=enforce_eager,
244 max_num_seqs=max_num_seqs,
245 ) as vllm_model:
248 - # It should fail when prefix caching is enable and chunk
249 - # size is not a multiple of block size (16).
250 - should_fail = chunk_size % 16 != 0 and enable
251 - check_result &= not should_fail
246 outputs[enable] = []
253 - # Send the request one-by-one to ensure the cache is populated.
254 - with pytest.raises(ValueError) if should_fail else nullcontext():
255 - for prompt in full_prompts:
256 - outputs[enable] += vllm_model.generate_greedy([prompt],
257 - max_tokens)
258 -
259 - # Check results only if we did not expect a failure.
260 - if check_result:
261 - check_outputs_equal(
262 - outputs_0_lst=outputs[False],
263 - outputs_1_lst=outputs[True],
264 - name_0="w/o prefix caching",
265 - name_1="with prefix caching",
266 - )
247 + for prompt in full_prompts:
248 + outputs[enable] += vllm_model.generate_greedy([prompt],
249 + max_tokens)
250 +
251 + check_outputs_equal(
252 + outputs_0_lst=outputs[False],
253 + outputs_1_lst=outputs[True],
254 + name_0="w/o prefix caching",
255 + name_1="with prefix caching",
256 + )
257
258
259 @pytest.mark.parametrize("model", ["facebook/opt-125m"])
tests/core/test_chunked_prefill_scheduler.py
+314 -2
@@ -7,6 +7,9 @@ import pytest # noqa
7
8 from vllm.config import CacheConfig, SchedulerConfig
9 from vllm.core.scheduler import Scheduler
10 +from vllm.engine.arg_utils import EngineArgs
11 +from vllm.engine.llm_engine import LLMEngine
12 +from vllm.sampling_params import SamplingParams
13 from vllm.sequence import Logprob, SequenceGroup
14
15 from .utils import create_dummy_prompt
@@ -16,7 +19,7 @@ def get_sequence_groups(scheduler_output):
19 return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
20
21
19 -def append_new_token(seq_group, token_id: int):
22 +def append_new_token(seq_group: SequenceGroup, token_id: int):
23 for seq in seq_group.get_seqs():
24 seq.append_token_id(token_id, {token_id: Logprob(token_id)})
25
@@ -123,6 +126,232 @@ def test_chunk():
126 assert out.num_batched_tokens == 57
127
128
129 +def test_concurrent_chunking():
130 + """Verify prefills are chunked properly when
131 + --max-num-partial-prefills is > 1"""
132 + block_size = 4
133 + max_seqs = 60
134 + max_model_len = 2000
135 + max_num_batched_tokens = 64
136 + scheduler_config = SchedulerConfig(
137 + "generate",
138 + max_num_batched_tokens,
139 + max_seqs,
140 + max_model_len,
141 + enable_chunked_prefill=True,
142 + max_num_partial_prefills=2, # Up to 2 partial prefills at a time
143 + )
144 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
145 + cache_config.num_cpu_blocks = 32
146 + cache_config.num_gpu_blocks = 32
147 + scheduler = Scheduler(scheduler_config, cache_config, None)
148 + running: List[SequenceGroup] = []
149 +
150 + # Add seq groups to scheduler.
151 + for i in range(2):
152 + _, seq_group = create_dummy_prompt(str(i),
153 + prompt_length=60,
154 + block_size=block_size)
155 + scheduler.add_seq_group(seq_group)
156 + running.append(seq_group)
157 +
158 + # Verify both requests are chunked with half of max_num_batched_tokens each
159 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
160 + assert set(get_sequence_groups(out)) == set(running)
161 + assert seq_group_meta[0].token_chunk_size == 32
162 + assert seq_group_meta[1].token_chunk_size == 32
163 + assert out.num_prefill_groups == 2
164 + assert out.num_batched_tokens == 64
165 +
166 + # After one iteration, both should have 60 - 32 = 28 tokens left to prefill
167 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
168 + assert set(get_sequence_groups(out)) == set(running)
169 + assert seq_group_meta[0].token_chunk_size == 28
170 + assert seq_group_meta[1].token_chunk_size == 28
171 + assert out.num_prefill_groups == 2
172 + assert out.num_batched_tokens == 56
173 +
174 +
175 +def test_concurrent_chunking_large_requests():
176 + """Verify large prefill requests are run one at a time"""
177 + block_size = 4
178 + max_seqs = 60
179 + max_model_len = 2000
180 + max_num_batched_tokens = 64
181 + scheduler_config = SchedulerConfig(
182 + "generate",
183 + max_num_batched_tokens,
184 + max_seqs,
185 + max_model_len,
186 + enable_chunked_prefill=True,
187 + max_num_partial_prefills=2, # Up to 2 partial prefills at a time
188 + )
189 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
190 + cache_config.num_cpu_blocks = 3200 # large KV cache size for large requests
191 + cache_config.num_gpu_blocks = 3200
192 + scheduler = Scheduler(scheduler_config, cache_config, None)
193 +
194 + # Add seq groups to scheduler.
195 + for i in range(2):
196 + _, seq_group = create_dummy_prompt(
197 + str(i),
198 + prompt_length=1200, # Very large prompt
199 + block_size=block_size)
200 + scheduler.add_seq_group(seq_group)
201 +
202 + # Verify only a single request is chunked, and it gets all 64 tokens
203 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
204 + assert len(get_sequence_groups(out)) == 1
205 + assert seq_group_meta[0].token_chunk_size == 64
206 + assert out.num_prefill_groups == 1
207 + assert out.num_batched_tokens == 64
208 +
209 +
210 +def test_short_prompts_jump_long_prompts_in_queue():
211 + """Verify large prefill requests are punted behind smaller ones if
212 + another large prefill request is already running"""
213 + block_size = 4
214 + max_seqs = 60
215 + max_model_len = 2000
216 + max_num_batched_tokens = 64
217 + scheduler_config = SchedulerConfig(
218 + "generate",
219 + max_num_batched_tokens,
220 + max_seqs,
221 + max_model_len,
222 + enable_chunked_prefill=True,
223 + max_num_partial_prefills=2, # Up to 2 partial prefills at a time
224 + )
225 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
226 + cache_config.num_cpu_blocks = 3200 # large KV cache size for large requests
227 + cache_config.num_gpu_blocks = 3200
228 + scheduler = Scheduler(scheduler_config, cache_config, None)
229 + long_seqs: List[SequenceGroup] = []
230 + short_seqs: List[SequenceGroup] = []
231 +
232 + # Add 2 large seq groups to scheduler.
233 + for i in range(2):
234 + _, seq_group = create_dummy_prompt(
235 + str(i),
236 + prompt_length=1200, # Very large prompt
237 + block_size=block_size)
238 + scheduler.add_seq_group(seq_group)
239 + long_seqs.append(seq_group)
240 + assert seq_group.is_prefill()
241 +
242 + # Add 2 small seq groups behind them
243 + for i in range(2):
244 + _, seq_group = create_dummy_prompt(
245 + str(i + 2),
246 + prompt_length=40, # Very small prompt
247 + block_size=block_size)
248 + scheduler.add_seq_group(seq_group)
249 + short_seqs.append(seq_group)
250 + assert seq_group.is_prefill()
251 +
252 + # Verify one large req and 1 small req chunked
253 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
254 + assert seq_group_meta[0].token_chunk_size == 32 # large req gets 32 tokens
255 + assert seq_group_meta[1].token_chunk_size == 32 # small req gets 32 tokens
256 +
257 + # all 4 are prefilling
258 + assert long_seqs[0].is_prefill()
259 + assert long_seqs[1].is_prefill()
260 + assert short_seqs[0].is_prefill()
261 + assert short_seqs[1].is_prefill()
262 + # First short and first long sequences have been scheduled
263 + assert long_seqs[0].first_seq.get_num_computed_tokens() == 32
264 + assert long_seqs[1].first_seq.get_num_computed_tokens() == 0
265 + assert short_seqs[0].first_seq.get_num_computed_tokens() == 32
266 + assert short_seqs[1].first_seq.get_num_computed_tokens() == 0
267 +
268 + assert out.num_prefill_groups == 2
269 + assert out.num_batched_tokens == 64
270 +
271 + # in the second iteration,
272 + # the first small request had only 8 tokens left
273 + # so it went to decode
274 + # The other small req is scheduled
275 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
276 + # the new small req got 64 - (32+8) tokens
277 + assert seq_group_meta[0].token_chunk_size == 24
278 + assert seq_group_meta[1].token_chunk_size == 32 # large req still got 32
279 + # the other small request had only 8 tokens left
280 + assert seq_group_meta[2].token_chunk_size == 8 # 40-32
281 +
282 + # The first small request got to decode now
283 + assert long_seqs[0].is_prefill()
284 + assert long_seqs[1].is_prefill()
285 + assert not short_seqs[0].is_prefill()
286 + assert short_seqs[1].is_prefill()
287 + # Both small requests have started in front of the second long request
288 + assert long_seqs[0].first_seq.get_num_computed_tokens() == 64
289 + assert long_seqs[1].first_seq.get_num_computed_tokens() == 0
290 + assert short_seqs[0].first_seq.get_num_computed_tokens() == 40
291 + assert short_seqs[1].first_seq.get_num_computed_tokens() == 24
292 +
293 + assert out.num_prefill_groups == 3
294 + assert out.num_batched_tokens == 64
295 + # the first small seq group has a new token appended.
296 + append_new_token(short_seqs[0], 1)
297 +
298 + # in the third iteration,
299 + # the first small request is already decoding
300 + # the second small request only has 16 tokens left and will enter decoding
301 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
302 + assert seq_group_meta[0].token_chunk_size == 32 # large still got 32
303 + # small req finished prefilling 40-24=16 tokens
304 + assert seq_group_meta[1].token_chunk_size == 16
305 + assert seq_group_meta[2].token_chunk_size == 1 # decode
306 + assert out.num_prefill_groups == 2
307 + assert out.num_batched_tokens == 49 # (32+16+1 decode)
308 +
309 + # both small requests have now reached decode
310 + assert long_seqs[0].is_prefill()
311 + assert long_seqs[1].is_prefill()
312 + assert not short_seqs[0].is_prefill()
313 + assert not short_seqs[1].is_prefill()
314 + assert long_seqs[0].first_seq.get_num_computed_tokens() == 96
315 + assert long_seqs[1].first_seq.get_num_computed_tokens() == 0
316 + assert short_seqs[0].first_seq.get_num_computed_tokens() == 41
317 + assert short_seqs[1].first_seq.get_num_computed_tokens() == 40
318 +
319 + # both the small seq groups have a new token appended
320 + append_new_token(short_seqs[0], 1)
321 + append_new_token(short_seqs[1], 1)
322 +
323 + # in the fourth iteration, both small requests are decoding
324 + # so large request gets all the budget
325 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
326 +
327 + # large req gets 62 tokens (minus 2 for decode)
328 + assert seq_group_meta[0].token_chunk_size == 62
329 + assert seq_group_meta[1].token_chunk_size == 1 # decode
330 + assert seq_group_meta[2].token_chunk_size == 1 # decode
331 + assert out.num_prefill_groups == 1
332 + assert out.num_batched_tokens == 64
333 +
334 + assert long_seqs[0].first_seq.get_num_computed_tokens() == 158
335 +
336 + # assert long_seqs[0].is_prefill()
337 + # assert long_seqs[1].is_prefill()
338 + # assert not short_seqs[0].is_prefill()
339 + # assert not short_seqs[1].is_prefill()
340 +
341 + # # both the small seq groups have a new token appended
342 + # append_new_token(short_seqs[0], 1)
343 + # append_new_token(short_seqs[1], 1)
344 +
345 + # # in the fifth iteration, large request gets all the budget
346 + # # while both small requests are decoding
347 + # seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
348 + # assert seq_group_meta[0].token_chunk_size == 62
349 + # assert seq_group_meta[1].token_chunk_size == 1 # decode
350 + # assert seq_group_meta[2].token_chunk_size == 1 # decode
351 + # assert out.num_prefill_groups == 1
352 + # assert out.num_batched_tokens == 64
353 +
354 +
355 def test_complex():
356 block_size = 4
357 max_seqs = 60
@@ -508,7 +737,7 @@ def test_chunked_prefill_max_seqs():
737 assert not running[1].is_prefill()
738
739
511 -def test_perfix_caching():
740 +def test_prefix_caching():
741 """Verify allocating full blocks when prefix caching is enabled."""
742 block_size = 4
743 max_seqs = 10
@@ -548,3 +777,86 @@ def test_perfix_caching():
777 assert seq_group_meta[1].token_chunk_size == 12
778 assert out.num_prefill_groups == 2
779 assert out.num_batched_tokens == 62
780 +
781 +
782 +def test_prefix_caching_with_concurrent_partial_prefills():
783 + """Verify allocating full blocks when prefix caching is enabled with
784 + --max-num-partial-prefills > 1."""
785 + block_size = 4
786 + max_seqs = 10
787 + max_model_len = 8000
788 + max_num_batched_tokens = 60 # With two slots, each slot will get 30 tokens
789 + scheduler_config = SchedulerConfig("generate",
790 + max_num_batched_tokens,
791 + max_seqs,
792 + max_model_len,
793 + enable_chunked_prefill=True,
794 + max_num_partial_prefills=2)
795 + cache_config = CacheConfig(block_size,
796 + 1.0,
797 + 1,
798 + "auto",
799 + enable_prefix_caching=True)
800 + cache_config.num_cpu_blocks = 0
801 + cache_config.num_gpu_blocks = 32
802 + scheduler = Scheduler(scheduler_config, cache_config, None)
803 + running: List[SequenceGroup] = []
804 +
805 + # Add seq groups to scheduler.
806 + for i in range(2):
807 + _, seq_group = create_dummy_prompt(str(i),
808 + block_size=block_size,
809 + prompt_length=50)
810 + scheduler.add_seq_group(seq_group)
811 + running.append(seq_group)
812 +
813 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
814 + assert set(get_sequence_groups(out)) == set(running)
815 + # To partially prefill both sequences, both can chunk up to 30 tokens
816 + # But the next lowest multiple of the block size (4) is 28
817 + assert seq_group_meta[0].token_chunk_size == 28
818 + assert seq_group_meta[1].token_chunk_size == 28
819 + assert out.num_prefill_groups == 2
820 + assert out.num_batched_tokens == 56
821 +
822 + # On the next iteration, both sequences should finish prefill
823 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
824 + assert set(get_sequence_groups(out)) == set(running)
825 + # Both sequences have 50 - 28 = 22 tokens left to prefill.
826 + # This is not a multiple of the block size, but we don't care since we don't
827 + # cache the final partial block of prefix sequences
828 + assert seq_group_meta[0].token_chunk_size == 22
829 + assert seq_group_meta[1].token_chunk_size == 22
830 + assert out.num_prefill_groups == 2
831 + assert out.num_batched_tokens == 44
832 +
833 +
834 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
835 +@pytest.mark.parametrize("max_num_partial_prefills", [2, 4, 8])
836 +def test_chunked_prefill_with_actual_engine(model: str,
837 + max_num_partial_prefills: int):
838 + """Make sure the model can actually sample with concurrent
839 + partial prefills
840 + """
841 +
842 + prompt = "hello" * 40
843 +
844 + engine_args = EngineArgs(
845 + model=model,
846 + max_num_partial_prefills=max_num_partial_prefills,
847 + max_num_batched_tokens=40,
848 + max_num_seqs=8,
849 + enable_chunked_prefill=True,
850 + gpu_memory_utilization=0.8,
851 + )
852 +
853 + engine = LLMEngine.from_engine_args(engine_args)
854 + sampling_params = SamplingParams(temperature=0)
855 +
856 + for req_num in range(max_num_partial_prefills):
857 + engine.add_request(f"{req_num}", prompt, sampling_params)
858 + # first step
859 + request_outputs = engine.step()
860 + # means all are prefilling
861 + assert len(request_outputs) == 0
862 + assert len(engine.scheduler[0].running) == max_num_partial_prefills
vllm/config.py
+46
@@ -1430,6 +1430,17 @@ class SchedulerConfig:
1430 # Maximum length of a sequence (including prompt and generated text).
1431 max_model_len: int = 8192
1432
1433 + # Maximum number of sequences that can be partially prefilled concurrently
1434 + max_num_partial_prefills: int = 1
1435 +
1436 + # Maximum number of "very long prompt" sequences that can be prefilled
1437 + # concurrently (long is defined by long_prefill_threshold)
1438 + max_long_partial_prefills: int = 1
1439 +
1440 + # calculate context length that determines which sequences are
1441 + # considered "long"
1442 + long_prefill_token_threshold: int = 0
1443 +
1444 # The number of slots to allocate per sequence per
1445 # step, beyond the known token ids. This is used in speculative
1446 # decoding to store KV activations of tokens which may or may not be
@@ -1537,6 +1548,18 @@ class SchedulerConfig:
1548 self.max_num_batched_tokens)
1549
1550 self.chunked_prefill_enabled = self.enable_chunked_prefill
1551 + if self.max_num_partial_prefills > 1:
1552 + if self.long_prefill_token_threshold == 0:
1553 + self.long_prefill_token_threshold = int(self.max_model_len *
1554 + 0.04)
1555 +
1556 + logger.info(
1557 + "Concurrent partial prefills enabled with "
1558 + "max_num_partial_prefills=%d, max_long_partial_prefills=%d, "
1559 + "long_prefill_token_threshold=%d",
1560 + self.max_num_partial_prefills, self.max_long_partial_prefills,
1561 + self.long_prefill_token_threshold)
1562 +
1563 self._verify_args()
1564
1565 def _verify_args(self) -> None:
@@ -1568,6 +1591,29 @@ class SchedulerConfig:
1591 f"({self.num_scheduler_steps}) must be greater than or "
1592 "equal to 1.")
1593
1594 + if self.max_num_partial_prefills < 1:
1595 + raise ValueError(
1596 + f"max_num_partial_prefills ({self.max_num_partial_prefills}) "
1597 + "must be greater than or equal to 1.")
1598 + elif self.max_num_partial_prefills > 1:
1599 + if not self.chunked_prefill_enabled:
1600 + raise ValueError("Chunked prefill must be enabled to set "
1601 + "max_num_partial_prefills > 1.")
1602 +
1603 + if self.long_prefill_token_threshold > self.max_model_len:
1604 + raise ValueError(
1605 + "long_prefill_token_threshold "
1606 + f"({self.long_prefill_token_threshold}) cannot be greater "
1607 + f"than the max_model_len ({self.max_model_len}).")
1608 +
1609 + if (self.max_long_partial_prefills
1610 + < 1) or (self.max_long_partial_prefills
1611 + > self.max_num_partial_prefills):
1612 + raise ValueError(
1613 + f"max_long_partial_prefills ({self.max_long_partial_prefills}) "
1614 + "must be greater than or equal to 1 and less than or equal to "
1615 + f"max_num_partial_prefills ({self.max_num_partial_prefills}).")
1616 +
1617 @property
1618 def is_multi_step(self) -> bool:
1619 return self.num_scheduler_steps > 1
vllm/core/scheduler.py
+291 -84
@@ -17,7 +17,7 @@ from vllm.lora.request import LoRARequest
17 from vllm.prompt_adapter.request import PromptAdapterRequest
18 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
19 SequenceGroupMetadata, SequenceGroupMetadataDelta,
20 - SequenceStatus)
20 + SequenceStage, SequenceStatus)
21 from vllm.utils import Device, PyObjectCache
22
23 logger = init_logger(__name__)
@@ -39,6 +39,7 @@ class PreemptionMode(enum.Enum):
39 recompute them when the sequences are resumed, treating the sequences as
40 new prompts.
41 """
42 +
43 SWAP = enum.auto()
44 RECOMPUTE = enum.auto()
45
@@ -54,6 +55,7 @@ class SchedulingBudget:
55 happen if we only have chunked prefill scheduling, we can remove this
56 feature from the API when chunked prefill is enabled by default.
57 """
58 +
59 token_budget: int
60 max_num_seqs: int
61 _request_ids_num_batched_tokens: Set[str] = field(default_factory=set)
@@ -132,6 +134,7 @@ class ScheduledSequenceGroup:
134 @dataclass
135 class SchedulerOutputs:
136 """The scheduling decision made from a scheduler."""
137 +
138 # Scheduled sequence groups.
139 scheduled_seq_groups: GenericSequence[ScheduledSequenceGroup]
140 # Number of prefill groups scheduled.
@@ -205,6 +208,7 @@ class SchedulerRunningOutputs:
208 Could contain prefill (prefill that's chunked) or decodes. If there's not
209 enough memory, it can be preempted (for recompute) or swapped out.
210 """
211 +
212 # Selected sequences that are running and in a decoding phase.
213 decode_seq_groups: List[ScheduledSequenceGroup]
214 # Selected sequences that are running and in a prefill phase.
@@ -246,6 +250,7 @@ class SchedulerSwappedInOutputs:
250
251 Could contain prefill (prefill that's chunked) or decodes.
252 """
253 +
254 # Selected sequences that are going to be swapped in and is in a
255 # decoding phase.
256 decode_seq_groups: List[ScheduledSequenceGroup]
@@ -280,6 +285,7 @@ class SchedulerPrefillOutputs:
285 Could contain a fresh prefill requests or preempted requests that need
286 to be recomputed from scratch.
287 """
288 +
289 # Selected sequences for prefill.
290 seq_groups: List[ScheduledSequenceGroup]
291 # Ignored sequence groups.
@@ -321,6 +327,100 @@ def scheduled_seq_group_builder():
327 # return ScheduledSequenceGroup(seq_group=None, token_chunk_size=0)
328
329
330 +@dataclass
331 +class PartialPrefillMetadata:
332 + """Holds information about the partial prefills that are currently running
333 + during a single iteration of the Scheduler.
334 + When chunked prefill is enabled, we allow a certain number of seqs to be
335 + partially prefilled during each iteration. Having multiple partial prefills
336 + in flight allows us to minimize TTFT and avoid decode starvation in cases
337 + where a single sequence group with a very large prompt blocks the queue for
338 + too many iterations.
339 + The number of long prefill requests is limited so that smaller
340 + requests may jump the queue in front of them and get to the decode
341 + phase faster.
342 + """
343 +
344 + # A minimum bound on the total number of prefills to be scheduled during
345 + # this iteration
346 + schedulable_prefills: int
347 +
348 + # The number of long prefill requests currently running
349 + long_prefills: int
350 +
351 + scheduler_config: SchedulerConfig
352 +
353 + def can_schedule(self, seq_group: SequenceGroup) -> bool:
354 + """When concurrent partial prefills are enabled,
355 + we limit the number of long requests and only accept
356 + shorter requests from the queue while running them
357 + concurrently"""
358 + return not (seq_group.first_seq.get_num_new_tokens()
359 + > self.scheduler_config.long_prefill_token_threshold
360 + and self.long_prefills
361 + >= self.scheduler_config.max_long_partial_prefills
362 + and self.scheduler_config.max_num_partial_prefills > 1)
363 +
364 + def maybe_increment_partial_prefills(self,
365 + seq_group: SequenceGroup) -> None:
366 + # When a new prefill is scheduled, we need to know if it is a
367 + # long request
368 + if (seq_group.first_seq.get_num_new_tokens()
369 + > self.scheduler_config.long_prefill_token_threshold):
370 + self.long_prefills += 1
371 +
372 + @classmethod
373 + def from_queues(
374 + cls,
375 + running: Deque[SequenceGroup],
376 + waiting: Deque[SequenceGroup],
377 + scheduler_config: SchedulerConfig,
378 + ) -> "PartialPrefillMetadata":
379 + """Create a PartialPrefillMetadata object from the current state of
380 + the scheduler's queues.
381 + This accounts for the currently running prefill requests, and peeks into
382 + the waiting queue to see if there are more prefills to potentially be
383 + scheduled during this iteration."""
384 + prefills = 0
385 + long_prefills = 0
386 +
387 + waiting_long_prefills = 0
388 +
389 + for sg in running:
390 + if sg.first_seq.data.stage == SequenceStage.PREFILL:
391 + prefills += 1
392 + if (sg.first_seq.get_num_new_tokens()
393 + > scheduler_config.long_prefill_token_threshold):
394 + long_prefills += 1
395 +
396 + for sg in waiting:
397 + # Don't bother looping through the rest of the queue if we know
398 + # there are already at
399 + # least max_partial_prefills requests to fill
400 + if prefills >= scheduler_config.max_num_partial_prefills:
401 + break
402 +
403 + # Don't count long requests from the waiting queue if we aren't
404 + # going to schedule them anyway
405 + if (sg.first_seq.get_num_new_tokens()
406 + > scheduler_config.long_prefill_token_threshold):
407 + if (long_prefills + waiting_long_prefills
408 + >= scheduler_config.max_long_partial_prefills):
409 + continue
410 + waiting_long_prefills += 1
411 + prefills += 1
412 +
413 + # NB: long_prefills and waiting_long_prefills are tracked separately.
414 + # We don't account for the waiting requests here because we need to use
415 + # this metadata to track how many have actually been scheduled.
416 + return PartialPrefillMetadata(
417 + schedulable_prefills=min(
418 + prefills, scheduler_config.max_num_partial_prefills),
419 + long_prefills=long_prefills,
420 + scheduler_config=scheduler_config,
421 + )
422 +
423 +
424 class Scheduler:
425
426 def __init__(
@@ -360,7 +460,8 @@ class Scheduler:
460 num_gpu_blocks=num_gpu_blocks,
461 num_cpu_blocks=num_cpu_blocks,
462 sliding_window=self.cache_config.sliding_window,
363 - enable_caching=self.cache_config.enable_prefix_caching)
463 + enable_caching=self.cache_config.enable_prefix_caching,
464 + )
465
466 # Sequence groups in the WAITING state.
467 # Contain new prefill or preempted requests.
@@ -421,6 +522,18 @@ class Scheduler:
522 # for processing and deallocation by the free_finished_seq_groups()
523 self._async_stopped: List[SequenceGroup] = []
524
525 + # List with the chunk sizes to hand out to each sequence depending
526 + # on how many partial prefills are running. This is slightly faster than
527 + # running an integer division every time a prefill is scheduled.
528 + # This splits the budget evenly among all prefills.
529 + self.partial_prefill_budget_lookup_list = [0] * (
530 + self.scheduler_config.max_num_partial_prefills + 1)
531 + self.partial_prefill_budget_lookup_list[0] = (
532 + scheduler_config.max_num_batched_tokens)
533 + for i in range(1, self.scheduler_config.max_num_partial_prefills + 1):
534 + self.partial_prefill_budget_lookup_list[i] = (
535 + scheduler_config.max_num_batched_tokens // i)
536 +
537 @property
538 def next_cache_id(self):
539 return (self.cache_id + 1) % self.num_cache_iters
@@ -500,8 +613,8 @@ class Scheduler:
613 self.block_manager.free_cross(seq_group)
614
615 def has_unfinished_seqs(self) -> bool:
503 - return len(self.waiting) != 0 or len(self.running) != 0 or len(
504 - self.swapped) != 0
616 + return (len(self.waiting) != 0 or len(self.running) != 0
617 + or len(self.swapped) != 0)
618
619 def get_prefix_cache_hit_rate(self, device: Device) -> float:
620 return self.block_manager.get_prefix_cache_hit_rate(device)
@@ -523,6 +636,7 @@ class Scheduler:
636 budget: SchedulingBudget,
637 curr_loras: Optional[Set[int]],
638 enable_chunking: bool = False,
639 + partial_prefill_metadata: Optional[PartialPrefillMetadata] = None,
640 ) -> SchedulerRunningOutputs:
641 """Schedule sequence groups that are running.
642
@@ -537,12 +651,14 @@ class Scheduler:
651 chunked number of tokens are scheduled if
652 `budget.num_batched_tokens` has not enough capacity to schedule
653 all tokens.
540 -
654 + partial_prefill_metadata: information about the partial prefills
655 + that are currently running
656 +
657 Returns:
658 SchedulerRunningOutputs.
659 """
544 - ret: SchedulerRunningOutputs = \
545 - self._scheduler_running_outputs_cache[self.cache_id].get_object()
660 + ret: SchedulerRunningOutputs = self._scheduler_running_outputs_cache[
661 + self.cache_id].get_object()
662 ret.blocks_to_swap_out.clear()
663 ret.blocks_to_copy.clear()
664 ret.decode_seq_groups.clear()
@@ -577,10 +693,14 @@ class Scheduler:
693 # 2. If a sequence is running with non-chunked prefill, then
694 # there it's a decoding sequence, and the cached tokens info is
695 # irrelevant.
580 - num_uncached_new_tokens, _ = (
696 + num_uncached_new_tokens, _ = \
697 self._get_num_new_uncached_and_cached_tokens(
582 - seq_group, SequenceStatus.RUNNING, enable_chunking,
583 - budget))
698 + seq_group,
699 + SequenceStatus.RUNNING,
700 + enable_chunking,
701 + budget,
702 + partial_prefill_metadata,
703 + )
704
705 num_running_tokens = num_uncached_new_tokens
706 if num_running_tokens == 0:
@@ -593,8 +713,8 @@ class Scheduler:
713 # to process the final tokens. The check below avoids this extra
714 # decode run when the model max len is reached, in order to avoid
715 # a memory overflow.
596 - if self.use_async_output_proc and seq_group.seqs[0].get_len(
597 - ) > self.scheduler_config.max_model_len:
716 + if (self.use_async_output_proc and seq_group.seqs[0].get_len()
717 + > self.scheduler_config.max_model_len):
718 self._async_stopped.append(seq_group)
719 continue
720
@@ -653,8 +773,9 @@ class Scheduler:
773 self._append_slots(seq_group, blocks_to_copy, enable_chunking)
774 is_prefill = seq_group.is_prefill()
775
656 - scheduled_seq_group: ScheduledSequenceGroup = \
657 - self._scheduled_seq_group_cache[self.cache_id].get_object()
776 + scheduled_seq_group: ScheduledSequenceGroup = (
777 + self._scheduled_seq_group_cache[
778 + self.cache_id].get_object())
779 scheduled_seq_group.seq_group = seq_group
780 if is_prefill:
781 scheduled_seq_group.token_chunk_size = num_running_tokens
@@ -731,7 +852,8 @@ class Scheduler:
852 logger.warning(
853 "Failing the request %s because there's not enough kv "
854 "cache blocks to run the entire sequence.",
734 - seq_group.request_id)
855 + seq_group.request_id,
856 + )
857 for seq in seq_group.get_seqs():
858 seq.status = SequenceStatus.FINISHED_IGNORED
859 infeasible_seq_groups.append(seq_group)
@@ -800,16 +922,17 @@ class Scheduler:
922 )
923
924 def _get_prompt_limit(self, seq_group: SequenceGroup) -> int:
803 - if self.scheduler_config.chunked_prefill_enabled and \
804 - not self.scheduler_config.is_multi_step:
925 + if (self.scheduler_config.chunked_prefill_enabled
926 + and not self.scheduler_config.is_multi_step):
927 prompt_limit = self.scheduler_config.max_model_len
928 else:
807 - prompt_limit = min(self.scheduler_config.max_model_len,
808 - self.scheduler_config.max_num_batched_tokens)
929 + prompt_limit = min(
930 + self.scheduler_config.max_model_len,
931 + self.scheduler_config.max_num_batched_tokens,
932 + )
933
934 # Model is fine tuned with long context. Return the fine tuned max_len.
811 - if (seq_group.lora_request
812 - and seq_group.lora_request.long_lora_max_len):
935 + if seq_group.lora_request and seq_group.lora_request.long_lora_max_len:
936 assert prompt_limit <= seq_group.lora_request.long_lora_max_len
937 return seq_group.lora_request.long_lora_max_len
938 else:
@@ -817,7 +940,7 @@ class Scheduler:
940
941 def _get_priority(self,
942 seq_group: SequenceGroup) -> Tuple[Optional[int], float]:
820 - """ Get the priority of the sequence group.
943 + """Get the priority of the sequence group.
944 Highest preference to user-defined priority, followed by arrival time.
945 Args:
946 seq_group: The sequence group input.
@@ -850,14 +973,14 @@ class Scheduler:
973 if waiting_queue:
974 seq_group = waiting_queue.popleft()
975 num_new_seqs = seq_group.get_max_num_running_seqs()
853 - num_new_tokens_uncached, _ = (
976 + num_new_tokens_uncached, _ = \
977 self._get_num_new_uncached_and_cached_tokens(
855 - seq_group, SequenceStatus.WAITING, False, budget))
978 + seq_group, SequenceStatus.WAITING, False, budget)
979
857 - #Only preempt if priority inversion exists
980 + # Only preempt if priority inversion exists
981 while running_queue and self._get_priority(
982 running_queue[-1]) > self._get_priority(seq_group):
860 - #Only preempt if waiting sequence cannot be allocated
983 + # Only preempt if waiting sequence cannot be allocated
984 can_allocate = self.block_manager.can_allocate(seq_group)
985 if (num_new_tokens_uncached > 0
986 and can_allocate == AllocStatus.OK
@@ -867,7 +990,7 @@ class Scheduler:
990 )):
991 break
992
870 - #Adjust budget to remove the victim sequence group
993 + # Adjust budget to remove the victim sequence group
994 vseq_group = running_queue.pop()
995 num_running_tokens_uncached, _ = (
996 self._get_num_new_uncached_and_cached_tokens(
@@ -878,11 +1001,11 @@ class Scheduler:
1001 budget.subtract_num_seqs(vseq_group.request_id,
1002 num_running_seqs)
1003
881 - #Preempt out the victim sequence group
1004 + # Preempt out the victim sequence group
1005 self._preempt(vseq_group, blocks_to_swap_out)
1006 waiting_queue.appendleft(vseq_group)
1007 force_preemption_count += 1
885 - #Put the sequence back into the waiting queue
1008 + # Put the sequence back into the waiting queue
1009 waiting_queue.appendleft(seq_group)
1010
1011 waiting_queue = deque(sorted(waiting_queue, key=self._get_priority))
@@ -896,6 +1019,7 @@ class Scheduler:
1019 budget: SchedulingBudget,
1020 curr_loras: Optional[Set[int]],
1021 enable_chunking: bool = False,
1022 + partial_prefill_metadata: Optional[PartialPrefillMetadata] = None,
1023 ) -> SchedulerPrefillOutputs:
1024 """Schedule sequence groups that are in prefill stage.
1025
@@ -916,10 +1040,20 @@ class Scheduler:
1040 chunked number of tokens are scheduled if
1041 `budget.num_batched_tokens` has not enough capacity to schedule
1042 all tokens.
1043 + partial_prefill_metadata: information about the partial prefills
1044 + that are currently running
1045
1046 Returns:
1047 SchedulerPrefillOutputs.
1048 """
1049 + if budget.remaining_token_budget() == 0:
1050 + # Do nothing: Can't add any more prefill anyway
1051 + return SchedulerPrefillOutputs(
1052 + seq_groups=[],
1053 + ignored_seq_groups=[],
1054 + num_lookahead_slots=self._get_num_lookahead_slots(
1055 + is_prefill=True, enable_chunking=enable_chunking),
1056 + )
1057 ignored_seq_groups: List[SequenceGroup] = []
1058 seq_groups: List[ScheduledSequenceGroup] = []
1059
@@ -933,10 +1067,19 @@ class Scheduler:
1067 assert len(waiting_seqs) == 1, (
1068 "Waiting sequence group should have only one prompt "
1069 "sequence.")
1070 + if (partial_prefill_metadata is not None
1071 + and not partial_prefill_metadata.can_schedule(seq_group)):
1072 + leftover_waiting_sequences.appendleft(seq_group)
1073 + waiting_queue.popleft()
1074 + continue
1075 num_new_tokens_uncached, num_new_tokens_cached = (
1076 self._get_num_new_uncached_and_cached_tokens(
938 - seq_group, SequenceStatus.WAITING, enable_chunking,
939 - budget))
1077 + seq_group,
1078 + SequenceStatus.WAITING,
1079 + enable_chunking,
1080 + budget,
1081 + partial_prefill_metadata=partial_prefill_metadata,
1082 + ))
1083 num_new_tokens = num_new_tokens_uncached + num_new_tokens_cached
1084
1085 if not enable_chunking:
@@ -947,7 +1090,10 @@ class Scheduler:
1090 if num_new_tokens > prompt_limit:
1091 logger.warning(
1092 "Input prompt (%d tokens) is too long"
950 - " and exceeds limit of %d", num_new_tokens, prompt_limit)
1093 + " and exceeds limit of %d",
1094 + num_new_tokens,
1095 + prompt_limit,
1096 + )
1097 for seq in waiting_seqs:
1098 seq.status = SequenceStatus.FINISHED_IGNORED
1099 ignored_seq_groups.append(seq_group)
@@ -968,7 +1114,9 @@ class Scheduler:
1114 logger.warning(
1115 "Input prompt (%d tokens) + lookahead slots (%d) is "
1116 "too long and exceeds the capacity of block_manager",
971 - num_new_tokens, num_lookahead_slots)
1117 + num_new_tokens,
1118 + num_lookahead_slots,
1119 + )
1120 for seq in waiting_seqs:
1121 seq.status = SequenceStatus.FINISHED_IGNORED
1122 ignored_seq_groups.append(seq_group)
@@ -1009,6 +1157,10 @@ class Scheduler:
1157 waiting_queue.popleft()
1158 self._allocate_and_set_running(seq_group)
1159
1160 + if partial_prefill_metadata is not None:
1161 + partial_prefill_metadata.maybe_increment_partial_prefills(
1162 + seq_group)
1163 +
1164 if enable_chunking and self.scheduler_config.is_multi_step:
1165 blocks_to_copy: List[Tuple[int, int]] = []
1166 # init_multi_step_from_lookahead_slots happens in append_slots
@@ -1024,7 +1176,8 @@ class Scheduler:
1176 num_scheduler_steps=self.scheduler_config.
1177 num_scheduler_steps,
1178 is_multi_step=self.scheduler_config.is_multi_step,
1027 - enable_chunking=enable_chunking)
1179 + enable_chunking=enable_chunking,
1180 + )
1181
1182 seq_groups.append(
1183 ScheduledSequenceGroup(seq_group=seq_group,
@@ -1045,11 +1198,12 @@ class Scheduler:
1198 seq_groups=seq_groups,
1199 ignored_seq_groups=ignored_seq_groups,
1200 num_lookahead_slots=self._get_num_lookahead_slots(
1048 - is_prefill=True, enable_chunking=enable_chunking))
1201 + is_prefill=True, enable_chunking=enable_chunking),
1202 + )
1203
1204 def _schedule_default(self) -> SchedulerOutputs:
1205 """Schedule queued requests.
1052 -
1206 +
1207 The current policy is designed to optimize the throughput. First,
1208 it batches as many prefill requests as possible. And it schedules
1209 decodes. If there's a pressure on GPU memory, decode requests can
@@ -1065,9 +1219,9 @@ class Scheduler:
1219 for seq_group in self.running:
1220 budget.add_num_seqs(seq_group.request_id,
1221 seq_group.get_max_num_running_seqs())
1068 - curr_loras = set(
1222 + curr_loras = (set(
1223 seq_group.lora_int_id for seq_group in self.running
1070 - if seq_group.lora_int_id > 0) if self.lora_enabled else None
1224 + if seq_group.lora_int_id > 0) if self.lora_enabled else None)
1225
1226 prefills = SchedulerPrefillOutputs.create_empty()
1227 running_scheduled = SchedulerRunningOutputs.create_empty()
@@ -1093,9 +1247,10 @@ class Scheduler:
1247
1248 # If any sequence group is preempted, do not swap in any sequence
1249 # group. because it means there's no slot for new running requests.
1096 - if len(running_scheduled.preempted) + len(
1097 - running_scheduled.swapped_out) == 0:
1098 - swapped_in = self._schedule_swapped(budget, curr_loras)
1250 + if (len(running_scheduled.preempted) +
1251 + len(running_scheduled.swapped_out) == 0):
1252 + swapped_in = \
1253 + self._schedule_swapped(budget, curr_loras)
1254
1255 assert (budget.num_batched_tokens
1256 <= self.scheduler_config.max_num_batched_tokens)
@@ -1115,8 +1270,8 @@ class Scheduler:
1270
1271 # Update swapped requests.
1272 self.swapped.extend(running_scheduled.swapped_out)
1118 - preempted = (len(running_scheduled.preempted) +
1119 - len(running_scheduled.swapped_out))
1273 + preempted = len(running_scheduled.preempted) + len(
1274 + running_scheduled.swapped_out)
1275
1276 # There should be no prefill from running queue because this policy
1277 # doesn't allow chunked prefills.
@@ -1154,7 +1309,7 @@ class Scheduler:
1309
1310 def _schedule_chunked_prefill(self) -> SchedulerOutputs:
1311 """Schedule queued requests.
1157 -
1312 +
1313 Chunked prefill allows to chunk prefill requests, batch them together
1314 with decode requests. This policy 1. schedule as many decoding requests
1315 as possible. 2. schedule chunked prefill requests that are not
@@ -1175,10 +1330,20 @@ class Scheduler:
1330 prefills = SchedulerPrefillOutputs.create_empty()
1331 swapped_in = SchedulerSwappedInOutputs.create_empty()
1332
1333 + # Create partial prefill metadata
1334 + partial_prefill_metadata = PartialPrefillMetadata.from_queues(
1335 + running=self.running,
1336 + waiting=self.waiting,
1337 + scheduler_config=self.scheduler_config,
1338 + )
1339 +
1340 # Decoding should be always scheduled first by fcfs.
1179 - running_scheduled = self._schedule_running(budget,
1180 - curr_loras,
1181 - enable_chunking=True)
1341 + running_scheduled = self._schedule_running(
1342 + budget,
1343 + curr_loras,
1344 + enable_chunking=True,
1345 + partial_prefill_metadata=partial_prefill_metadata,
1346 + )
1347
1348 # Schedule swapped out requests.
1349 # If preemption happens, it means we don't have space for swap-in.
@@ -1186,9 +1351,12 @@ class Scheduler:
1351 running_scheduled.swapped_out) == 0:
1352 swapped_in = self._schedule_swapped(budget, curr_loras)
1353
1189 - prefills = self._schedule_prefills(budget,
1190 - curr_loras,
1191 - enable_chunking=True)
1354 + prefills = self._schedule_prefills(
1355 + budget,
1356 + curr_loras,
1357 + enable_chunking=True,
1358 + partial_prefill_metadata=partial_prefill_metadata,
1359 + )
1360
1361 assert (budget.num_batched_tokens
1362 <= self.scheduler_config.max_num_batched_tokens)
@@ -1207,8 +1375,15 @@ class Scheduler:
1375 [s.seq_group for s in swapped_in.prefill_seq_groups])
1376 self.running.extend(
1377 [s.seq_group for s in running_scheduled.decode_seq_groups])
1378 + # Because multiple prefills may be running concurrently, we need to
1379 + # make sure that prefills which are scheduled to finish are listed
1380 + # before those that won't. This is so that on the next scheduling
1381 + # iteration when they have transitioned to the decode stage, they are
1382 + # properly prioritized over sequences that are still in the prefill
1383 + # stage.
1384 self.running.extend(
1211 - [s.seq_group for s in running_scheduled.prefill_seq_groups])
1385 + self._order_finishing_prefills_first(
1386 + running_scheduled.prefill_seq_groups))
1387 self.running.extend([s.seq_group for s in prefills.seq_groups])
1388
1389 # Update swapped requests.
@@ -1225,7 +1400,7 @@ class Scheduler:
1400 # If all prompts, then we set num_lookahead_slots to 0
1401 # this allows us to go through the `no_spec` path in
1402 # `spec_decode_worker.py`
1228 - all_prefills = (len(scheduled_seq_groups) == num_prefill_groups)
1403 + all_prefills = len(scheduled_seq_groups) == num_prefill_groups
1404 num_lookahead_slots = (0 if
1405 (all_prefills
1406 and not self.scheduler_config.is_multi_step)
@@ -1247,6 +1422,21 @@ class Scheduler:
1422 len(running_scheduled.swapped_out)),
1423 )
1424
1425 + def _order_finishing_prefills_first(
1426 + self, scheduled_prefill_seqs: List[ScheduledSequenceGroup]
1427 + ) -> List[SequenceGroup]:
1428 + """Returns a list of prefilling SequenceGroups where sequences that are
1429 + scheduled to finish prefilling are listed first"""
1430 + finishing = [
1431 + s.seq_group for s in scheduled_prefill_seqs
1432 + if s.seq_group.get_num_uncomputed_tokens() == s.token_chunk_size
1433 + ]
1434 + not_finishing = [
1435 + s.seq_group for s in scheduled_prefill_seqs
1436 + if s.seq_group.get_num_uncomputed_tokens() != s.token_chunk_size
1437 + ]
1438 + return finishing + not_finishing
1439 +
1440 def _schedule(self) -> SchedulerOutputs:
1441 """Schedule queued requests."""
1442 if self.scheduler_config.chunked_prefill_enabled:
@@ -1385,10 +1575,12 @@ class Scheduler:
1575 # between engine and worker.
1576 # the subsequent comms can still use delta, but
1577 # `multi_modal_data` will be None.
1388 - multi_modal_data=seq_group.multi_modal_data
1389 - if scheduler_outputs.num_prefill_groups > 0 else None,
1390 - multi_modal_placeholders=seq_group.multi_modal_placeholders
1391 - if scheduler_outputs.num_prefill_groups > 0 else None,
1578 + multi_modal_data=(seq_group.multi_modal_data
1579 + if scheduler_outputs.num_prefill_groups
1580 + > 0 else None),
1581 + multi_modal_placeholders=(
1582 + seq_group.multi_modal_placeholders
1583 + if scheduler_outputs.num_prefill_groups > 0 else None),
1584 mm_processor_kwargs=seq_group.mm_processor_kwargs,
1585 prompt_adapter_request=seq_group.prompt_adapter_request,
1586 )
@@ -1494,10 +1686,12 @@ class Scheduler:
1686 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
1687 seq.status = SequenceStatus.RUNNING
1688
1497 - def _append_slots(self,
1498 - seq_group: SequenceGroup,
1499 - blocks_to_copy: List[Tuple[int, int]],
1500 - enable_chunking: bool = False) -> None:
1689 + def _append_slots(
1690 + self,
1691 + seq_group: SequenceGroup,
1692 + blocks_to_copy: List[Tuple[int, int]],
1693 + enable_chunking: bool = False,
1694 + ) -> None:
1695 """Appends new slots to the sequences in the given sequence group.
1696
1697 Args:
@@ -1518,7 +1712,8 @@ class Scheduler:
1712 num_lookahead_slots,
1713 num_scheduler_steps=self.scheduler_config.num_scheduler_steps,
1714 is_multi_step=self.scheduler_config.is_multi_step,
1521 - enable_chunking=enable_chunking)
1715 + enable_chunking=enable_chunking,
1716 + )
1717
1718 seq_status: Optional[SequenceStatus] = SequenceStatus.RUNNING
1719 if self.scheduler_config.is_multi_step and enable_chunking:
@@ -1561,8 +1756,11 @@ class Scheduler:
1756 "not enough KV cache space. This can affect the end-to-end "
1757 "performance. Increase gpu_memory_utilization or "
1758 "tensor_parallel_size to provide more KV cache memory. "
1564 - "total_num_cumulative_preemption=%d", seq_group.request_id,
1565 - preemption_mode, self.num_cumulative_preemption + 1)
1759 + "total_num_cumulative_preemption=%d",
1760 + seq_group.request_id,
1761 + preemption_mode,
1762 + self.num_cumulative_preemption + 1,
1763 + )
1764 self.num_cumulative_preemption += 1
1765
1766 if preemption_mode == PreemptionMode.RECOMPUTE:
@@ -1668,6 +1866,7 @@ class Scheduler:
1866 status: SequenceStatus,
1867 enable_chunking: bool,
1868 budget: SchedulingBudget,
1869 + partial_prefill_metadata: Optional[PartialPrefillMetadata] = None,
1870 ) -> Tuple[int, int]:
1871 """
1872 Returns the number of new uncached and cached tokens to schedule for a
@@ -1691,6 +1890,8 @@ class Scheduler:
1890 to schedule.
1891 enable_chunking: Whether to chunk the number of tokens to compute.
1892 budget: The budget to chunk the number of tokens to compute.
1893 + partial_prefill_metadata: information about the partial prefills
1894 + that are currently running
1895
1896
1897 Returns:
@@ -1768,6 +1969,8 @@ class Scheduler:
1969 budget,
1970 self._get_prompt_limit(seq_group),
1971 num_uncached_new_tokens,
1972 + self.partial_prefill_budget_lookup_list,
1973 + partial_prefill_metadata,
1974 )
1975
1976 return num_uncached_new_tokens, num_cached_new_tokens
@@ -1779,6 +1982,8 @@ class Scheduler:
1982 budget: SchedulingBudget,
1983 prompt_limit: int,
1984 num_new_tokens: int,
1985 + partial_prefill_budget_lookup_list: List[int],
1986 + partial_prefill_metadata: Optional[PartialPrefillMetadata] = None,
1987 ) -> int:
1988 """
1989 Chunks the number of new tokens to schedule based on the budget when
@@ -1811,29 +2016,31 @@ class Scheduler:
2016 # the sequence.
2017 return num_new_tokens
2018
1814 - return (0 if num_new_tokens > remaining_token_budget else
1815 - num_new_tokens)
2019 + return 0 if num_new_tokens > \
2020 + remaining_token_budget else num_new_tokens
2021
1817 - if cache_config.enable_prefix_caching:
1818 - # Adjust the remaining token budget to be divisible by the block
1819 - # size when prefix caching is enabled.
2022 + # Get the number of tokens to allocate to this prefill slot
2023 + prefill_slot_budget = (
2024 + remaining_token_budget if partial_prefill_metadata is None else
2025 + partial_prefill_budget_lookup_list[
2026 + partial_prefill_metadata.schedulable_prefills])
2027
1821 - # When prefix caching is enabled, we always allocate
1822 - # the number of new tokens that is dividable by the block
1823 - # size to avoid partial block matching.
2028 + if cache_config.enable_prefix_caching:
2029 + # When prefix caching is enabled and we're partially prefilling
2030 + # a sequence, we always allocate a number of new tokens that is
2031 + # divisible by the block size to avoid partial block matching.
2032 block_size = cache_config.block_size
1825 - remainder = budget.token_budget % block_size
1826 - if remainder != 0:
1827 - raise ValueError("When enabling chunked prefill and "
1828 - "prefix caching, max_num_batched_tokens "
1829 - "(chunk size) must be dividable by "
1830 - "block size, but got chunk_size "
1831 - f"({budget.token_budget}) % block_size "
1832 - f"({block_size}) = {remainder}")
1833 - # Round down to block size.
1834 - remaining_token_budget = (remaining_token_budget // block_size *
1835 - block_size)
1836 -
1837 - num_new_tokens = min(num_new_tokens, remaining_token_budget)
2033 + # Don't exceed either the total budget or slot budget.
2034 + # Take min of those and get the next lowest multiple of the
2035 + # block size:
2036 + remaining_token_budget = (
2037 + min(remaining_token_budget, prefill_slot_budget) //
2038 + block_size) * block_size
2039 + # NB: In the case where num_new_tokens < budget, we are
2040 + # finishing prefill for this sequence, so we do not need to
2041 + # allocate a full block.
2042 +
2043 + num_new_tokens = min(num_new_tokens, remaining_token_budget,
2044 + prefill_slot_budget)
2045
2046 return num_new_tokens
vllm/engine/arg_utils.py
+33 -1
@@ -120,6 +120,9 @@ class EngineArgs:
120 cpu_offload_gb: float = 0 # GiB
121 gpu_memory_utilization: float = 0.90
122 max_num_batched_tokens: Optional[int] = None
123 + max_num_partial_prefills: Optional[int] = 1
124 + max_long_partial_prefills: Optional[int] = 1
125 + long_prefill_token_threshold: Optional[int] = 0
126 max_num_seqs: Optional[int] = None
127 max_logprobs: int = 20 # Default value for OpenAI Chat Completions API
128 disable_log_stats: bool = False
@@ -515,6 +518,31 @@ class EngineArgs:
518 default=EngineArgs.max_num_batched_tokens,
519 help='Maximum number of batched tokens per '
520 'iteration.')
521 + parser.add_argument(
522 + "--max-num-partial-prefills",
523 + type=int,
524 + default=EngineArgs.max_num_partial_prefills,
525 + help="For chunked prefill, the max number of concurrent \
526 + partial prefills."
527 + "Defaults to 1",
528 + )
529 + parser.add_argument(
530 + "--max-long-partial-prefills",
531 + type=int,
532 + default=EngineArgs.max_long_partial_prefills,
533 + help="For chunked prefill, the maximum number of prompts longer "
534 + "than --long-prefill-token-threshold that will be prefilled "
535 + "concurrently. Setting this less than --max-num-partial-prefills "
536 + "will allow shorter prompts to jump the queue in front of longer "
537 + "prompts in some cases, improving latency. Defaults to 1.")
538 + parser.add_argument(
539 + "--long-prefill-token-threshold",
540 + type=float,
541 + default=EngineArgs.long_prefill_token_threshold,
542 + help="For chunked prefill, a request is considered long if the "
543 + "prompt is longer than this number of tokens. Defaults to 4%% of "
544 + "the model's context length.",
545 + )
546 parser.add_argument('--max-num-seqs',
547 type=int,
548 default=EngineArgs.max_num_seqs,
@@ -1244,7 +1272,11 @@ class EngineArgs:
1272 multi_step_stream_outputs=self.multi_step_stream_outputs,
1273 send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
1274 and parallel_config.use_ray),
1247 - policy=self.scheduling_policy)
1275 + policy=self.scheduling_policy,
1276 + max_num_partial_prefills=self.max_num_partial_prefills,
1277 + max_long_partial_prefills=self.max_long_partial_prefills,
1278 + long_prefill_token_threshold=self.long_prefill_token_threshold,
1279 + )
1280 lora_config = LoRAConfig(
1281 bias_enabled=self.enable_lora_bias,
1282 max_lora_rank=self.max_lora_rank,
vllm/model_executor/layers/sampler.py
+7 -1
@@ -958,7 +958,9 @@ def get_logprobs(
958 if len(query_indices) == 0:
959 empty_sampled_logprob: SampleLogprobs = []
960 empty_prompt_logprob: Optional[PromptLogprobs] = None
961 - return [empty_prompt_logprob], [empty_sampled_logprob]
961 + num_seq_groups = len(sampling_metadata.seq_groups)
962 + return [empty_prompt_logprob
963 + ] * num_seq_groups, [empty_sampled_logprob] * num_seq_groups
964
965 selected_logprobs, ranks = None, None
966 top_logprobs, top_token_ids = None, None
@@ -1225,6 +1227,10 @@ def _build_sampler_output(
1227 assert sample_logprobs is not None
1228 assert not isinstance(maybe_deferred_sample_results,
1229 SampleResultArgsType)
1230 + assert len(sampling_metadata.seq_groups) \
1231 + == len(maybe_deferred_sample_results) \
1232 + == len(prompt_logprobs) \
1233 + == len(sample_logprobs)
1234 deferred_sample_results_args = None
1235
1236 for (seq_group, sample_result, group_prompt_logprobs,