@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18de883489
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Chunked Prefill][4/n] Chunked prefill scheduler. (#3853)

SangBin Cho committed Apr 6, 2024 at 02:17 UTC 18de88348954b7e535a62c0b7e55004f253e9f21
10 files changed +1218 -183
requirements-common.txt
+1 -1
@@ -11,4 +11,4 @@ uvicorn[standard]
11 pydantic >= 2.0 # Required for OpenAI server.
12 prometheus_client >= 0.18.0
13 tiktoken == 0.6.0 # Required for DBRX tokenizer
14 -outlines == 0.0.34 # Requires torch >= 2.1.0
14 +outlines == 0.0.34 # Requires torch >= 2.1.0
\ No newline at end of file
tests/core/test_chunked_prefill_scheduler.py new
+563
@@ -0,0 +1,563 @@
1 +from typing import List
2 +from unittest.mock import MagicMock
3 +
4 +import pytest # noqa
5 +
6 +from vllm.config import CacheConfig, SchedulerConfig
7 +from vllm.core.scheduler import Scheduler
8 +from vllm.sequence import Logprob, SequenceGroup
9 +
10 +from .utils import create_dummy_prompt
11 +
12 +
13 +def get_sequence_groups(scheduler_output):
14 + return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
15 +
16 +
17 +def append_new_token(seq_group, token_id: int):
18 + for seq in seq_group.get_seqs():
19 + seq.append_token_id(token_id, {token_id: Logprob(token_id)})
20 +
21 +
22 +def schedule_and_update_computed_tokens(scheduler):
23 + metas, out = scheduler.schedule()
24 + for s, meta in zip(out.scheduled_seq_groups, metas):
25 + s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
26 + return metas, out
27 +
28 +
29 +def test_simple():
30 + """Verify basic scheduling works."""
31 + block_size = 4
32 + num_seq_group = 4
33 + max_model_len = 16
34 + max_num_batched_tokens = 64
35 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
36 + num_seq_group,
37 + max_model_len,
38 + enable_chunked_prefill=True)
39 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
40 + cache_config.num_cpu_blocks = 8
41 + cache_config.num_gpu_blocks = 8
42 + scheduler = Scheduler(scheduler_config, cache_config, None)
43 + running: List[SequenceGroup] = []
44 +
45 + # Add seq groups to scheduler.
46 + for i in range(num_seq_group):
47 + _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
48 + scheduler.add_seq_group(seq_group)
49 + running.append(seq_group)
50 +
51 + # Schedule seq groups prompts.
52 + num_tokens = block_size * num_seq_group
53 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
54 + assert set(get_sequence_groups(out)) == set(running)
55 + assert out.num_batched_tokens == num_tokens
56 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
57 + and not out.blocks_to_swap_out)
58 + assert len(seq_group_meta) == num_seq_group
59 + for s in running:
60 + append_new_token(s, 1)
61 +
62 + # Schedule seq groups generation.
63 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
64 + assert set(get_sequence_groups(out)) == set(running)
65 + assert out.num_batched_tokens == num_seq_group
66 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
67 + and not out.blocks_to_swap_out)
68 + assert len(seq_group_meta) == num_seq_group
69 +
70 +
71 +def test_chunk():
72 + """Verify prefills are chunked properly."""
73 + block_size = 4
74 + max_seqs = 60
75 + max_model_len = 80
76 + max_num_batched_tokens = 64
77 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
78 + max_seqs,
79 + max_model_len,
80 + enable_chunked_prefill=True)
81 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
82 + cache_config.num_cpu_blocks = 8
83 + cache_config.num_gpu_blocks = 8
84 + scheduler = Scheduler(scheduler_config, cache_config, None)
85 + running: List[SequenceGroup] = []
86 +
87 + # Add seq groups to scheduler.
88 + for i in range(2):
89 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
90 + scheduler.add_seq_group(seq_group)
91 + running.append(seq_group)
92 +
93 + # Verify the second request is chunked.
94 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
95 + assert set(get_sequence_groups(out)) == set(running)
96 + assert seq_group_meta[0].token_chunk_size == 60
97 + # Verify it is chunked.
98 + assert seq_group_meta[1].token_chunk_size == 4
99 + assert out.num_prefill_groups == 2
100 + assert out.num_batched_tokens == 64
101 + # Only the first seq group has a new token appended.
102 + append_new_token(running[0], 1)
103 +
104 + # One chunked prefill, and one decoding.
105 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
106 + assert set(get_sequence_groups(out)) == set(running)
107 + # The first one is decoding.
108 + assert seq_group_meta[0].token_chunk_size == 1
109 + # The second one is a chunked prefill.
110 + assert seq_group_meta[1].token_chunk_size == 56
111 + assert out.num_prefill_groups == 1
112 + assert out.num_batched_tokens == 57
113 +
114 +
115 +def test_complex():
116 + block_size = 4
117 + max_seqs = 60
118 + max_model_len = 80
119 + max_num_batched_tokens = 64
120 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
121 + max_seqs,
122 + max_model_len,
123 + enable_chunked_prefill=True)
124 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
125 + cache_config.num_cpu_blocks = 8
126 + cache_config.num_gpu_blocks = 8
127 + scheduler = Scheduler(scheduler_config, cache_config, None)
128 + running: List[SequenceGroup] = []
129 +
130 + # Add seq groups to scheduler.
131 + for i in range(2):
132 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
133 + scheduler.add_seq_group(seq_group)
134 + running.append(seq_group)
135 + assert seq_group.is_prefill()
136 +
137 + # Verify the second request is chunked.
138 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
139 +
140 + assert set(get_sequence_groups(out)) == set(running)
141 + assert seq_group_meta[0].token_chunk_size == 60
142 + # Verify it is chunked.
143 + assert seq_group_meta[1].token_chunk_size == 4
144 + assert not running[0].is_prefill()
145 + assert running[1].is_prefill()
146 + assert out.num_prefill_groups == 2
147 + assert out.num_batched_tokens == 64
148 + # Only the first seq group has a new token appended.
149 + append_new_token(running[0], 1)
150 +
151 + # Add 2 more requsets.
152 + for i in range(2, 4):
153 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
154 + scheduler.add_seq_group(seq_group)
155 + running.append(seq_group)
156 +
157 + # Decoding & chunked prefill & first chunk of 3rd request is scheduled.
158 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
159 + assert len(get_sequence_groups(out)) == 3
160 + # The first one is decoding.
161 + assert seq_group_meta[0].token_chunk_size == 1
162 + # The second one is a chunked prefill.
163 + assert seq_group_meta[1].token_chunk_size == 56
164 + # The third one is also chunked.
165 + assert seq_group_meta[2].token_chunk_size == 7
166 + # Two of them are in chunked prefill.
167 + assert out.num_prefill_groups == 2
168 + assert out.num_batched_tokens == 64
169 + # The first 2 requests are now in decodine phase.
170 + append_new_token(running[0], 1)
171 + assert not running[0].is_prefill()
172 + append_new_token(running[1], 1)
173 + assert not running[1].is_prefill()
174 + # The third request is still in prefill stage.
175 + assert running[2].is_prefill()
176 +
177 +
178 +def test_maximal_decoding():
179 + """Verify decoding requests are prioritized."""
180 + block_size = 4
181 + max_seqs = 2
182 + max_model_len = 2
183 + max_num_batched_tokens = 2
184 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
185 + max_seqs,
186 + max_model_len,
187 + enable_chunked_prefill=True)
188 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
189 + cache_config.num_cpu_blocks = 8
190 + cache_config.num_gpu_blocks = 8
191 + scheduler = Scheduler(scheduler_config, cache_config, None)
192 + running: List[SequenceGroup] = []
193 +
194 + # Add seq groups to scheduler.
195 + for i in range(2):
196 + _, seq_group = create_dummy_prompt(str(i), prompt_length=2)
197 + scheduler.add_seq_group(seq_group)
198 + running.append(seq_group)
199 + assert seq_group.is_prefill()
200 +
201 + # The first prefill is scheduled.
202 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
203 + assert len(get_sequence_groups(out)) == 1
204 + assert seq_group_meta[0].token_chunk_size == 2
205 + assert not running[0].is_prefill()
206 + assert running[1].is_prefill()
207 + assert out.num_prefill_groups == 1
208 + assert out.num_batched_tokens == 2
209 + # Only the first seq group has a new token appended.
210 + append_new_token(running[0], 1)
211 +
212 + # Create one more seq_group.
213 + _, seq_group = create_dummy_prompt("3", prompt_length=2)
214 + scheduler.add_seq_group(seq_group)
215 + running.append(seq_group)
216 + assert seq_group.is_prefill()
217 + # The first decoding + second chunk is scheduled.
218 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
219 + assert len(get_sequence_groups(out)) == 2
220 + assert seq_group_meta[0].token_chunk_size == 1
221 + assert seq_group_meta[1].token_chunk_size == 1
222 + assert not running[0].is_prefill()
223 + assert running[1].is_prefill()
224 + assert running[2].is_prefill()
225 + assert out.num_prefill_groups == 1
226 + assert out.num_batched_tokens == 2
227 + append_new_token(running[0], 1)
228 +
229 + # Decoding + running prefill is prioritized.
230 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
231 + assert len(get_sequence_groups(out)) == 2
232 + assert seq_group_meta[0].token_chunk_size == 1
233 + assert seq_group_meta[1].token_chunk_size == 1
234 + assert not running[0].is_prefill()
235 + assert not running[1].is_prefill()
236 + assert out.num_prefill_groups == 1
237 + assert out.num_batched_tokens == 2
238 + append_new_token(running[0], 1)
239 + append_new_token(running[1], 1)
240 +
241 + # Only decoding is prioritized.
242 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
243 + assert len(get_sequence_groups(out)) == 2
244 + assert seq_group_meta[0].token_chunk_size == 1
245 + assert seq_group_meta[1].token_chunk_size == 1
246 + assert not running[0].is_prefill()
247 + assert not running[1].is_prefill()
248 + assert out.num_prefill_groups == 0
249 + assert out.num_batched_tokens == 2
250 + append_new_token(running[0], 1)
251 + append_new_token(running[1], 1)
252 +
253 + # After aborting the decoding request, the fcfs new prefill is prioritized.
254 + scheduler.abort_seq_group(running[0].request_id)
255 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
256 + assert len(get_sequence_groups(out)) == 2
257 + assert seq_group_meta[0].token_chunk_size == 1
258 + assert seq_group_meta[1].token_chunk_size == 1
259 + assert not running[1].is_prefill()
260 + assert running[2].is_prefill()
261 + assert out.num_prefill_groups == 1
262 + assert out.num_batched_tokens == 2
263 +
264 +
265 +def test_prompt_limit():
266 + """Verify max_num_batched_tokens < max_model_len is possible."""
267 + block_size = 4
268 + max_seqs = 32
269 + max_model_len = 64
270 + max_num_batched_tokens = 32
271 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
272 + max_seqs,
273 + max_model_len,
274 + enable_chunked_prefill=True)
275 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
276 + cache_config.num_cpu_blocks = 8
277 + cache_config.num_gpu_blocks = 8
278 + scheduler = Scheduler(scheduler_config, cache_config, None)
279 + running: List[SequenceGroup] = []
280 +
281 + _, seq_group = create_dummy_prompt("1", prompt_length=48)
282 + scheduler.add_seq_group(seq_group)
283 + running.append(seq_group)
284 + assert seq_group.is_prefill()
285 +
286 + # The prompt length > max_num_batched_tokens should be still scheduled.
287 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
288 + assert len(get_sequence_groups(out)) == 1
289 + assert seq_group_meta[0].token_chunk_size == 32
290 + assert running[0].is_prefill()
291 + assert out.num_prefill_groups == 1
292 + assert out.num_batched_tokens == 32
293 +
294 +
295 +def test_prompt_limit_exceed():
296 + block_size = 4
297 + max_seqs = 64
298 + max_model_len = 32
299 + max_num_batched_tokens = 64
300 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
301 + max_seqs,
302 + max_model_len,
303 + enable_chunked_prefill=True)
304 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
305 + cache_config.num_cpu_blocks = 8
306 + cache_config.num_gpu_blocks = 8
307 + scheduler = Scheduler(scheduler_config, cache_config, None)
308 + running: List[SequenceGroup] = []
309 +
310 + _, seq_group = create_dummy_prompt("2", prompt_length=48)
311 + scheduler.add_seq_group(seq_group)
312 + running.append(seq_group)
313 + assert seq_group.is_prefill()
314 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
315 + assert len(out.ignored_seq_groups) == 1
316 + assert out.ignored_seq_groups[0] == seq_group
317 +
318 +
319 +def test_swap():
320 + """Verify swapping works with chunked prefill requests"""
321 + block_size = 4
322 + max_seqs = 30
323 + max_model_len = 200
324 + max_num_batched_tokens = 30
325 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
326 + max_seqs,
327 + max_model_len,
328 + enable_chunked_prefill=True)
329 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
330 + cache_config.num_cpu_blocks = 8
331 + cache_config.num_gpu_blocks = 8
332 + scheduler = Scheduler(scheduler_config, cache_config, None)
333 +
334 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
335 + scheduler.add_seq_group(seq_group)
336 + _, out = schedule_and_update_computed_tokens(scheduler)
337 + # The request is chunked.
338 + # prefill scheduled now.
339 + assert len(out.scheduled_seq_groups) == 1
340 + assert out.num_prefill_groups == 1
341 + assert seq_group.is_prefill()
342 + assert out.num_batched_tokens == max_num_batched_tokens
343 +
344 + # The last request should be swapped out.
345 + scheduler.block_manager.can_append_slots = MagicMock()
346 +
347 + def cannot_append_second_group(seq_group, num_lookahead_slots):
348 + return seq_group.request_id != "1"
349 +
350 + scheduler.block_manager.can_append_slots.side_effect = (
351 + cannot_append_second_group)
352 +
353 + # The running prefill is now swapped.
354 + _, out = schedule_and_update_computed_tokens(scheduler)
355 + assert len(out.scheduled_seq_groups) == 0
356 + assert out.num_batched_tokens == 0
357 + assert out.blocks_to_swap_out != {}
358 + assert out.blocks_to_swap_in == {}
359 +
360 + # Add 1 more task. Swap should be prioritized over new prefill.
361 + _, seq_group = create_dummy_prompt("2", prompt_length=60)
362 + scheduler.add_seq_group(seq_group)
363 + _, out = schedule_and_update_computed_tokens(scheduler)
364 + assert len(out.scheduled_seq_groups) == 1
365 + # 3 decodes. It is swapped in.
366 + assert out.num_batched_tokens == 30
367 + assert out.blocks_to_swap_in != {}
368 + assert out.blocks_to_swap_out == {}
369 +
370 +
371 +def test_running_prefill_prioritized_over_swap():
372 + block_size = 4
373 + max_seqs = 30
374 + max_model_len = 200
375 + max_num_batched_tokens = 30
376 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
377 + max_seqs,
378 + max_model_len,
379 + enable_chunked_prefill=True)
380 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
381 + cache_config.num_cpu_blocks = 8
382 + cache_config.num_gpu_blocks = 8
383 + scheduler = Scheduler(scheduler_config, cache_config, None)
384 +
385 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
386 + scheduler.add_seq_group(seq_group)
387 + _, out = schedule_and_update_computed_tokens(scheduler)
388 + # The request is chunked.
389 + # prefill scheduled now.
390 + assert len(out.scheduled_seq_groups) == 1
391 + assert out.num_prefill_groups == 1
392 + assert seq_group.is_prefill()
393 + assert out.num_batched_tokens == max_num_batched_tokens
394 +
395 + # The request should be swapped out.
396 + scheduler.block_manager.can_append_slots = MagicMock()
397 +
398 + def cannot_append_second_group(seq_group, num_lookahead_slots):
399 + return seq_group.request_id != "1"
400 +
401 + scheduler.block_manager.can_append_slots.side_effect = (
402 + cannot_append_second_group)
403 +
404 + # The running prefill is now swapped.
405 + _, out = schedule_and_update_computed_tokens(scheduler)
406 + assert len(out.scheduled_seq_groups) == 0
407 + assert out.num_batched_tokens == 0
408 + assert out.blocks_to_swap_out != {}
409 + assert out.blocks_to_swap_in == {}
410 +
411 + # Add 1 more task. Swap is not possible, so prefill is running.
412 + scheduler.block_manager.can_swap_in = MagicMock()
413 + scheduler.block_manager.can_swap_in.return_value = False
414 +
415 + _, seq_group2 = create_dummy_prompt("2", prompt_length=60)
416 + scheduler.add_seq_group(seq_group2)
417 + _, out = schedule_and_update_computed_tokens(scheduler)
418 + assert len(out.scheduled_seq_groups) == 1
419 + # 3 decodes. It is swapped in.
420 + assert out.num_batched_tokens == 30
421 + assert out.blocks_to_swap_in == {}
422 + assert out.blocks_to_swap_out == {}
423 + assert out.scheduled_seq_groups[0].seq_group == seq_group2
424 +
425 + # Now although swap is possible, running prefill is prioritized.
426 + scheduler.block_manager.can_swap_in.return_value = True
427 + _, out = schedule_and_update_computed_tokens(scheduler)
428 + assert len(out.scheduled_seq_groups) == 1
429 + # 3 decodes. It is swapped in.
430 + assert out.num_batched_tokens == 30
431 + assert out.blocks_to_swap_in == {}
432 + assert out.blocks_to_swap_out == {}
433 + assert not seq_group2.is_prefill()
434 + assert out.scheduled_seq_groups[0].seq_group == seq_group2
435 + append_new_token(seq_group2, 1)
436 +
437 + # Decoding is prioritized.
438 + _, out = schedule_and_update_computed_tokens(scheduler)
439 + assert len(out.scheduled_seq_groups) == 1
440 + # 3 decodes. It is swapped in.
441 + assert out.num_batched_tokens == 1
442 + assert out.blocks_to_swap_in == {}
443 + assert out.blocks_to_swap_out == {}
444 + assert not seq_group2.is_prefill()
445 + assert out.scheduled_seq_groups[0].seq_group == seq_group2
446 + append_new_token(seq_group2, 1)
447 +
448 + # Since we abort the sequence group, we can finally swap.
449 + scheduler.abort_seq_group(seq_group2.request_id)
450 + _, out = schedule_and_update_computed_tokens(scheduler)
451 + assert len(out.scheduled_seq_groups) == 1
452 + assert out.num_batched_tokens == 30
453 + assert out.blocks_to_swap_in != {}
454 + assert out.blocks_to_swap_out == {}
455 +
456 +
457 +def test_chunked_prefill_preempt():
458 + """Verify preempt works with chunked prefill requests"""
459 + block_size = 4
460 + max_seqs = 30
461 + max_model_len = 200
462 + max_num_batched_tokens = 30
463 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
464 + max_seqs,
465 + max_model_len,
466 + enable_chunked_prefill=True)
467 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
468 + cache_config.num_cpu_blocks = 8
469 + cache_config.num_gpu_blocks = 8
470 + scheduler = Scheduler(scheduler_config, cache_config, None)
471 +
472 + _, seq_group = create_dummy_prompt("1", prompt_length=60)
473 + scheduler.add_seq_group(seq_group)
474 + _, out = schedule_and_update_computed_tokens(scheduler)
475 + # The request is chunked.
476 + # prefill scheduled now.
477 + assert len(out.scheduled_seq_groups) == 1
478 + assert out.num_prefill_groups == 1
479 + assert seq_group.is_prefill()
480 + assert out.num_batched_tokens == max_num_batched_tokens
481 +
482 + # The request should be preempted.
483 + scheduler.block_manager.can_append_slots = MagicMock()
484 +
485 + def cannot_append_second_group(seq_group, num_lookahead_slots):
486 + return seq_group.request_id != "1"
487 +
488 + scheduler.block_manager.can_append_slots.side_effect = (
489 + cannot_append_second_group)
490 +
491 + # The running prefill is now preempted.
492 + _, out = schedule_and_update_computed_tokens(scheduler)
493 + assert len(out.scheduled_seq_groups) == 0
494 + assert out.num_batched_tokens == 0
495 + assert out.blocks_to_swap_out == {}
496 + assert out.blocks_to_swap_in == {}
497 +
498 + # Make sure we can reschedule preempted request.
499 + _, out = schedule_and_update_computed_tokens(scheduler)
500 + assert len(out.scheduled_seq_groups) == 1
501 + assert out.num_prefill_groups == 1
502 + assert seq_group.is_prefill()
503 + assert out.num_batched_tokens == max_num_batched_tokens
504 + assert seq_group.get_num_uncomputed_tokens() == 30
505 +
506 + # We should be able to run prefill twice as it is chunked.
507 + def cannot_append_second_group(seq_group, num_lookahead_slots):
508 + return True
509 +
510 + scheduler.block_manager.can_append_slots.side_effect = (
511 + cannot_append_second_group)
512 + _, out = schedule_and_update_computed_tokens(scheduler)
513 + assert len(out.scheduled_seq_groups) == 1
514 + assert out.num_prefill_groups == 1
515 + assert not seq_group.is_prefill()
516 + assert out.num_batched_tokens == max_num_batched_tokens
517 +
518 +
519 +def test_chunked_prefill_max_seqs():
520 + block_size = 4
521 + max_seqs = 2
522 + max_model_len = 80
523 + max_num_batched_tokens = 64
524 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
525 + max_seqs,
526 + max_model_len,
527 + enable_chunked_prefill=True)
528 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
529 + cache_config.num_cpu_blocks = 8
530 + cache_config.num_gpu_blocks = 8
531 + scheduler = Scheduler(scheduler_config, cache_config, None)
532 + running = []
533 +
534 + _, seq_group = create_dummy_prompt("1", prompt_length=65)
535 + scheduler.add_seq_group(seq_group)
536 + running.append(seq_group)
537 + # The first prefill is chunked.
538 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
539 + assert seq_group_meta[0].token_chunk_size == max_num_batched_tokens
540 + assert len(get_sequence_groups(out)) == 1
541 +
542 + # Add new requests.
543 + for i in range(4):
544 + _, seq_group = create_dummy_prompt(str(i), prompt_length=65)
545 + scheduler.add_seq_group(seq_group)
546 + running.append(seq_group)
547 +
548 + # Make sure only 2 requests are scheduled.
549 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
550 + assert out.num_batched_tokens == max_num_batched_tokens
551 + assert len(get_sequence_groups(out)) == 2
552 + assert not running[0].is_prefill()
553 + assert running[1].is_prefill()
554 + append_new_token(running[0], 1)
555 +
556 + # Although we have enough token budget, we can only schedule max_seqs.
557 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
558 + assert seq_group_meta[0].token_chunk_size == 2
559 + assert seq_group_meta[1].token_chunk_size == 1
560 + assert out.num_batched_tokens == 3
561 + assert len(get_sequence_groups(out)) == max_seqs
562 + assert not running[0].is_prefill()
563 + assert not running[1].is_prefill()
tests/core/test_scheduler.py
+201 -65
@@ -10,7 +10,7 @@ from vllm.core.interfaces import AllocStatus
10 from vllm.core.policy import PolicyFactory
11 from vllm.core.scheduler import Scheduler, SchedulingBudget
12 from vllm.lora.request import LoRARequest
13 -from vllm.sequence import Logprob, SequenceGroup
13 +from vllm.sequence import Logprob, SequenceGroup, SequenceStatus
14
15 from .utils import create_dummy_prompt
16
@@ -19,6 +19,26 @@ def get_sequence_groups(scheduler_output):
19 return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
20
21
22 +def append_new_token(out, token_id: int):
23 + seq_groups = get_sequence_groups(out)
24 + for seq_group in seq_groups:
25 + for seq in seq_group.get_seqs():
26 + seq.append_token_id(token_id, {token_id: Logprob(token_id)})
27 +
28 +
29 +def schedule_and_update_computed_tokens(scheduler):
30 + metas, out = scheduler.schedule()
31 + for s, meta in zip(out.scheduled_seq_groups, metas):
32 + s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
33 + return metas, out
34 +
35 +
36 +def append_new_token_seq_group(token_chunk_size, seq_group, token_id: int):
37 + seq_group.update_num_computed_tokens(token_chunk_size)
38 + for seq in seq_group.get_seqs():
39 + seq.append_token_id(token_id, {token_id: Logprob(token_id)})
40 +
41 +
42 def test_scheduler_add_seq_group():
43 block_size = 4
44 scheduler_config = SchedulerConfig(100, 64, 1)
@@ -76,20 +96,52 @@ def test_scheduler_schedule_simple():
96
97 # Schedule seq groups prompts.
98 num_tokens = block_size * num_seq_group
79 - seq_group_meta, out = scheduler.schedule()
99 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
100 assert set(get_sequence_groups(out)) == set(running)
101 assert out.num_batched_tokens == num_tokens
102 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
103 and not out.blocks_to_swap_out)
104 assert len(seq_group_meta) == num_seq_group
105 + append_new_token(out, 1)
106
107 # Schedule seq groups generation.
87 - seq_group_meta, out = scheduler.schedule()
108 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
109 assert set(get_sequence_groups(out)) == set(running)
110 assert out.num_batched_tokens == num_seq_group
111 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
112 and not out.blocks_to_swap_out)
113 assert len(seq_group_meta) == num_seq_group
114 + append_new_token(out, 1)
115 +
116 +
117 +def test_scheduler_prefill_prioritized():
118 + """Verify running batched tokens are not applied to prefill requests."""
119 + block_size = 4
120 + max_model_len = 30
121 + max_batched_num_tokens = 30
122 + scheduler_config = SchedulerConfig(max_batched_num_tokens, 2,
123 + max_model_len)
124 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
125 + cache_config.num_cpu_blocks = 2
126 + cache_config.num_gpu_blocks = 2
127 + scheduler = Scheduler(scheduler_config, cache_config, None)
128 +
129 + # Add seq groups to scheduler.
130 + _, seq_group_a = create_dummy_prompt("1", 1)
131 + scheduler.add_seq_group(seq_group_a)
132 +
133 + # Schedule seq groups prompts.
134 + _, out = schedule_and_update_computed_tokens(scheduler)
135 + assert get_sequence_groups(out) == [seq_group_a]
136 +
137 + # Add a new prefill request B.
138 + _, seq_group_b = create_dummy_prompt("2", 30)
139 + scheduler.add_seq_group(seq_group_b)
140 +
141 + # Verify prefill requests are prioritized. Since max_batched_num_tokens
142 + # is 1, new prefill request has to be scheduled first.
143 + _, out = schedule_and_update_computed_tokens(scheduler)
144 + assert get_sequence_groups(out) == [seq_group_b]
145
146
147 def test_scheduler_schedule_preempt_abort():
@@ -108,7 +160,7 @@ def test_scheduler_schedule_preempt_abort():
160 scheduler.add_seq_group(seq_group_b)
161
162 # Schedule seq groups prompts.
111 - seq_group_meta, out = scheduler.schedule()
163 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
164 assert get_sequence_groups(out) == [seq_group_a, seq_group_b]
165 assert out.num_batched_tokens == block_size * 2 # seq_a and seq_b
166 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
@@ -118,12 +170,10 @@ def test_scheduler_schedule_preempt_abort():
170
171 # Append "generated" tokens, allowing the sequence to mark prompt tokens as
172 # processed.
121 - token_id = 0
122 - seq_a.append_token_id(token_id, {token_id: Logprob(0.0)})
123 - seq_b.append_token_id(token_id, {token_id: Logprob(0.0)})
173 + append_new_token(out, 1)
174
175 # Schedule seq groups generation and preempt seq group b.
126 - seq_group_meta, out = scheduler.schedule()
176 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
177 assert get_sequence_groups(out) == [seq_group_a]
178 assert out.num_batched_tokens == 1
179 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
@@ -133,7 +183,7 @@ def test_scheduler_schedule_preempt_abort():
183
184 # Abort seq group a. Re-schedule seq group b prompt with recomputation.
185 scheduler.abort_seq_group("1")
136 - seq_group_meta, out = scheduler.schedule()
186 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
187 assert get_sequence_groups(out) == [seq_group_b]
188 assert out.num_batched_tokens == 5 # 4 prompt + 1 generation.
189 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
@@ -163,12 +213,14 @@ def test_scheduler_max_seqs():
213 scheduler.add_seq_group(all_seq_groups[0])
214
215 # Schedule seq groups prompts.
166 - _, out = scheduler.schedule()
216 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
217 assert set(get_sequence_groups(out)) == set([all_seq_groups[0]])
218 + append_new_token(out, 1)
219
220 # Schedule seq groups generation.
170 - _, out = scheduler.schedule()
221 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
222 assert set(get_sequence_groups(out)) == set([all_seq_groups[0]])
223 + append_new_token(out, 1)
224
225 # Append 2 more seq group
226 scheduler.add_seq_group(all_seq_groups[1])
@@ -177,7 +229,7 @@ def test_scheduler_max_seqs():
229 # Schedule seq groups prompts.
230 # Only 1 seq group should be scheduled since max_seq_group is 2
231 # and one is prompting.
180 - _, out = scheduler.schedule()
232 + _, out = schedule_and_update_computed_tokens(scheduler)
233 assert set(get_sequence_groups(out)) == set([all_seq_groups[1]])
234
235
@@ -190,27 +242,32 @@ def test_scheduler_delay_factor():
242 scheduler = Scheduler(scheduler_config, cache_config, None)
243
244 # schedule first prompt
193 - _, seq_group = create_dummy_prompt("0", prompt_length=block_size)
245 + seq_group_meta, seq_group = create_dummy_prompt("0",
246 + prompt_length=block_size)
247 scheduler.add_seq_group(seq_group)
195 - seq_group_meta, out = scheduler.schedule()
248 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
249 assert out.num_prefill_groups > 0
250 assert seq_group_meta[0].request_id == '0'
251 + append_new_token(out, 1)
252
253 # wait for a second before scheduling next prompt
254 time.sleep(1)
201 - _, seq_group = create_dummy_prompt("1", prompt_length=block_size)
255 + seq_group_meta, seq_group = create_dummy_prompt("1",
256 + prompt_length=block_size)
257 scheduler.add_seq_group(seq_group)
258
259 # second prompt should *not* be scheduled
205 - seq_group_meta, out = scheduler.schedule()
260 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
261 assert out.num_prefill_groups == 0
262 assert seq_group_meta[0].request_id == '0'
263 + append_new_token(out, 1)
264
265 # wait for more than 0.5 second and try again
266 time.sleep(0.6)
211 - seq_group_meta, out = scheduler.schedule()
267 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
268 assert out.num_prefill_groups > 0
269 assert seq_group_meta[0].request_id == '1'
270 + append_new_token(out, 1)
271
272
273 def test_swapped_out_prioritized():
@@ -219,9 +276,10 @@ def test_swapped_out_prioritized():
276 for i in range(3):
277 _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
278 scheduler.add_seq_group(seq_group)
222 - _, out = scheduler.schedule()
279 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
280 # prefill scheduled now.
281 assert len(out.scheduled_seq_groups) == 3
282 + append_new_token(out, 1)
283
284 # The last request should be swapped out.
285 scheduler.block_manager.can_append_slots = MagicMock()
@@ -232,16 +290,18 @@ def test_swapped_out_prioritized():
290 scheduler.block_manager.can_append_slots.side_effect = (
291 cannot_append_second_group)
292
235 - _, out = scheduler.schedule()
293 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
294 assert len(out.scheduled_seq_groups) == 2
295 assert out.num_batched_tokens == 2
296 assert out.blocks_to_swap_out != {}
297 assert out.blocks_to_swap_in == {}
298 + append_new_token(out, 1)
299
300 # Add 1 more task. Swap should be prioritized over prefill.
301 _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
302 scheduler.add_seq_group(seq_group)
244 - _, out = scheduler.schedule()
303 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
304 + append_new_token(out, 1)
305 assert len(out.scheduled_seq_groups) == 3
306 # 3 decodes. It is swapped in.
307 assert out.num_batched_tokens == 3
@@ -264,18 +324,23 @@ def initialize_scheduler(*,
324 return scheduler
325
326
267 -def create_token_budget(num_batched_tokens: int = 0,
268 - num_curr_seqs: int = 0,
269 - token_budget: int = 10000,
327 +def create_token_budget(token_budget: int = 10000,
328 max_num_seqs: int = 10000) -> SchedulingBudget:
329 return SchedulingBudget(
272 - num_batched_tokens=num_batched_tokens,
273 - num_curr_seqs=num_curr_seqs,
330 token_budget=token_budget,
331 max_num_seqs=max_num_seqs,
332 )
333
334
335 +def add_token_budget(budget: SchedulingBudget,
336 + num_batched_tokens: int = 0,
337 + num_curr_seqs: int = 0):
338 + mock_seq_group = create_dummy_prompt('10', prompt_length=60)[1]
339 + budget.add_num_batched_tokens(mock_seq_group.request_id,
340 + num_batched_tokens)
341 + budget.add_num_seqs(mock_seq_group.request_id, num_curr_seqs)
342 +
343 +
344 def test_prefill_schedule_max_prompt_len():
345 """
346 Test prompt longer than max_prompt_len is aborted.
@@ -326,7 +391,8 @@ def test_prefill_schedule_token_budget():
391 # Test when current_batched_tokens respected.
392 scheduler = initialize_scheduler()
393 waiting = deque()
329 - budget = create_token_budget(num_batched_tokens=30, token_budget=60)
394 + budget = create_token_budget(token_budget=60)
395 + add_token_budget(budget, 30, 0)
396 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
397 # Cannot schedule a prompt that doesn't fit the budget.
398 waiting.append(seq_group)
@@ -337,7 +403,8 @@ def test_prefill_schedule_token_budget():
403 assert budget.num_batched_tokens == 30
404 assert budget.num_curr_seqs == 0
405 assert len(remaining_waiting) == 1
340 - budget = create_token_budget(num_batched_tokens=30, token_budget=90)
406 + budget = create_token_budget(token_budget=90)
407 + add_token_budget(budget, 30, 0)
408 remaining_waiting, output = scheduler._schedule_prefills(
409 waiting, budget, None)
410 assert len(output.seq_groups) == 1
@@ -366,7 +433,8 @@ def test_prefill_schedule_max_seqs():
433
434 # Verify curr_num_seqs respected.
435 waiting = deque()
369 - budget = create_token_budget(num_curr_seqs=2, max_num_seqs=2)
436 + budget = create_token_budget(max_num_seqs=2)
437 + add_token_budget(budget, 0, 2)
438 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
439 waiting.append(seq_group)
440 remaining_waiting, output = scheduler._schedule_prefills(
@@ -472,7 +540,8 @@ def test_decode_schedule_preempted():
540 curr_loras = None
541 for i in range(3):
542 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
475 - scheduler._allocate_and_set_running(seq_group)
543 + scheduler._allocate_and_set_running(seq_group, 60)
544 + append_new_token_seq_group(60, seq_group, 1)
545 running.append(seq_group)
546 scheduler.block_manager.can_append_slots = MagicMock()
547
@@ -484,12 +553,13 @@ def test_decode_schedule_preempted():
553
554 # 1 cannot be scheduled, and the lowest priority (request 2)
555 # should be preempted. 1 will also be preempted.
487 - budget = create_token_budget(num_batched_tokens=3, num_curr_seqs=3)
488 - remainig_running, output = scheduler._schedule_decodes(
556 + budget = create_token_budget()
557 + remainig_running, output = scheduler._schedule_running(
558 running, budget, curr_loras, policy)
559 assert len(remainig_running) == 0
491 - assert len(output.seq_groups) == 1
492 - assert output.seq_groups[0].seq_group.request_id == "0"
560 + assert len(output.decode_seq_groups) == 1
561 + assert len(output.prefill_seq_groups) == 0
562 + assert output.decode_seq_groups[0].seq_group.request_id == "0"
563 assert len(output.preempted) == 2
564 # Verify budgets are updated.
565 assert budget.num_batched_tokens == 1
@@ -508,10 +578,16 @@ def test_decode_swap_beam_search():
578 running = deque()
579 policy = PolicyFactory.get_policy(policy_name="fcfs")
580 curr_loras = None
581 + budget = create_token_budget()
582 for i in range(3):
583 _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
513 - scheduler._allocate_and_set_running(seq_group)
584 + scheduler._allocate_and_set_running(seq_group, 60)
585 running.append(seq_group)
586 + append_new_token_seq_group(60, seq_group, 1)
587 + budget.add_num_seqs(seq_group.request_id,
588 + seq_group.get_max_num_running_seqs())
589 + budget.add_num_batched_tokens(
590 + seq_group.request_id, seq_group.num_seqs(SequenceStatus.RUNNING))
591
592 # The last request should be swapped out.
593 scheduler.block_manager.can_append_slots = MagicMock()
@@ -525,19 +601,19 @@ def test_decode_swap_beam_search():
601 expected_swap_mapping = {"5": "7"}
602 scheduler.block_manager.swap_out.return_value = expected_swap_mapping
603
528 - budget = create_token_budget(num_batched_tokens=3, num_curr_seqs=3)
529 - remainig_running, output = scheduler._schedule_decodes(
604 + remainig_running, output = scheduler._schedule_running(
605 running, budget, curr_loras, policy)
606 assert len(remainig_running) == 0
532 - assert len(output.seq_groups) == 2
533 - assert output.seq_groups[0].seq_group.request_id == "0"
534 - assert output.seq_groups[1].seq_group.request_id == "1"
607 + assert len(output.decode_seq_groups) == 2
608 + assert len(output.prefill_seq_groups) == 0
609 + assert output.decode_seq_groups[0].seq_group.request_id == "0"
610 + assert output.decode_seq_groups[1].seq_group.request_id == "1"
611 assert len(output.preempted) == 0
612 assert len(output.swapped_out) == 1
613 # Budget should refledct preempted requests.
614 assert budget.num_batched_tokens == 2
615 # since there are 2 sequences, 2 should be subtracted.
540 - assert budget.num_curr_seqs == 1
616 + assert budget.num_curr_seqs == 4
617 # Both should be preempted, not swapped.
618 assert output.blocks_to_swap_out == expected_swap_mapping
619 # Nothing is copied.
@@ -553,7 +629,8 @@ def test_schedule_decode_blocks_to_copy_update():
629 running = deque()
630 policy = PolicyFactory.get_policy(policy_name="fcfs")
631 curr_loras = None
556 - scheduler._allocate_and_set_running(seq_group)
632 + scheduler._allocate_and_set_running(seq_group, 60)
633 + append_new_token_seq_group(60, seq_group, 1)
634 running.append(seq_group)
635
636 # The last request should be swapped out.
@@ -561,10 +638,11 @@ def test_schedule_decode_blocks_to_copy_update():
638 scheduler.block_manager.append_slots.return_value = {2: [3]}
639
640 budget = create_token_budget()
564 - remaining_running, output = scheduler._schedule_decodes(
641 + remaining_running, output = scheduler._schedule_running(
642 running, budget, curr_loras, policy)
643 assert len(remaining_running) == 0
567 - assert len(output.seq_groups) == 1
644 + assert len(output.decode_seq_groups) == 1
645 + assert len(output.prefill_seq_groups) == 0
646 assert len(output.preempted) == 0
647 assert len(output.swapped_out) == 0
648 # Nothing is preempted.
@@ -581,7 +659,8 @@ def test_schedule_swapped_simple():
659 curr_loras = None
660 blocks_to_swap_out = {}
661 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
584 - scheduler._allocate_and_set_running(seq_group)
662 + scheduler._allocate_and_set_running(seq_group, 60)
663 + append_new_token_seq_group(60, seq_group, 1)
664 scheduler._swap_out(seq_group, blocks_to_swap_out)
665 swapped.append(seq_group)
666
@@ -591,7 +670,8 @@ def test_schedule_swapped_simple():
670 assert len(remaining_swapped) == 0
671 assert budget.num_batched_tokens == 1
672 assert budget.num_curr_seqs == 2
594 - assert len(output.seq_groups) == 1
673 + assert len(output.decode_seq_groups) == 1
674 + assert len(output.prefill_seq_groups) == 0
675 # swap in is the reverse of swap out
676 blocks_to_swap_in_reverse = {}
677 for swapin, swapout in output.blocks_to_swap_in.items():
@@ -607,7 +687,8 @@ def test_schedule_swapped_max_token_budget():
687 blocks_to_swap_out = {}
688 for _ in range(2):
689 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
610 - scheduler._allocate_and_set_running(seq_group)
690 + scheduler._allocate_and_set_running(seq_group, 60)
691 + append_new_token_seq_group(60, seq_group, 1)
692 scheduler._swap_out(seq_group, blocks_to_swap_out)
693 swapped.append(seq_group)
694
@@ -617,16 +698,19 @@ def test_schedule_swapped_max_token_budget():
698 assert len(remaining_swapped) == 1
699 assert budget.num_batched_tokens == 1
700 assert budget.num_curr_seqs == 2
620 - assert len(output.seq_groups) == 1
701 + assert len(output.decode_seq_groups) == 1
702 + assert len(output.prefill_seq_groups) == 0
703
704 # Verify num_batched_tokens are respected.
623 - budget = create_token_budget(num_batched_tokens=1, token_budget=1)
705 + budget = create_token_budget(token_budget=1)
706 + add_token_budget(budget, 1, 0)
707 remaining_swapped, output = scheduler._schedule_swapped(
708 remaining_swapped, budget, curr_loras, policy)
709 assert len(remaining_swapped) == 1
710 assert budget.num_batched_tokens == 1
711 assert budget.num_curr_seqs == 0
629 - assert len(output.seq_groups) == 0
712 + assert len(output.decode_seq_groups) == 0
713 + assert len(output.prefill_seq_groups) == 0
714
715
716 def test_schedule_swapped_max_seqs():
@@ -635,28 +719,30 @@ def test_schedule_swapped_max_seqs():
719 policy = PolicyFactory.get_policy(policy_name="fcfs")
720 curr_loras = None
721 blocks_to_swap_out = {}
638 - for _ in range(2):
639 - _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
640 - scheduler._allocate_and_set_running(seq_group)
722 + for i in range(4):
723 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
724 + scheduler._allocate_and_set_running(seq_group, 60)
725 + append_new_token_seq_group(60, seq_group, 1)
726 scheduler._swap_out(seq_group, blocks_to_swap_out)
727 swapped.append(seq_group)
728
729 budget = create_token_budget(max_num_seqs=2)
730 remaining_swapped, output = scheduler._schedule_swapped(
731 swapped, budget, curr_loras, policy)
647 - assert len(remaining_swapped) == 1
648 - assert budget.num_batched_tokens == 1
732 + assert len(remaining_swapped) == 2
733 + assert budget.num_batched_tokens == 2
734 assert budget.num_curr_seqs == 2
650 - assert len(output.seq_groups) == 1
735 + assert len(output.decode_seq_groups) == 2
736 + assert len(output.prefill_seq_groups) == 0
737
738 # Verify num_curr_seqs are respected.
653 - budget = create_token_budget(num_curr_seqs=2, max_num_seqs=2)
739 remaining_swapped, output = scheduler._schedule_swapped(
740 remaining_swapped, budget, curr_loras, policy)
656 - assert len(remaining_swapped) == 1
657 - assert budget.num_batched_tokens == 0
741 + assert len(remaining_swapped) == 2
742 + assert budget.num_batched_tokens == 2
743 assert budget.num_curr_seqs == 2
659 - assert len(output.seq_groups) == 0
744 + assert len(output.decode_seq_groups) == 0
745 + assert len(output.prefill_seq_groups) == 0
746
747
748 def test_schedule_swapped_max_loras():
@@ -673,7 +759,8 @@ def test_schedule_swapped_max_loras():
759 lora_name=str(i),
760 lora_int_id=i + 1,
761 lora_local_path="abc"))
676 - scheduler._allocate_and_set_running(seq_group)
762 + scheduler._allocate_and_set_running(seq_group, 60)
763 + append_new_token_seq_group(60, seq_group, 1)
764 scheduler._swap_out(seq_group, blocks_to_swap_out)
765 swapped.append(seq_group)
766
@@ -683,7 +770,8 @@ def test_schedule_swapped_max_loras():
770 assert len(remaining_swapped) == 1
771 assert budget.num_batched_tokens == 1
772 assert budget.num_curr_seqs == 1
686 - assert len(output.seq_groups) == 1
773 + assert len(output.decode_seq_groups) == 1
774 + assert len(output.prefill_seq_groups) == 0
775 assert len(curr_loras) == 1
776
777
@@ -695,7 +783,8 @@ def test_schedule_swapped_cannot_swap_in():
783 blocks_to_swap_out = {}
784 for _ in range(2):
785 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
698 - scheduler._allocate_and_set_running(seq_group)
786 + scheduler._allocate_and_set_running(seq_group, 60)
787 + append_new_token_seq_group(60, seq_group, 1)
788 scheduler._swap_out(seq_group, blocks_to_swap_out)
789 swapped.append(seq_group)
790
@@ -709,7 +798,8 @@ def test_schedule_swapped_cannot_swap_in():
798 assert len(remaining_swapped) == 2
799 assert budget.num_batched_tokens == 0
800 assert budget.num_curr_seqs == 0
712 - assert len(output.seq_groups) == 0
801 + assert len(output.decode_seq_groups) == 0
802 + assert len(output.prefill_seq_groups) == 0
803
804
805 def test_schedule_swapped_blocks_to_copy():
@@ -718,7 +808,8 @@ def test_schedule_swapped_blocks_to_copy():
808 policy = PolicyFactory.get_policy(policy_name="fcfs")
809 curr_loras = None
810 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
721 - scheduler._allocate_and_set_running(seq_group)
811 + scheduler._allocate_and_set_running(seq_group, 60)
812 + append_new_token_seq_group(60, seq_group, 1)
813 blocks_to_swap_out = {}
814 scheduler._swap_out(seq_group, blocks_to_swap_out)
815 swapped.append(seq_group)
@@ -731,5 +822,50 @@ def test_schedule_swapped_blocks_to_copy():
822 remaining_swapped, output = scheduler._schedule_swapped(
823 swapped, budget, curr_loras, policy)
824 assert len(remaining_swapped) == 0
734 - assert len(output.seq_groups) == 1
825 + assert len(output.decode_seq_groups) == 1
826 + assert len(output.prefill_seq_groups) == 0
827 assert output.blocks_to_copy == {2: [3]}
828 +
829 +
830 +def test_scheduling_budget():
831 + TOKEN_BUDGET = 4
832 + MAX_SEQS = 4
833 + budget = SchedulingBudget(token_budget=TOKEN_BUDGET, max_num_seqs=MAX_SEQS)
834 + assert budget.can_schedule(num_new_tokens=1, num_new_seqs=1)
835 + assert budget.can_schedule(num_new_tokens=4, num_new_seqs=4)
836 + assert not budget.can_schedule(num_new_tokens=1, num_new_seqs=5)
837 + assert not budget.can_schedule(num_new_tokens=5, num_new_seqs=1)
838 + assert not budget.can_schedule(num_new_tokens=5, num_new_seqs=5)
839 + assert budget.remaining_token_budget() == TOKEN_BUDGET
840 +
841 + # Verify add/subtract num batched tokens.
842 + _, seq_group = create_dummy_prompt("1", 3)
843 + budget.add_num_batched_tokens(seq_group.request_id, 2)
844 + assert budget.remaining_token_budget() == 2
845 + assert budget.num_batched_tokens == 2
846 + assert budget.can_schedule(num_new_tokens=2, num_new_seqs=1)
847 + assert not budget.can_schedule(num_new_tokens=3, num_new_seqs=1)
848 + # Verify adding another seq group is no-op.
849 + budget.add_num_batched_tokens(seq_group.request_id, 2)
850 + assert budget.remaining_token_budget() == 2
851 + assert budget.num_batched_tokens == 2
852 + budget.subtract_num_batched_tokens(seq_group.request_id, 2)
853 + assert budget.remaining_token_budget() == 4
854 + assert budget.num_batched_tokens == 0
855 + budget.subtract_num_batched_tokens(seq_group.request_id, 2)
856 + assert budget.remaining_token_budget() == 4
857 + assert budget.num_batched_tokens == 0
858 +
859 + # Verify add/subtract max seqs.
860 + _, seq_group = create_dummy_prompt("1", 3)
861 + budget.add_num_seqs(seq_group.request_id, 2)
862 + assert budget.can_schedule(num_new_tokens=1, num_new_seqs=2)
863 + assert not budget.can_schedule(num_new_tokens=1, num_new_seqs=3)
864 + assert budget.num_curr_seqs == 2
865 + # Verify adding another seq group is no-op.
866 + budget.add_num_seqs(seq_group.request_id, 2)
867 + assert budget.num_curr_seqs == 2
868 + budget.subtract_num_seqs(seq_group.request_id, 2)
869 + assert budget.num_curr_seqs == 0
870 + budget.subtract_num_seqs(seq_group.request_id, 2)
871 + assert budget.num_curr_seqs == 0
tests/test_sequence.py
+55 -3
@@ -1,7 +1,36 @@
1 +import time
2 +from typing import Optional
3 +
4 import pytest
5
3 -from vllm.sequence import (SamplerOutput, SequenceData, SequenceGroupOutput,
4 - SequenceOutput)
6 +from vllm import SamplingParams
7 +from vllm.lora.request import LoRARequest
8 +from vllm.sequence import (SamplerOutput, Sequence, SequenceData,
9 + SequenceGroup, SequenceGroupOutput, SequenceOutput)
10 +
11 +
12 +def create_dummy_prompt(
13 + request_id: str,
14 + prompt_length: int,
15 + block_size: Optional[int] = None,
16 + lora_request: Optional[LoRARequest] = None,
17 + use_beam_search: bool = False,
18 + best_of: int = 1,
19 +) -> SequenceGroup:
20 + if not block_size:
21 + block_size = prompt_length
22 +
23 + # Create dummy prompt sequence with tokens 0...block_size-1
24 + # and prompt "0 ... block_size".
25 + prompt_tokens = list(range(prompt_length))
26 + prompt_str = " ".join([str(t) for t in prompt_tokens])
27 + prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
28 + seq_group = SequenceGroup(
29 + request_id, [prompt],
30 + SamplingParams(use_beam_search=use_beam_search, best_of=best_of),
31 + time.time(), lora_request)
32 +
33 + return seq_group
34
35
36 @pytest.fixture
@@ -67,6 +96,29 @@ def test_sequence_data_prefill():
96
97 # append tokens and reset, simulating recompute
98 seq_data.append_token_id(1, logprob=0.0)
70 - seq_data.reset_num_computed_tokens()
99 + seq_data.reset_state_for_recompute()
100 assert seq_data.get_num_uncomputed_tokens() == 5
101 assert seq_data.get_num_computed_tokens() == 0
102 +
103 +
104 +def test_sequence_group_stage():
105 + seq_group = create_dummy_prompt("1", 12)
106 + assert seq_group.is_prefill() is True
107 + seq_group.update_num_computed_tokens(6)
108 + assert seq_group.is_prefill() is True
109 + seq_group.update_num_computed_tokens(5)
110 + assert seq_group.is_prefill() is True
111 + seq_group.update_num_computed_tokens(1)
112 + assert seq_group.is_prefill() is False
113 + seqs = seq_group.get_seqs()
114 + assert len(seqs) == 1
115 + seqs[0].data.append_token_id(1, logprob=0.0)
116 + for seq in seq_group.get_seqs():
117 + seq.reset_state_for_recompute()
118 + assert seq_group.is_prefill() is True
119 + seq_group.update_num_computed_tokens(5)
120 + assert seq_group.is_prefill() is True
121 + seq_group.update_num_computed_tokens(7)
122 + assert seq_group.is_prefill() is True
123 + seq_group.update_num_computed_tokens(1)
124 + assert seq_group.is_prefill() is False
vllm/config.py
+2 -1
@@ -576,7 +576,8 @@ class SchedulerConfig:
576 self._verify_args()
577
578 def _verify_args(self) -> None:
579 - if self.max_num_batched_tokens < self.max_model_len:
579 + if (self.max_num_batched_tokens < self.max_model_len
580 + and not self.chunked_prefill_enabled):
581 raise ValueError(
582 f"max_num_batched_tokens ({self.max_num_batched_tokens}) is "
583 f"smaller than max_model_len ({self.max_model_len}). "
vllm/core/policy.py
+1 -3
@@ -38,9 +38,7 @@ class FCFS(Policy):
38
39 class PolicyFactory:
40
41 - _POLICY_REGISTRY = {
42 - 'fcfs': FCFS,
43 - }
41 + _POLICY_REGISTRY = {'fcfs': FCFS}
42
43 @classmethod
44 def get_policy(cls, policy_name: str, **kwargs) -> Policy:
vllm/core/scheduler.py
+345 -95
@@ -1,7 +1,7 @@
1 import enum
2 import time
3 from collections import deque
4 -from dataclasses import dataclass
4 +from dataclasses import dataclass, field
5 from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
6
7 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
@@ -31,16 +31,64 @@ class PreemptionMode(enum.Enum):
31
32 @dataclass
33 class SchedulingBudget:
34 - """The available slots for scheduling."""
35 - num_batched_tokens: int
36 - num_curr_seqs: int
34 + """The available slots for scheduling.
35 +
36 + TODO(sang): Right now, the budget is request_id-aware meaning it can ignore
37 + budget update from the same request_id. It is because in normal scheduling
38 + path, we update RUNNING num_seqs ahead of time, meaning it could be
39 + updated more than once when scheduling RUNNING requests. Since this won't
40 + happen if we only have chunked prefill scheduling, we can remove this
41 + feature from the API when chunked prefill is enabled by default.
42 + """
43 token_budget: int
44 max_num_seqs: int
45 + _requeset_ids_num_batched_tokens: Set[int] = field(default_factory=set)
46 + _requeset_ids_num_curr_seqs: Set[int] = field(default_factory=set)
47 + _num_batched_tokens: int = 0
48 + _num_curr_seqs: int = 0
49
50 def can_schedule(self, *, num_new_tokens: int, num_new_seqs: int):
51 + assert num_new_tokens != 0
52 + assert num_new_seqs != 0
53 return (self.num_batched_tokens + num_new_tokens <= self.token_budget
54 and self.num_curr_seqs + num_new_seqs <= self.max_num_seqs)
55
56 + def remaining_token_budget(self):
57 + return self.token_budget - self.num_batched_tokens
58 +
59 + def add_num_batched_tokens(self, req_id: str, num_batched_tokens: int):
60 + if req_id in self._requeset_ids_num_batched_tokens:
61 + return
62 +
63 + self._requeset_ids_num_batched_tokens.add(req_id)
64 + self._num_batched_tokens += num_batched_tokens
65 +
66 + def subtract_num_batched_tokens(self, req_id: str,
67 + num_batched_tokens: int):
68 + if req_id in self._requeset_ids_num_batched_tokens:
69 + self._requeset_ids_num_batched_tokens.remove(req_id)
70 + self._num_batched_tokens -= num_batched_tokens
71 +
72 + def add_num_seqs(self, req_id: str, num_curr_seqs: int):
73 + if req_id in self._requeset_ids_num_curr_seqs:
74 + return
75 +
76 + self._requeset_ids_num_curr_seqs.add(req_id)
77 + self._num_curr_seqs += num_curr_seqs
78 +
79 + def subtract_num_seqs(self, req_id: str, num_curr_seqs: int):
80 + if req_id in self._requeset_ids_num_curr_seqs:
81 + self._requeset_ids_num_curr_seqs.remove(req_id)
82 + self._num_curr_seqs -= num_curr_seqs
83 +
84 + @property
85 + def num_batched_tokens(self):
86 + return self._num_batched_tokens
87 +
88 + @property
89 + def num_curr_seqs(self):
90 + return self._num_curr_seqs
91 +
92
93 @dataclass
94 class ScheduledSequenceGroup:
@@ -54,6 +102,7 @@ class ScheduledSequenceGroup:
102
103 @dataclass
104 class SchedulerOutputs:
105 + """The scheduling decision made from a scheduler."""
106 # Scheduled sequence groups.
107 scheduled_seq_groups: Iterable[ScheduledSequenceGroup]
108 # Number of prefill groups scheduled.
@@ -95,10 +144,17 @@ class SchedulerOutputs:
144
145
146 @dataclass
98 -class SchedulerDecodeOutputs:
99 - """Outputs of the decoding phase of the scheduler."""
100 - # Selected sequence groups for decoding.
101 - seq_groups: List[SequenceGroup]
147 +class SchedulerRunningOutputs:
148 + """The requests that are scheduled from a running queue.
149 +
150 + Could contain prefill (prefill that's chunked) or decodes. If there's not
151 + enough memory, it can be preempted (for recompute) or swapped out.
152 + """
153 + # Selected sequences that are running and in a decoding phase.
154 + decode_seq_groups: List[SequenceGroup]
155 + # Selected sequences that are running and in a prefill phase.
156 + # I.e., it means the prefill has been chunked.
157 + prefill_seq_groups: List[SequenceGroup]
158 # The preempted sequences.
159 preempted: List[SequenceGroup]
160 # Sequences that are swapped out.
@@ -107,12 +163,14 @@ class SchedulerDecodeOutputs:
163 blocks_to_swap_out: Dict[int, int]
164 # The blocks to copy.
165 blocks_to_copy: Dict[int, List[int]]
166 + # The number of slots for lookahead decoding.
167 num_lookahead_slots: int
168
169 @classmethod
113 - def create_empty(cls) -> "SchedulerDecodeOutputs":
114 - return SchedulerDecodeOutputs(
115 - seq_groups=[],
170 + def create_empty(cls) -> "SchedulerRunningOutputs":
171 + return SchedulerRunningOutputs(
172 + decode_seq_groups=[],
173 + prefill_seq_groups=[],
174 preempted=[],
175 swapped_out=[],
176 blocks_to_swap_out={},
@@ -123,20 +181,28 @@ class SchedulerDecodeOutputs:
181
182 @dataclass
183 class SchedulerSwappedInOutputs:
126 - """Outputs of the decoding phase of the scheduler."""
127 - # Selected sequence groups for decoding.
128 - seq_groups: List[SequenceGroup]
184 + """The requests that are scheduled from a swap queue.
185 +
186 + Could contain prefill (prefill that's chunked) or decodes.
187 + """
188 + # Selected sequences that are going to be swapped in and is in a
189 + # decoding phase.
190 + decode_seq_groups: List[SequenceGroup]
191 + # Selected sequences that are going to be swapped in and in a prefill
192 + # phase. I.e., it means the prefill has been chunked.
193 + prefill_seq_groups: List[SequenceGroup]
194 # The blocks to swap in.
195 blocks_to_swap_in: Dict[int, int]
196 # The blocks to copy.
197 blocks_to_copy: Dict[int, List[int]]
133 - # # The number of batched tokens.
198 + # The number of slots for lookahead decoding.
199 num_lookahead_slots: int
200
201 @classmethod
202 def create_empty(cls) -> "SchedulerSwappedInOutputs":
203 return SchedulerSwappedInOutputs(
139 - seq_groups=[],
204 + decode_seq_groups=[],
205 + prefill_seq_groups=[],
206 blocks_to_swap_in={},
207 blocks_to_copy={},
208 num_lookahead_slots=0,
@@ -145,8 +211,12 @@ class SchedulerSwappedInOutputs:
211
212 @dataclass
213 class SchedulerPrefillOutputs:
148 - """Outputs of the prefill phase of the scheduler."""
149 - # Selected sequence groups for prefill.
214 + """The requests that are scheduled from a waiting queue.
215 +
216 + Could contain a fresh prefill requests or preempted requests that need
217 + to be recomputed from scratch.
218 + """
219 + # Selected sequences for prefill.
220 seq_groups: List[SequenceGroup]
221 # Ignored sequence groups.
222 ignored_seq_groups: List[SequenceGroup]
@@ -176,12 +246,12 @@ class Scheduler:
246 # LoRAs. This should be improved in the future.
247 self.lora_config = lora_config
248
179 - # TODO(sang): Fix it after chunked prefill is enabled.
180 - self.prompt_limit = min(self.scheduler_config.max_model_len,
181 - self.scheduler_config.max_num_batched_tokens)
182 -
183 - # Instantiate the scheduling policy.
184 - self.policy = PolicyFactory.get_policy(policy_name="fcfs")
249 + if self.scheduler_config.chunked_prefill_enabled:
250 + self.prompt_limit = self.scheduler_config.max_model_len
251 + else:
252 + self.prompt_limit = min(
253 + self.scheduler_config.max_model_len,
254 + self.scheduler_config.max_num_batched_tokens)
255
256 BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
257 version="v2" if self.scheduler_config.
@@ -268,21 +338,17 @@ class Scheduler:
338 def get_num_unfinished_seq_groups(self) -> int:
339 return len(self.waiting) + len(self.running) + len(self.swapped)
340
271 - def _schedule_decodes(
341 + def _schedule_running(
342 self,
343 running_queue: deque,
344 budget: SchedulingBudget,
345 curr_loras: Optional[Set[int]],
346 policy: Policy,
277 - ) -> Tuple[deque, SchedulerDecodeOutputs]:
278 - """Schedule sequence groups in a decoding stage.
279 -
280 - NOTE(sang): All the RUNNING num_batched_tokens, num_curr_seqs,
281 - and curr_loras should be already included in `budget` and `curr_loras`.
282 - The API doesn't ADD UP these values.
347 + enable_chunking: bool = False,
348 + ) -> Tuple[deque, SchedulerRunningOutputs]:
349 + """Schedule sequence groups that are running.
350
284 - Note that `budget` and `curr_loras` are still subtracted/popped when
285 - any running requests are preempted from this API.
351 + Running queue should include decode and chunked prefill requests.
352
353 Args:
354 running_queue: The queue that contains running requests (i.e.,
@@ -292,16 +358,21 @@ class Scheduler:
358 curr_loras: Currently batched lora request ids. The argument is
359 in-place updated when any decodes are preempted.
360 policy: The sorting policy to sort running_queue.
295 -
361 + enable_chunking: If True, seq group can be chunked and only a
362 + chunked number of tokens are scheduled if
363 + `budget.num_batched_tokens` has not enough capacity to schedule
364 + all tokens.
365 +
366 Returns:
367 A tuple of remaining running queue (should be always 0) after
298 - scheduling and SchedulerDecodeOutputs.
368 + scheduling and SchedulerRunningOutputs.
369 """
370 # Blocks that need to be swapped or copied before model execution.
371 blocks_to_swap_out: Dict[int, int] = {}
372 blocks_to_copy: Dict[int, List[int]] = {}
373
304 - seq_groups: List[ScheduledSequenceGroup] = []
374 + decode_seq_groups: List[ScheduledSequenceGroup] = []
375 + prefill_seq_groups: List[ScheduledSequenceGroup] = []
376 preempted: List[SequenceGroup] = []
377 swapped_out: List[SequenceGroup] = []
378
@@ -313,18 +384,21 @@ class Scheduler:
384 running_queue = policy.sort_by_priority(now, running_queue)
385
386 while running_queue:
316 - # NOTE: running
387 seq_group = running_queue[0]
318 - num_running_tokens = (
319 - seq_group.num_seqs(status=SequenceStatus.RUNNING) *
320 - self.num_decoding_tokens_per_seq)
388 + num_running_tokens = self._get_num_new_tokens(
389 + seq_group, SequenceStatus.RUNNING, enable_chunking, budget)
390 +
391 + # We can have up to 1 running prefill at any given time in running
392 + # queue, which means we can guarantee chunk size is at least 1.
393 + assert num_running_tokens != 0
394 num_running_seqs = seq_group.get_max_num_running_seqs()
395
396 running_queue.popleft()
397 while not self._can_append_slots(seq_group):
325 - # Increase the budget as requests are preempted.
326 - budget.num_batched_tokens -= num_running_tokens
327 - budget.num_curr_seqs -= num_running_seqs
398 + budget.subtract_num_batched_tokens(seq_group.request_id,
399 + num_running_tokens)
400 + budget.subtract_num_seqs(seq_group.request_id,
401 + num_running_seqs)
402 if curr_loras is not None and seq_group.lora_int_id > 0:
403 curr_loras.pop(seq_group.lora_int_id)
404
@@ -350,14 +424,28 @@ class Scheduler:
424 else:
425 logger.debug(f"append slot for {seq_group}")
426 self._append_slots(seq_group, blocks_to_copy)
353 - seq_groups.append(
354 - ScheduledSequenceGroup(seq_group=seq_group,
355 - token_chunk_size=1))
427 + is_prefill = seq_group.is_prefill()
428 + if is_prefill:
429 + prefill_seq_groups.append(
430 + ScheduledSequenceGroup(
431 + seq_group=seq_group,
432 + token_chunk_size=num_running_tokens))
433 + else:
434 + decode_seq_groups.append(
435 + ScheduledSequenceGroup(seq_group=seq_group,
436 + token_chunk_size=1))
437 + budget.add_num_batched_tokens(seq_group.request_id,
438 + num_running_tokens)
439 + budget.add_num_seqs(seq_group.request_id, num_running_seqs)
440 + if curr_loras is not None and seq_group.lora_int_id > 0:
441 + curr_loras.add(seq_group.lora_int_id)
442 +
443 # Make sure all queues are updated.
444 assert len(running_queue) == 0
445
359 - return running_queue, SchedulerDecodeOutputs(
360 - seq_groups=seq_groups,
446 + return running_queue, SchedulerRunningOutputs(
447 + decode_seq_groups=decode_seq_groups,
448 + prefill_seq_groups=prefill_seq_groups,
449 preempted=preempted,
450 swapped_out=swapped_out,
451 blocks_to_swap_out=blocks_to_swap_out,
@@ -371,6 +459,7 @@ class Scheduler:
459 budget: SchedulingBudget,
460 curr_loras: Optional[Set[int]],
461 policy: Policy,
462 + enable_chunking: bool = False,
463 ) -> Tuple[deque, SchedulerSwappedInOutputs]:
464 """Schedule sequence groups that are swapped out.
465
@@ -386,7 +475,11 @@ class Scheduler:
475 curr_loras: Currently batched lora request ids. The argument is
476 in-place updated when any requests are swapped in.
477 policy: The sorting policy to sort swapped_queue.
389 -
478 + enable_chunking: If True, seq group can be chunked and only a
479 + chunked number of tokens are scheduled if
480 + `budget.num_batched_tokens` has not enough capacity to schedule
481 + all tokens.
482 +
483 Returns:
484 A tuple of remaining swapped_queue after scheduling and
485 SchedulerSwappedInOutputs.
@@ -394,7 +487,8 @@ class Scheduler:
487 # Blocks that need to be swapped or copied before model execution.
488 blocks_to_swap_in: Dict[int, int] = {}
489 blocks_to_copy: Dict[int, List[int]] = {}
397 - seq_groups: List[ScheduledSequenceGroup] = []
490 + decode_seq_groups: List[ScheduledSequenceGroup] = []
491 + prefill_seq_groups: List[ScheduledSequenceGroup] = []
492 now = time.time()
493 swapped_queue = policy.sort_by_priority(now, swapped_queue)
494
@@ -420,12 +514,13 @@ class Scheduler:
514 # The total number of sequences in the RUNNING state should not
515 # exceed the maximum number of sequences.
516 num_new_seqs = seq_group.get_max_num_running_seqs()
423 - num_new_tokens = (
424 - seq_group.num_seqs(status=SequenceStatus.SWAPPED) *
425 - self.num_decoding_tokens_per_seq)
517 + num_new_tokens = self._get_num_new_tokens(seq_group,
518 + SequenceStatus.SWAPPED,
519 + enable_chunking, budget)
520
427 - if not budget.can_schedule(num_new_tokens=num_new_tokens,
428 - num_new_seqs=num_new_seqs):
521 + if (num_new_tokens == 0
522 + or not budget.can_schedule(num_new_tokens=num_new_tokens,
523 + num_new_seqs=num_new_seqs)):
524 break
525
526 if lora_int_id > 0 and curr_loras is not None:
@@ -433,15 +528,23 @@ class Scheduler:
528 swapped_queue.popleft()
529 self._swap_in(seq_group, blocks_to_swap_in)
530 self._append_slots(seq_group, blocks_to_copy)
436 - seq_groups.append(
437 - ScheduledSequenceGroup(seq_group, token_chunk_size=1))
438 - budget.num_batched_tokens += num_new_tokens
439 - budget.num_curr_seqs += num_new_seqs
531 + is_prefill = seq_group.is_prefill()
532 + if is_prefill:
533 + prefill_seq_groups.append(
534 + ScheduledSequenceGroup(seq_group,
535 + token_chunk_size=num_new_tokens))
536 + else:
537 + assert num_new_tokens == 1
538 + decode_seq_groups.append(
539 + ScheduledSequenceGroup(seq_group, token_chunk_size=1))
540 + budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
541 + budget.add_num_seqs(seq_group.request_id, num_new_seqs)
542
543 swapped_queue.extendleft(leftover_swapped)
544
545 return swapped_queue, SchedulerSwappedInOutputs(
444 - seq_groups=seq_groups,
546 + decode_seq_groups=decode_seq_groups,
547 + prefill_seq_groups=prefill_seq_groups,
548 blocks_to_swap_in=blocks_to_swap_in,
549 blocks_to_copy=blocks_to_copy,
550 num_lookahead_slots=self._get_num_lookahead_slots(
@@ -452,6 +555,7 @@ class Scheduler:
555 waiting_queue: deque,
556 budget: SchedulingBudget,
557 curr_loras: Optional[Set[int]],
558 + enable_chunking: bool = False,
559 ) -> Tuple[deque, SchedulerPrefillOutputs]:
560 """Schedule sequence groups that are in prefill stage.
561
@@ -470,6 +574,10 @@ class Scheduler:
574 when any requests are scheduled.
575 curr_loras: Currently batched lora request ids. The argument is
576 in-place updated when any requests are scheduled.
577 + enable_chunking: If True, seq group can be chunked and only a
578 + chunked number of tokens are scheduled if
579 + `budget.num_batched_tokens` has not enough capacity to schedule
580 + all tokens.
581
582 Returns:
583 A tuple of remaining waiting_queue after scheduling and
@@ -489,11 +597,16 @@ class Scheduler:
597 assert len(waiting_seqs) == 1, (
598 "Waiting sequence group should have only one prompt "
599 "sequence.")
492 -
493 - num_prompt_tokens = waiting_seqs[0].get_len()
494 - if num_prompt_tokens > self.prompt_limit:
600 + num_new_tokens = self._get_num_new_tokens(seq_group,
601 + SequenceStatus.WAITING,
602 + enable_chunking, budget)
603 + if not enable_chunking:
604 + num_prompt_tokens = waiting_seqs[0].get_len()
605 + assert num_new_tokens == num_prompt_tokens
606 +
607 + if num_new_tokens > self.prompt_limit:
608 logger.warning(
496 - f"Input prompt ({num_prompt_tokens} tokens) is too long"
609 + f"Input prompt ({num_new_tokens} tokens) is too long"
610 f" and exceeds limit of {self.prompt_limit}")
611 for seq in waiting_seqs:
612 seq.status = SequenceStatus.FINISHED_IGNORED
@@ -507,7 +620,7 @@ class Scheduler:
620 break
621 elif can_allocate == AllocStatus.NEVER:
622 logger.warning(
510 - f"Input prompt ({num_prompt_tokens} tokens) is too long"
623 + f"Input prompt ({num_new_tokens} tokens) is too long"
624 f" and exceeds the capacity of block_manager")
625 for seq in waiting_seqs:
626 seq.status = SequenceStatus.FINISHED_IGNORED
@@ -528,20 +641,21 @@ class Scheduler:
641 continue
642
643 num_new_seqs = seq_group.get_max_num_running_seqs()
531 - if not budget.can_schedule(num_new_tokens=num_prompt_tokens,
532 - num_new_seqs=num_new_seqs):
644 + if (num_new_tokens == 0
645 + or not budget.can_schedule(num_new_tokens=num_new_tokens,
646 + num_new_seqs=num_new_seqs)):
647 break
648
649 # Can schedule this request.
650 if curr_loras is not None and lora_int_id > 0:
651 curr_loras.add(lora_int_id)
652 waiting_queue.popleft()
539 - self._allocate_and_set_running(seq_group)
653 + self._allocate_and_set_running(seq_group, num_new_tokens)
654 seq_groups.append(
655 ScheduledSequenceGroup(seq_group=seq_group,
542 - token_chunk_size=num_prompt_tokens))
543 - budget.num_batched_tokens += num_prompt_tokens
544 - budget.num_curr_seqs += num_new_seqs
656 + token_chunk_size=num_new_tokens))
657 + budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
658 + budget.add_num_seqs(seq_group.request_id, num_new_seqs)
659
660 # Queue requests that couldn't be scheduled.
661 waiting_queue.extendleft(leftover_waiting_sequences)
@@ -553,8 +667,8 @@ class Scheduler:
667 ignored_seq_groups=ignored_seq_groups,
668 num_lookahead_slots=self._get_num_lookahead_slots(is_prefill=True))
669
556 - def _schedule(self) -> SchedulerOutputs:
557 - """Batch requests that are queued..
670 + def _schedule_default(self) -> SchedulerOutputs:
671 + """Schedule queued requests.
672
673 The current policy is designed to opimimize the throughput. First,
674 it batches as many prefill requests as possible. And it schedules
@@ -563,39 +677,48 @@ class Scheduler:
677 """
678 # Include running requests to the budget.
679 budget = SchedulingBudget(
566 - num_batched_tokens=sum(
567 - seq_group.num_seqs(status=SequenceStatus.RUNNING)
568 - for seq_group in self.running),
569 - num_curr_seqs=sum(seq_group.get_max_num_running_seqs()
570 - for seq_group in self.running),
680 token_budget=self.scheduler_config.max_num_batched_tokens,
681 max_num_seqs=self.scheduler_config.max_num_seqs,
682 )
683 + # Make sure we include num running seqs before scheduling prefill,
684 + # so that we don't schedule beyond max_num_seqs for prefill.
685 + for seq_group in self.running:
686 + budget.add_num_seqs(seq_group.request_id,
687 + seq_group.get_max_num_running_seqs())
688 curr_loras = set(
689 seq_group.lora_int_id
690 for seq_group in self.running) if self.lora_enabled else None
691
692 remaining_waiting, prefills = (self.waiting,
693 SchedulerPrefillOutputs.create_empty())
580 - remaining_running, decodes = (self.running,
581 - SchedulerDecodeOutputs.create_empty())
694 + remaining_running, running_scheduled = (
695 + self.running, SchedulerRunningOutputs.create_empty())
696 remaining_swapped, swapped_in = (
697 self.swapped, SchedulerSwappedInOutputs.create_empty())
698
699 # If any requests are swapped, prioritized swapped requests.
700 if not self.swapped:
701 remaining_waiting, prefills = self._schedule_prefills(
588 - self.waiting, budget, curr_loras)
702 + self.waiting, budget, curr_loras, enable_chunking=False)
703
704 + fcfs_policy = PolicyFactory.get_policy(policy_name="fcfs")
705 # Don't schedule decodes if prefills are scheduled.
706 + # NOTE: If `_schedule_prefills` doesn't enable chunking, self.running
707 + # only contains decode requests, not chunked prefills.
708 if len(prefills.seq_groups) == 0:
592 - remaining_running, decodes = self._schedule_decodes(
593 - self.running, budget, curr_loras, self.policy)
709 + remaining_running, running_scheduled = self._schedule_running(
710 + self.running,
711 + budget,
712 + curr_loras,
713 + fcfs_policy,
714 + enable_chunking=False)
715 +
716 # If any sequence group is preempted, do not swap in any sequence
717 # group. because it means there's no slot for new running requests.
596 - if len(decodes.preempted) + len(decodes.swapped_out) == 0:
718 + if len(running_scheduled.preempted) + len(
719 + running_scheduled.swapped_out) == 0:
720 remaining_swapped, swapped_in = self._schedule_swapped(
598 - self.swapped, budget, curr_loras, self.policy)
721 + self.swapped, budget, curr_loras, fcfs_policy)
722
723 assert (budget.num_batched_tokens <=
724 self.scheduler_config.max_num_batched_tokens)
@@ -603,31 +726,134 @@ class Scheduler:
726
727 # Update waiting requests.
728 self.waiting = remaining_waiting
606 - self.waiting.extendleft(decodes.preempted)
729 + self.waiting.extendleft(running_scheduled.preempted)
730 # Update new running requests.
731 self.running = remaining_running
732 self.running.extend([s.seq_group for s in prefills.seq_groups])
610 - self.running.extend([s.seq_group for s in decodes.seq_groups])
611 - self.running.extend([s.seq_group for s in swapped_in.seq_groups])
733 + self.running.extend(
734 + [s.seq_group for s in running_scheduled.decode_seq_groups])
735 + self.running.extend(
736 + [s.seq_group for s in swapped_in.decode_seq_groups])
737 # Update swapped requests.
738 self.swapped = remaining_swapped
614 - self.swapped.extend(decodes.swapped_out)
739 + self.swapped.extend(running_scheduled.swapped_out)
740
741 + # There should be no prefill from running queue because this policy
742 + # doesn't allow chunked prefills.
743 + assert len(running_scheduled.prefill_seq_groups) == 0
744 + assert len(swapped_in.prefill_seq_groups) == 0
745 return SchedulerOutputs(
617 - scheduled_seq_groups=prefills.seq_groups + decodes.seq_groups +
618 - swapped_in.seq_groups,
746 + scheduled_seq_groups=(prefills.seq_groups +
747 + running_scheduled.decode_seq_groups +
748 + swapped_in.decode_seq_groups),
749 num_prefill_groups=len(prefills.seq_groups),
750 num_batched_tokens=budget.num_batched_tokens,
751 blocks_to_swap_in=swapped_in.blocks_to_swap_in,
622 - blocks_to_swap_out=decodes.blocks_to_swap_out,
623 - blocks_to_copy=merge_dicts(decodes.blocks_to_copy,
752 + blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
753 + blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
754 + swapped_in.blocks_to_copy),
755 + ignored_seq_groups=prefills.ignored_seq_groups,
756 + num_lookahead_slots=(prefills.num_lookahead_slots +
757 + running_scheduled.num_lookahead_slots +
758 + swapped_in.num_lookahead_slots),
759 + )
760 +
761 + def _schedule_chunked_prefill(self):
762 + """Schedule queued requests.
763 +
764 + Chunked prefill allows to chunk prefill requests, batch them together
765 + with decode requests. This policy 1. schedule as many decoding requests
766 + as possible. 2. schedule chunked prefill requests that are not
767 + finished. 3. schedule swapped request. 4. schedule new prefill
768 + requests.
769 +
770 + The policy can sustain the high GPU utilization because it can put
771 + prefill and decodes requests to the same batch, while it improves
772 + inter token latency because decodes requests don't need to blocked
773 + by prefill requests.
774 + """
775 + budget = SchedulingBudget(
776 + token_budget=self.scheduler_config.max_num_batched_tokens,
777 + max_num_seqs=self.scheduler_config.max_num_seqs,
778 + )
779 + curr_loras = set()
780 +
781 + remaining_waiting, prefills = (self.waiting,
782 + SchedulerPrefillOutputs.create_empty())
783 + remaining_running, running_scheduled = (
784 + self.running, SchedulerRunningOutputs.create_empty())
785 + remaining_swapped, swapped_in = (
786 + self.swapped, SchedulerSwappedInOutputs.create_empty())
787 +
788 + # Decoding should be always scheduled first by fcfs.
789 + fcfs_policy = PolicyFactory.get_policy(policy_name="fcfs")
790 + remaining_running, running_scheduled = self._schedule_running(
791 + self.running,
792 + budget,
793 + curr_loras,
794 + fcfs_policy,
795 + enable_chunking=True)
796 +
797 + # Schedule swapped out requests.
798 + # If preemption happens, it means we don't have space for swap-in.
799 + if len(running_scheduled.preempted) + len(
800 + running_scheduled.swapped_out) == 0:
801 + remaining_swapped, swapped_in = self._schedule_swapped(
802 + self.swapped, budget, curr_loras, fcfs_policy)
803 +
804 + # Schedule new prefills.
805 + remaining_waiting, prefills = self._schedule_prefills(
806 + self.waiting, budget, curr_loras, enable_chunking=True)
807 +
808 + assert (budget.num_batched_tokens <=
809 + self.scheduler_config.max_num_batched_tokens)
810 + assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
811 +
812 + # Update waiting requests.
813 + self.waiting = remaining_waiting
814 + self.waiting.extendleft(running_scheduled.preempted)
815 + # Update new running requests.
816 + self.running = remaining_running
817 + self.running.extend([s.seq_group for s in prefills.seq_groups])
818 + self.running.extend(
819 + [s.seq_group for s in running_scheduled.decode_seq_groups])
820 + self.running.extend(
821 + [s.seq_group for s in running_scheduled.prefill_seq_groups])
822 + self.running.extend(
823 + [s.seq_group for s in swapped_in.decode_seq_groups])
824 + self.running.extend(
825 + [s.seq_group for s in swapped_in.prefill_seq_groups])
826 + # Update swapped requests.
827 + self.swapped = remaining_swapped
828 + self.swapped.extend(running_scheduled.swapped_out)
829 +
830 + return SchedulerOutputs(
831 + scheduled_seq_groups=(prefills.seq_groups +
832 + running_scheduled.decode_seq_groups +
833 + running_scheduled.prefill_seq_groups +
834 + swapped_in.decode_seq_groups +
835 + swapped_in.prefill_seq_groups),
836 + num_prefill_groups=(len(prefills.seq_groups) +
837 + len(swapped_in.prefill_seq_groups) +
838 + len(running_scheduled.prefill_seq_groups)),
839 + num_batched_tokens=budget.num_batched_tokens,
840 + blocks_to_swap_in=swapped_in.blocks_to_swap_in,
841 + blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
842 + blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
843 swapped_in.blocks_to_copy),
844 ignored_seq_groups=prefills.ignored_seq_groups,
845 num_lookahead_slots=(prefills.num_lookahead_slots +
627 - decodes.num_lookahead_slots +
846 + running_scheduled.num_lookahead_slots +
847 swapped_in.num_lookahead_slots),
848 )
849
850 + def _schedule(self) -> SchedulerOutputs:
851 + """Schedule queued requests."""
852 + if self.scheduler_config.chunked_prefill_enabled:
853 + return self._schedule_chunked_prefill()
854 + else:
855 + return self._schedule_default()
856 +
857 def _can_append_slots(self, seq_group: SequenceGroup) -> bool:
858 """Determine whether or not we have enough space in the KV cache to
859 continue generation of the sequence group.
@@ -722,7 +948,8 @@ class Scheduler:
948 self.running = deque(seq_group for seq_group in self.running
949 if not seq_group.is_finished())
950
725 - def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
951 + def _allocate_and_set_running(self, seq_group: SequenceGroup,
952 + num_new_tokens: int) -> None:
953 self.block_manager.allocate(seq_group)
954 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
955 seq.status = SequenceStatus.RUNNING
@@ -854,3 +1081,26 @@ class Scheduler:
1081 return 0
1082
1083 return self.scheduler_config.num_lookahead_slots
1084 +
1085 + def _get_num_new_tokens(self, seq_group: SequenceGroup,
1086 + status: SequenceStatus, enable_chunking: bool,
1087 + budget: SchedulingBudget) -> Tuple[int, bool]:
1088 + """Get the next new tokens to compute for a given sequence group
1089 + that's in a given `status`.
1090 +
1091 + The API could chunk the number of tokens to compute based on `budget`
1092 + if `enable_chunking` is True. If a sequence group has multiple
1093 + sequences (e.g., running beam search), it means it is in decoding
1094 + phase, so chunking doesn't happen.
1095 + """
1096 + num_new_tokens = 0
1097 + seqs = seq_group.get_seqs(status=status)
1098 + for seq in seqs:
1099 + num_new_tokens += seq.get_num_new_tokens()
1100 + # Chunk if a running request cannot fit in.
1101 + # If number of seq > 1, it means it is doing beam search in a
1102 + # decode phase. Do not chunk in that case.
1103 + if enable_chunking and len(seqs) == 1:
1104 + num_new_tokens = min(num_new_tokens,
1105 + budget.remaining_token_budget())
1106 + return num_new_tokens
vllm/engine/llm_engine.py
+2 -3
@@ -607,11 +607,10 @@ class LLMEngine:
607 now = time.time()
608 # Update the scheduled sequence groups with the model outputs.
609 scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
610 -
610 for scheduled_seq_group, outputs in zip(scheduled_seq_groups, output):
611 seq_group = scheduled_seq_group.seq_group
613 - token_chunk_size = scheduled_seq_group.token_chunk_size
614 - seq_group.update_num_computed_tokens(token_chunk_size)
612 + seq_group.update_num_computed_tokens(
613 + scheduled_seq_group.token_chunk_size)
614 self._process_sequence_group_outputs(seq_group, outputs)
615
616 # Free the finished sequence groups.
vllm/sequence.py
+48 -11
@@ -69,6 +69,11 @@ class SequenceStatus(enum.Enum):
69 return finish_reason
70
71
72 +class SequenceStage(enum.Enum):
73 + PREFILL = enum.auto()
74 + DECODE = enum.auto()
75 +
76 +
77 @dataclass
78 class RequestMetrics:
79 """Metrics associated with a request.
@@ -115,6 +120,7 @@ class SequenceData:
120 self.cumulative_logprob = 0.0
121 # The number of tokens that are computed (that run against the model).
122 self._num_computed_tokens = 0
123 + self._stage: SequenceStage = SequenceStage.PREFILL
124
125 def append_token_id(self, token_id: int, logprob: float) -> None:
126 self.output_token_ids.append(token_id)
@@ -136,16 +142,22 @@ class SequenceData:
142 """Return the number of prefill tokens that are already computed."""
143 return self._num_computed_tokens
144
139 - def update_num_computed_tokens(self, num_new_computed_tokens: int) -> int:
145 + def update_num_computed_tokens(self, num_new_computed_tokens: int):
146 """Update number of tokens computed so far."""
147 self._num_computed_tokens += num_new_computed_tokens
148 + assert self._num_computed_tokens <= self.get_len(), (
149 + self._num_computed_tokens, self.get_len())
150 + # If all tokens are computed, it means it is in decoding phase.
151 + if self.get_num_uncomputed_tokens() == 0:
152 + self._stage = SequenceStage.DECODE
153
143 - def reset_num_computed_tokens(self) -> None:
154 + def reset_state_for_recompute(self) -> None:
155 """Reset the number of computed tokens from this sequence. It is
156 supposed to be called when a sequence needs to be started from
157 the beginning again (e.g., sequence is preempted).
158 """
159 self._num_computed_tokens = 0
160 + self._stage = SequenceStage.PREFILL
161
162 def get_num_uncomputed_tokens(self) -> int:
163 """Return the number of prefil tokens that are not computed."""
@@ -165,6 +177,10 @@ class SequenceData:
177 def get_output_token_ids(self) -> int:
178 return self.output_token_ids
179
180 + @property
181 + def stage(self) -> SequenceStage:
182 + return self._stage
183 +
184 def __repr__(self) -> str:
185 return (f"SequenceData("
186 f"prompt_token_ids={self.prompt_token_ids}, "
@@ -234,7 +250,7 @@ class Sequence:
250
251 def reset_state_for_recompute(self):
252 """Reset the sequence states for recomputation."""
237 - self.data.reset_num_computed_tokens()
253 + self.data.reset_state_for_recompute()
254
255 def _append_logical_block(self) -> None:
256 block = LogicalTokenBlock(
@@ -320,6 +336,23 @@ class Sequence:
336 new_seq.seq_id = new_seq_id
337 return new_seq
338
339 + def get_num_new_tokens(self) -> int:
340 + """Get the number of new tokens to be computed.
341 +
342 + Args:
343 + remainig_token_budget: The remaining token budgets.
344 + Returns:
345 + The new number of tokens to be computed. I.e., 1 for decode, prompt
346 + size for prefill. If there's not enough remainig_token_budget, it
347 + can return the chunked number of new tokens.
348 + """
349 + if self.data.stage == SequenceStage.DECODE:
350 + return 1
351 + return self.data.get_num_uncomputed_tokens()
352 +
353 + def is_prefill(self) -> bool:
354 + return self.data.stage == SequenceStage.PREFILL
355 +
356 def __repr__(self) -> str:
357 return (f"Sequence(seq_id={self.seq_id}, "
358 f"status={self.status.name}, "
@@ -461,14 +494,14 @@ class SequenceGroup:
494 def update_num_computed_tokens(self, num_new_computed_tokens: int):
495 """Update number of tokens computed so far."""
496 for seq in self.seqs_dict.values():
464 - seq.data.update_num_computed_tokens(num_new_computed_tokens)
497 + if not seq.is_finished():
498 + seq.data.update_num_computed_tokens(num_new_computed_tokens)
499
500 def get_num_uncomputed_tokens(self) -> int:
467 - # All sequences in the group should have the same prompt, so the
468 - # number of unfinished prefill tokens are the same across all
469 - # sequences.
470 - return list(
471 - self.seqs_dict.values())[0].data.get_num_uncomputed_tokens()
501 + num_uncomputed_tokens = 0
502 + for seq in self.get_seqs():
503 + num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
504 + return num_uncomputed_tokens
505
506 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
507 return len(self.get_seqs(status))
@@ -497,6 +530,10 @@ class SequenceGroup:
530 def is_finished(self) -> bool:
531 return all(seq.is_finished() for seq in self.get_seqs())
532
533 + def is_prefill(self) -> bool:
534 + # Every sequences should be in the same stage.
535 + return self.get_seqs()[0].is_prefill()
536 +
537 def __repr__(self) -> str:
538 return (f"SequenceGroup(request_id={self.request_id}, "
539 f"sampling_params={self.sampling_params}, "
@@ -513,8 +550,8 @@ class SequenceGroupMetadata:
550 sampling_params: The sampling parameters used to generate the outputs.
551 block_tables: The block tables. (Seq id -> list of physical block
552 numbers)
516 - token_chunk_size: The number of tokens to be processed. None if
517 - chunking is not required.
553 + token_chunk_size: The number of tokens to be processed (per sequence).
554 + None if chunking is not required.
555 state: Internal state tied to this sequence group.
556 lora_request: LoRA request.
557 multi_modal_data: Multi modal data.
vllm/worker/model_runner.py
-1
@@ -222,7 +222,6 @@ class ModelRunner:
222 # NOTE(woosuk): Here we assume that the first token in the prompt
223 # is always the first token in the sequence.
224 input_positions.extend(list(range(computed_len, prefill_end)))
225 -
225 lora_id = seq_group_metadata.lora_int_id
226
227 if lora_id > 0: