A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Chunked Prefill][4/n] Chunked prefill scheduler. (#3853)
SangBin Cho committed
Apr 6, 2024 at 02:17 UTC
18de88348954b7e535a62c0b7e55004f253e9f21
10 files changed
+1218
-183
requirements-common.txt
+1
-1
@@ -11,4 +11,4 @@ uvicorn[standard]
11
pydantic >= 2.0 # Required for OpenAI server.
12
prometheus_client >= 0.18.0
13
tiktoken == 0.6.0 # Required for DBRX tokenizer
14
-outlines == 0.0.34 # Requires torch >= 2.1.0
14
+outlines == 0.0.34 # Requires torch >= 2.1.0
\ No newline at end of file
tests/core/test_chunked_prefill_scheduler.py
new
+563
@@ -0,0 +1,563 @@
1
+from typing import List
2
+from unittest.mock import MagicMock
3
+
4
+import pytest # noqa
5
+
6
+from vllm.config import CacheConfig, SchedulerConfig
7
+from vllm.core.scheduler import Scheduler
8
+from vllm.sequence import Logprob, SequenceGroup
9
+
10
+from .utils import create_dummy_prompt
11
+
12
+
13
+def get_sequence_groups(scheduler_output):
14
+ return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
15
+
16
+
17
+def append_new_token(seq_group, token_id: int):
18
+ for seq in seq_group.get_seqs():
19
+ seq.append_token_id(token_id, {token_id: Logprob(token_id)})
20
+
21
+
22
+def schedule_and_update_computed_tokens(scheduler):
23
+ metas, out = scheduler.schedule()
24
+ for s, meta in zip(out.scheduled_seq_groups, metas):
25
+ s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
26
+ return metas, out
27
+
28
+
29
+def test_simple():
30
+ """Verify basic scheduling works."""
31
+ block_size = 4
32
+ num_seq_group = 4
33
+ max_model_len = 16
34
+ max_num_batched_tokens = 64
35
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
36
+ num_seq_group,
37
+ max_model_len,
38
+ enable_chunked_prefill=True)
39
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
40
+ cache_config.num_cpu_blocks = 8
41
+ cache_config.num_gpu_blocks = 8
42
+ scheduler = Scheduler(scheduler_config, cache_config, None)
43
+ running: List[SequenceGroup] = []
44
+
45
+ # Add seq groups to scheduler.
46
+ for i in range(num_seq_group):
47
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
48
+ scheduler.add_seq_group(seq_group)
49
+ running.append(seq_group)
50
+
51
+ # Schedule seq groups prompts.
52
+ num_tokens = block_size * num_seq_group
53
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
54
+ assert set(get_sequence_groups(out)) == set(running)
55
+ assert out.num_batched_tokens == num_tokens
56
+ assert (not out.blocks_to_copy and not out.blocks_to_swap_in
57
+ and not out.blocks_to_swap_out)
58
+ assert len(seq_group_meta) == num_seq_group
59
+ for s in running:
60
+ append_new_token(s, 1)
61
+
62
+ # Schedule seq groups generation.
63
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
64
+ assert set(get_sequence_groups(out)) == set(running)
65
+ assert out.num_batched_tokens == num_seq_group
66
+ assert (not out.blocks_to_copy and not out.blocks_to_swap_in
67
+ and not out.blocks_to_swap_out)
68
+ assert len(seq_group_meta) == num_seq_group
69
+
70
+
71
+def test_chunk():
72
+ """Verify prefills are chunked properly."""
73
+ block_size = 4
74
+ max_seqs = 60
75
+ max_model_len = 80
76
+ max_num_batched_tokens = 64
77
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
78
+ max_seqs,
79
+ max_model_len,
80
+ enable_chunked_prefill=True)
81
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
82
+ cache_config.num_cpu_blocks = 8
83
+ cache_config.num_gpu_blocks = 8
84
+ scheduler = Scheduler(scheduler_config, cache_config, None)
85
+ running: List[SequenceGroup] = []
86
+
87
+ # Add seq groups to scheduler.
88
+ for i in range(2):
89
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
90
+ scheduler.add_seq_group(seq_group)
91
+ running.append(seq_group)
92
+
93
+ # Verify the second request is chunked.
94
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
95
+ assert set(get_sequence_groups(out)) == set(running)
96
+ assert seq_group_meta[0].token_chunk_size == 60
97
+ # Verify it is chunked.
98
+ assert seq_group_meta[1].token_chunk_size == 4
99
+ assert out.num_prefill_groups == 2
100
+ assert out.num_batched_tokens == 64
101
+ # Only the first seq group has a new token appended.
102
+ append_new_token(running[0], 1)
103
+
104
+ # One chunked prefill, and one decoding.
105
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
106
+ assert set(get_sequence_groups(out)) == set(running)
107
+ # The first one is decoding.
108
+ assert seq_group_meta[0].token_chunk_size == 1
109
+ # The second one is a chunked prefill.
110
+ assert seq_group_meta[1].token_chunk_size == 56
111
+ assert out.num_prefill_groups == 1
112
+ assert out.num_batched_tokens == 57
113
+
114
+
115
+def test_complex():
116
+ block_size = 4
117
+ max_seqs = 60
118
+ max_model_len = 80
119
+ max_num_batched_tokens = 64
120
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
121
+ max_seqs,
122
+ max_model_len,
123
+ enable_chunked_prefill=True)
124
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
125
+ cache_config.num_cpu_blocks = 8
126
+ cache_config.num_gpu_blocks = 8
127
+ scheduler = Scheduler(scheduler_config, cache_config, None)
128
+ running: List[SequenceGroup] = []
129
+
130
+ # Add seq groups to scheduler.
131
+ for i in range(2):
132
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
133
+ scheduler.add_seq_group(seq_group)
134
+ running.append(seq_group)
135
+ assert seq_group.is_prefill()
136
+
137
+ # Verify the second request is chunked.
138
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
139
+
140
+ assert set(get_sequence_groups(out)) == set(running)
141
+ assert seq_group_meta[0].token_chunk_size == 60
142
+ # Verify it is chunked.
143
+ assert seq_group_meta[1].token_chunk_size == 4
144
+ assert not running[0].is_prefill()
145
+ assert running[1].is_prefill()
146
+ assert out.num_prefill_groups == 2
147
+ assert out.num_batched_tokens == 64
148
+ # Only the first seq group has a new token appended.
149
+ append_new_token(running[0], 1)
150
+
151
+ # Add 2 more requsets.
152
+ for i in range(2, 4):
153
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
154
+ scheduler.add_seq_group(seq_group)
155
+ running.append(seq_group)
156
+
157
+ # Decoding & chunked prefill & first chunk of 3rd request is scheduled.
158
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
159
+ assert len(get_sequence_groups(out)) == 3
160
+ # The first one is decoding.
161
+ assert seq_group_meta[0].token_chunk_size == 1
162
+ # The second one is a chunked prefill.
163
+ assert seq_group_meta[1].token_chunk_size == 56
164
+ # The third one is also chunked.
165
+ assert seq_group_meta[2].token_chunk_size == 7
166
+ # Two of them are in chunked prefill.
167
+ assert out.num_prefill_groups == 2
168
+ assert out.num_batched_tokens == 64
169
+ # The first 2 requests are now in decodine phase.
170
+ append_new_token(running[0], 1)
171
+ assert not running[0].is_prefill()
172
+ append_new_token(running[1], 1)
173
+ assert not running[1].is_prefill()
174
+ # The third request is still in prefill stage.
175
+ assert running[2].is_prefill()
176
+
177
+
178
+def test_maximal_decoding():
179
+ """Verify decoding requests are prioritized."""
180
+ block_size = 4
181
+ max_seqs = 2
182
+ max_model_len = 2
183
+ max_num_batched_tokens = 2
184
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
185
+ max_seqs,
186
+ max_model_len,
187
+ enable_chunked_prefill=True)
188
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
189
+ cache_config.num_cpu_blocks = 8
190
+ cache_config.num_gpu_blocks = 8
191
+ scheduler = Scheduler(scheduler_config, cache_config, None)
192
+ running: List[SequenceGroup] = []
193
+
194
+ # Add seq groups to scheduler.
195
+ for i in range(2):
196
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=2)
197
+ scheduler.add_seq_group(seq_group)
198
+ running.append(seq_group)
199
+ assert seq_group.is_prefill()
200
+
201
+ # The first prefill is scheduled.
202
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
203
+ assert len(get_sequence_groups(out)) == 1
204
+ assert seq_group_meta[0].token_chunk_size == 2
205
+ assert not running[0].is_prefill()
206
+ assert running[1].is_prefill()
207
+ assert out.num_prefill_groups == 1
208
+ assert out.num_batched_tokens == 2
209
+ # Only the first seq group has a new token appended.
210
+ append_new_token(running[0], 1)
211
+
212
+ # Create one more seq_group.
213
+ _, seq_group = create_dummy_prompt("3", prompt_length=2)
214
+ scheduler.add_seq_group(seq_group)
215
+ running.append(seq_group)
216
+ assert seq_group.is_prefill()
217
+ # The first decoding + second chunk is scheduled.
218
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
219
+ assert len(get_sequence_groups(out)) == 2
220
+ assert seq_group_meta[0].token_chunk_size == 1
221
+ assert seq_group_meta[1].token_chunk_size == 1
222
+ assert not running[0].is_prefill()
223
+ assert running[1].is_prefill()
224
+ assert running[2].is_prefill()
225
+ assert out.num_prefill_groups == 1
226
+ assert out.num_batched_tokens == 2
227
+ append_new_token(running[0], 1)
228
+
229
+ # Decoding + running prefill is prioritized.
230
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
231
+ assert len(get_sequence_groups(out)) == 2
232
+ assert seq_group_meta[0].token_chunk_size == 1
233
+ assert seq_group_meta[1].token_chunk_size == 1
234
+ assert not running[0].is_prefill()
235
+ assert not running[1].is_prefill()
236
+ assert out.num_prefill_groups == 1
237
+ assert out.num_batched_tokens == 2
238
+ append_new_token(running[0], 1)
239
+ append_new_token(running[1], 1)
240
+
241
+ # Only decoding is prioritized.
242
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
243
+ assert len(get_sequence_groups(out)) == 2
244
+ assert seq_group_meta[0].token_chunk_size == 1
245
+ assert seq_group_meta[1].token_chunk_size == 1
246
+ assert not running[0].is_prefill()
247
+ assert not running[1].is_prefill()
248
+ assert out.num_prefill_groups == 0
249
+ assert out.num_batched_tokens == 2
250
+ append_new_token(running[0], 1)
251
+ append_new_token(running[1], 1)
252
+
253
+ # After aborting the decoding request, the fcfs new prefill is prioritized.
254
+ scheduler.abort_seq_group(running[0].request_id)
255
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
256
+ assert len(get_sequence_groups(out)) == 2
257
+ assert seq_group_meta[0].token_chunk_size == 1
258
+ assert seq_group_meta[1].token_chunk_size == 1
259
+ assert not running[1].is_prefill()
260
+ assert running[2].is_prefill()
261
+ assert out.num_prefill_groups == 1
262
+ assert out.num_batched_tokens == 2
263
+
264
+
265
+def test_prompt_limit():
266
+ """Verify max_num_batched_tokens < max_model_len is possible."""
267
+ block_size = 4
268
+ max_seqs = 32
269
+ max_model_len = 64
270
+ max_num_batched_tokens = 32
271
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
272
+ max_seqs,
273
+ max_model_len,
274
+ enable_chunked_prefill=True)
275
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
276
+ cache_config.num_cpu_blocks = 8
277
+ cache_config.num_gpu_blocks = 8
278
+ scheduler = Scheduler(scheduler_config, cache_config, None)
279
+ running: List[SequenceGroup] = []
280
+
281
+ _, seq_group = create_dummy_prompt("1", prompt_length=48)
282
+ scheduler.add_seq_group(seq_group)
283
+ running.append(seq_group)
284
+ assert seq_group.is_prefill()
285
+
286
+ # The prompt length > max_num_batched_tokens should be still scheduled.
287
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
288
+ assert len(get_sequence_groups(out)) == 1
289
+ assert seq_group_meta[0].token_chunk_size == 32
290
+ assert running[0].is_prefill()
291
+ assert out.num_prefill_groups == 1
292
+ assert out.num_batched_tokens == 32
293
+
294
+
295
+def test_prompt_limit_exceed():
296
+ block_size = 4
297
+ max_seqs = 64
298
+ max_model_len = 32
299
+ max_num_batched_tokens = 64
300
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
301
+ max_seqs,
302
+ max_model_len,
303
+ enable_chunked_prefill=True)
304
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
305
+ cache_config.num_cpu_blocks = 8
306
+ cache_config.num_gpu_blocks = 8
307
+ scheduler = Scheduler(scheduler_config, cache_config, None)
308
+ running: List[SequenceGroup] = []
309
+
310
+ _, seq_group = create_dummy_prompt("2", prompt_length=48)
311
+ scheduler.add_seq_group(seq_group)
312
+ running.append(seq_group)
313
+ assert seq_group.is_prefill()
314
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
315
+ assert len(out.ignored_seq_groups) == 1
316
+ assert out.ignored_seq_groups[0] == seq_group
317
+
318
+
319
+def test_swap():
320
+ """Verify swapping works with chunked prefill requests"""
321
+ block_size = 4
322
+ max_seqs = 30
323
+ max_model_len = 200
324
+ max_num_batched_tokens = 30
325
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
326
+ max_seqs,
327
+ max_model_len,
328
+ enable_chunked_prefill=True)
329
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
330
+ cache_config.num_cpu_blocks = 8
331
+ cache_config.num_gpu_blocks = 8
332
+ scheduler = Scheduler(scheduler_config, cache_config, None)
333
+
334
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
335
+ scheduler.add_seq_group(seq_group)
336
+ _, out = schedule_and_update_computed_tokens(scheduler)
337
+ # The request is chunked.
338
+ # prefill scheduled now.
339
+ assert len(out.scheduled_seq_groups) == 1
340
+ assert out.num_prefill_groups == 1
341
+ assert seq_group.is_prefill()
342
+ assert out.num_batched_tokens == max_num_batched_tokens
343
+
344
+ # The last request should be swapped out.
345
+ scheduler.block_manager.can_append_slots = MagicMock()
346
+
347
+ def cannot_append_second_group(seq_group, num_lookahead_slots):
348
+ return seq_group.request_id != "1"
349
+
350
+ scheduler.block_manager.can_append_slots.side_effect = (
351
+ cannot_append_second_group)
352
+
353
+ # The running prefill is now swapped.
354
+ _, out = schedule_and_update_computed_tokens(scheduler)
355
+ assert len(out.scheduled_seq_groups) == 0
356
+ assert out.num_batched_tokens == 0
357
+ assert out.blocks_to_swap_out != {}
358
+ assert out.blocks_to_swap_in == {}
359
+
360
+ # Add 1 more task. Swap should be prioritized over new prefill.
361
+ _, seq_group = create_dummy_prompt("2", prompt_length=60)
362
+ scheduler.add_seq_group(seq_group)
363
+ _, out = schedule_and_update_computed_tokens(scheduler)
364
+ assert len(out.scheduled_seq_groups) == 1
365
+ # 3 decodes. It is swapped in.
366
+ assert out.num_batched_tokens == 30
367
+ assert out.blocks_to_swap_in != {}
368
+ assert out.blocks_to_swap_out == {}
369
+
370
+
371
+def test_running_prefill_prioritized_over_swap():
372
+ block_size = 4
373
+ max_seqs = 30
374
+ max_model_len = 200
375
+ max_num_batched_tokens = 30
376
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
377
+ max_seqs,
378
+ max_model_len,
379
+ enable_chunked_prefill=True)
380
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
381
+ cache_config.num_cpu_blocks = 8
382
+ cache_config.num_gpu_blocks = 8
383
+ scheduler = Scheduler(scheduler_config, cache_config, None)
384
+
385
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
386
+ scheduler.add_seq_group(seq_group)
387
+ _, out = schedule_and_update_computed_tokens(scheduler)
388
+ # The request is chunked.
389
+ # prefill scheduled now.
390
+ assert len(out.scheduled_seq_groups) == 1
391
+ assert out.num_prefill_groups == 1
392
+ assert seq_group.is_prefill()
393
+ assert out.num_batched_tokens == max_num_batched_tokens
394
+
395
+ # The request should be swapped out.
396
+ scheduler.block_manager.can_append_slots = MagicMock()
397
+
398
+ def cannot_append_second_group(seq_group, num_lookahead_slots):
399
+ return seq_group.request_id != "1"
400
+
401
+ scheduler.block_manager.can_append_slots.side_effect = (
402
+ cannot_append_second_group)
403
+
404
+ # The running prefill is now swapped.
405
+ _, out = schedule_and_update_computed_tokens(scheduler)
406
+ assert len(out.scheduled_seq_groups) == 0
407
+ assert out.num_batched_tokens == 0
408
+ assert out.blocks_to_swap_out != {}
409
+ assert out.blocks_to_swap_in == {}
410
+
411
+ # Add 1 more task. Swap is not possible, so prefill is running.
412
+ scheduler.block_manager.can_swap_in = MagicMock()
413
+ scheduler.block_manager.can_swap_in.return_value = False
414
+
415
+ _, seq_group2 = create_dummy_prompt("2", prompt_length=60)
416
+ scheduler.add_seq_group(seq_group2)
417
+ _, out = schedule_and_update_computed_tokens(scheduler)
418
+ assert len(out.scheduled_seq_groups) == 1
419
+ # 3 decodes. It is swapped in.
420
+ assert out.num_batched_tokens == 30
421
+ assert out.blocks_to_swap_in == {}
422
+ assert out.blocks_to_swap_out == {}
423
+ assert out.scheduled_seq_groups[0].seq_group == seq_group2
424
+
425
+ # Now although swap is possible, running prefill is prioritized.
426
+ scheduler.block_manager.can_swap_in.return_value = True
427
+ _, out = schedule_and_update_computed_tokens(scheduler)
428
+ assert len(out.scheduled_seq_groups) == 1
429
+ # 3 decodes. It is swapped in.
430
+ assert out.num_batched_tokens == 30
431
+ assert out.blocks_to_swap_in == {}
432
+ assert out.blocks_to_swap_out == {}
433
+ assert not seq_group2.is_prefill()
434
+ assert out.scheduled_seq_groups[0].seq_group == seq_group2
435
+ append_new_token(seq_group2, 1)
436
+
437
+ # Decoding is prioritized.
438
+ _, out = schedule_and_update_computed_tokens(scheduler)
439
+ assert len(out.scheduled_seq_groups) == 1
440
+ # 3 decodes. It is swapped in.
441
+ assert out.num_batched_tokens == 1
442
+ assert out.blocks_to_swap_in == {}
443
+ assert out.blocks_to_swap_out == {}
444
+ assert not seq_group2.is_prefill()
445
+ assert out.scheduled_seq_groups[0].seq_group == seq_group2
446
+ append_new_token(seq_group2, 1)
447
+
448
+ # Since we abort the sequence group, we can finally swap.
449
+ scheduler.abort_seq_group(seq_group2.request_id)
450
+ _, out = schedule_and_update_computed_tokens(scheduler)
451
+ assert len(out.scheduled_seq_groups) == 1
452
+ assert out.num_batched_tokens == 30
453
+ assert out.blocks_to_swap_in != {}
454
+ assert out.blocks_to_swap_out == {}
455
+
456
+
457
+def test_chunked_prefill_preempt():
458
+ """Verify preempt works with chunked prefill requests"""
459
+ block_size = 4
460
+ max_seqs = 30
461
+ max_model_len = 200
462
+ max_num_batched_tokens = 30
463
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
464
+ max_seqs,
465
+ max_model_len,
466
+ enable_chunked_prefill=True)
467
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
468
+ cache_config.num_cpu_blocks = 8
469
+ cache_config.num_gpu_blocks = 8
470
+ scheduler = Scheduler(scheduler_config, cache_config, None)
471
+
472
+ _, seq_group = create_dummy_prompt("1", prompt_length=60)
473
+ scheduler.add_seq_group(seq_group)
474
+ _, out = schedule_and_update_computed_tokens(scheduler)
475
+ # The request is chunked.
476
+ # prefill scheduled now.
477
+ assert len(out.scheduled_seq_groups) == 1
478
+ assert out.num_prefill_groups == 1
479
+ assert seq_group.is_prefill()
480
+ assert out.num_batched_tokens == max_num_batched_tokens
481
+
482
+ # The request should be preempted.
483
+ scheduler.block_manager.can_append_slots = MagicMock()
484
+
485
+ def cannot_append_second_group(seq_group, num_lookahead_slots):
486
+ return seq_group.request_id != "1"
487
+
488
+ scheduler.block_manager.can_append_slots.side_effect = (
489
+ cannot_append_second_group)
490
+
491
+ # The running prefill is now preempted.
492
+ _, out = schedule_and_update_computed_tokens(scheduler)
493
+ assert len(out.scheduled_seq_groups) == 0
494
+ assert out.num_batched_tokens == 0
495
+ assert out.blocks_to_swap_out == {}
496
+ assert out.blocks_to_swap_in == {}
497
+
498
+ # Make sure we can reschedule preempted request.
499
+ _, out = schedule_and_update_computed_tokens(scheduler)
500
+ assert len(out.scheduled_seq_groups) == 1
501
+ assert out.num_prefill_groups == 1
502
+ assert seq_group.is_prefill()
503
+ assert out.num_batched_tokens == max_num_batched_tokens
504
+ assert seq_group.get_num_uncomputed_tokens() == 30
505
+
506
+ # We should be able to run prefill twice as it is chunked.
507
+ def cannot_append_second_group(seq_group, num_lookahead_slots):
508
+ return True
509
+
510
+ scheduler.block_manager.can_append_slots.side_effect = (
511
+ cannot_append_second_group)
512
+ _, out = schedule_and_update_computed_tokens(scheduler)
513
+ assert len(out.scheduled_seq_groups) == 1
514
+ assert out.num_prefill_groups == 1
515
+ assert not seq_group.is_prefill()
516
+ assert out.num_batched_tokens == max_num_batched_tokens
517
+
518
+
519
+def test_chunked_prefill_max_seqs():
520
+ block_size = 4
521
+ max_seqs = 2
522
+ max_model_len = 80
523
+ max_num_batched_tokens = 64
524
+ scheduler_config = SchedulerConfig(max_num_batched_tokens,
525
+ max_seqs,
526
+ max_model_len,
527
+ enable_chunked_prefill=True)
528
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
529
+ cache_config.num_cpu_blocks = 8
530
+ cache_config.num_gpu_blocks = 8
531
+ scheduler = Scheduler(scheduler_config, cache_config, None)
532
+ running = []
533
+
534
+ _, seq_group = create_dummy_prompt("1", prompt_length=65)
535
+ scheduler.add_seq_group(seq_group)
536
+ running.append(seq_group)
537
+ # The first prefill is chunked.
538
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
539
+ assert seq_group_meta[0].token_chunk_size == max_num_batched_tokens
540
+ assert len(get_sequence_groups(out)) == 1
541
+
542
+ # Add new requests.
543
+ for i in range(4):
544
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=65)
545
+ scheduler.add_seq_group(seq_group)
546
+ running.append(seq_group)
547
+
548
+ # Make sure only 2 requests are scheduled.
549
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
550
+ assert out.num_batched_tokens == max_num_batched_tokens
551
+ assert len(get_sequence_groups(out)) == 2
552
+ assert not running[0].is_prefill()
553
+ assert running[1].is_prefill()
554
+ append_new_token(running[0], 1)
555
+
556
+ # Although we have enough token budget, we can only schedule max_seqs.
557
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
558
+ assert seq_group_meta[0].token_chunk_size == 2
559
+ assert seq_group_meta[1].token_chunk_size == 1
560
+ assert out.num_batched_tokens == 3
561
+ assert len(get_sequence_groups(out)) == max_seqs
562
+ assert not running[0].is_prefill()
563
+ assert not running[1].is_prefill()
tests/core/test_scheduler.py
+201
-65
@@ -10,7 +10,7 @@ from vllm.core.interfaces import AllocStatus
10
from vllm.core.policy import PolicyFactory
11
from vllm.core.scheduler import Scheduler, SchedulingBudget
12
from vllm.lora.request import LoRARequest
13
-from vllm.sequence import Logprob, SequenceGroup
13
+from vllm.sequence import Logprob, SequenceGroup, SequenceStatus
14
15
from .utils import create_dummy_prompt
16
@@ -19,6 +19,26 @@ def get_sequence_groups(scheduler_output):
19
return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
20
21
22
+def append_new_token(out, token_id: int):
23
+ seq_groups = get_sequence_groups(out)
24
+ for seq_group in seq_groups:
25
+ for seq in seq_group.get_seqs():
26
+ seq.append_token_id(token_id, {token_id: Logprob(token_id)})
27
+
28
+
29
+def schedule_and_update_computed_tokens(scheduler):
30
+ metas, out = scheduler.schedule()
31
+ for s, meta in zip(out.scheduled_seq_groups, metas):
32
+ s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
33
+ return metas, out
34
+
35
+
36
+def append_new_token_seq_group(token_chunk_size, seq_group, token_id: int):
37
+ seq_group.update_num_computed_tokens(token_chunk_size)
38
+ for seq in seq_group.get_seqs():
39
+ seq.append_token_id(token_id, {token_id: Logprob(token_id)})
40
+
41
+
42
def test_scheduler_add_seq_group():
43
block_size = 4
44
scheduler_config = SchedulerConfig(100, 64, 1)
@@ -76,20 +96,52 @@ def test_scheduler_schedule_simple():
96
97
# Schedule seq groups prompts.
98
num_tokens = block_size * num_seq_group
79
- seq_group_meta, out = scheduler.schedule()
99
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
100
assert set(get_sequence_groups(out)) == set(running)
101
assert out.num_batched_tokens == num_tokens
102
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
103
and not out.blocks_to_swap_out)
104
assert len(seq_group_meta) == num_seq_group
105
+ append_new_token(out, 1)
106
107
# Schedule seq groups generation.
87
- seq_group_meta, out = scheduler.schedule()
108
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
109
assert set(get_sequence_groups(out)) == set(running)
110
assert out.num_batched_tokens == num_seq_group
111
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
112
and not out.blocks_to_swap_out)
113
assert len(seq_group_meta) == num_seq_group
114
+ append_new_token(out, 1)
115
+
116
+
117
+def test_scheduler_prefill_prioritized():
118
+ """Verify running batched tokens are not applied to prefill requests."""
119
+ block_size = 4
120
+ max_model_len = 30
121
+ max_batched_num_tokens = 30
122
+ scheduler_config = SchedulerConfig(max_batched_num_tokens, 2,
123
+ max_model_len)
124
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
125
+ cache_config.num_cpu_blocks = 2
126
+ cache_config.num_gpu_blocks = 2
127
+ scheduler = Scheduler(scheduler_config, cache_config, None)
128
+
129
+ # Add seq groups to scheduler.
130
+ _, seq_group_a = create_dummy_prompt("1", 1)
131
+ scheduler.add_seq_group(seq_group_a)
132
+
133
+ # Schedule seq groups prompts.
134
+ _, out = schedule_and_update_computed_tokens(scheduler)
135
+ assert get_sequence_groups(out) == [seq_group_a]
136
+
137
+ # Add a new prefill request B.
138
+ _, seq_group_b = create_dummy_prompt("2", 30)
139
+ scheduler.add_seq_group(seq_group_b)
140
+
141
+ # Verify prefill requests are prioritized. Since max_batched_num_tokens
142
+ # is 1, new prefill request has to be scheduled first.
143
+ _, out = schedule_and_update_computed_tokens(scheduler)
144
+ assert get_sequence_groups(out) == [seq_group_b]
145
146
147
def test_scheduler_schedule_preempt_abort():
@@ -108,7 +160,7 @@ def test_scheduler_schedule_preempt_abort():
160
scheduler.add_seq_group(seq_group_b)
161
162
# Schedule seq groups prompts.
111
- seq_group_meta, out = scheduler.schedule()
163
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
164
assert get_sequence_groups(out) == [seq_group_a, seq_group_b]
165
assert out.num_batched_tokens == block_size * 2 # seq_a and seq_b
166
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
@@ -118,12 +170,10 @@ def test_scheduler_schedule_preempt_abort():
170
171
# Append "generated" tokens, allowing the sequence to mark prompt tokens as
172
# processed.
121
- token_id = 0
122
- seq_a.append_token_id(token_id, {token_id: Logprob(0.0)})
123
- seq_b.append_token_id(token_id, {token_id: Logprob(0.0)})
173
+ append_new_token(out, 1)
174
175
# Schedule seq groups generation and preempt seq group b.
126
- seq_group_meta, out = scheduler.schedule()
176
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
177
assert get_sequence_groups(out) == [seq_group_a]
178
assert out.num_batched_tokens == 1
179
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
@@ -133,7 +183,7 @@ def test_scheduler_schedule_preempt_abort():
183
184
# Abort seq group a. Re-schedule seq group b prompt with recomputation.
185
scheduler.abort_seq_group("1")
136
- seq_group_meta, out = scheduler.schedule()
186
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
187
assert get_sequence_groups(out) == [seq_group_b]
188
assert out.num_batched_tokens == 5 # 4 prompt + 1 generation.
189
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
@@ -163,12 +213,14 @@ def test_scheduler_max_seqs():
213
scheduler.add_seq_group(all_seq_groups[0])
214
215
# Schedule seq groups prompts.
166
- _, out = scheduler.schedule()
216
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
217
assert set(get_sequence_groups(out)) == set([all_seq_groups[0]])
218
+ append_new_token(out, 1)
219
220
# Schedule seq groups generation.
170
- _, out = scheduler.schedule()
221
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
222
assert set(get_sequence_groups(out)) == set([all_seq_groups[0]])
223
+ append_new_token(out, 1)
224
225
# Append 2 more seq group
226
scheduler.add_seq_group(all_seq_groups[1])
@@ -177,7 +229,7 @@ def test_scheduler_max_seqs():
229
# Schedule seq groups prompts.
230
# Only 1 seq group should be scheduled since max_seq_group is 2
231
# and one is prompting.
180
- _, out = scheduler.schedule()
232
+ _, out = schedule_and_update_computed_tokens(scheduler)
233
assert set(get_sequence_groups(out)) == set([all_seq_groups[1]])
234
235
@@ -190,27 +242,32 @@ def test_scheduler_delay_factor():
242
scheduler = Scheduler(scheduler_config, cache_config, None)
243
244
# schedule first prompt
193
- _, seq_group = create_dummy_prompt("0", prompt_length=block_size)
245
+ seq_group_meta, seq_group = create_dummy_prompt("0",
246
+ prompt_length=block_size)
247
scheduler.add_seq_group(seq_group)
195
- seq_group_meta, out = scheduler.schedule()
248
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
249
assert out.num_prefill_groups > 0
250
assert seq_group_meta[0].request_id == '0'
251
+ append_new_token(out, 1)
252
253
# wait for a second before scheduling next prompt
254
time.sleep(1)
201
- _, seq_group = create_dummy_prompt("1", prompt_length=block_size)
255
+ seq_group_meta, seq_group = create_dummy_prompt("1",
256
+ prompt_length=block_size)
257
scheduler.add_seq_group(seq_group)
258
259
# second prompt should *not* be scheduled
205
- seq_group_meta, out = scheduler.schedule()
260
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
261
assert out.num_prefill_groups == 0
262
assert seq_group_meta[0].request_id == '0'
263
+ append_new_token(out, 1)
264
265
# wait for more than 0.5 second and try again
266
time.sleep(0.6)
211
- seq_group_meta, out = scheduler.schedule()
267
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
268
assert out.num_prefill_groups > 0
269
assert seq_group_meta[0].request_id == '1'
270
+ append_new_token(out, 1)
271
272
273
def test_swapped_out_prioritized():
@@ -219,9 +276,10 @@ def test_swapped_out_prioritized():
276
for i in range(3):
277
_, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
278
scheduler.add_seq_group(seq_group)
222
- _, out = scheduler.schedule()
279
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
280
# prefill scheduled now.
281
assert len(out.scheduled_seq_groups) == 3
282
+ append_new_token(out, 1)
283
284
# The last request should be swapped out.
285
scheduler.block_manager.can_append_slots = MagicMock()
@@ -232,16 +290,18 @@ def test_swapped_out_prioritized():
290
scheduler.block_manager.can_append_slots.side_effect = (
291
cannot_append_second_group)
292
235
- _, out = scheduler.schedule()
293
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
294
assert len(out.scheduled_seq_groups) == 2
295
assert out.num_batched_tokens == 2
296
assert out.blocks_to_swap_out != {}
297
assert out.blocks_to_swap_in == {}
298
+ append_new_token(out, 1)
299
300
# Add 1 more task. Swap should be prioritized over prefill.
301
_, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
302
scheduler.add_seq_group(seq_group)
244
- _, out = scheduler.schedule()
303
+ seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
304
+ append_new_token(out, 1)
305
assert len(out.scheduled_seq_groups) == 3
306
# 3 decodes. It is swapped in.
307
assert out.num_batched_tokens == 3
@@ -264,18 +324,23 @@ def initialize_scheduler(*,
324
return scheduler
325
326
267
-def create_token_budget(num_batched_tokens: int = 0,
268
- num_curr_seqs: int = 0,
269
- token_budget: int = 10000,
327
+def create_token_budget(token_budget: int = 10000,
328
max_num_seqs: int = 10000) -> SchedulingBudget:
329
return SchedulingBudget(
272
- num_batched_tokens=num_batched_tokens,
273
- num_curr_seqs=num_curr_seqs,
330
token_budget=token_budget,
331
max_num_seqs=max_num_seqs,
332
)
333
334
335
+def add_token_budget(budget: SchedulingBudget,
336
+ num_batched_tokens: int = 0,
337
+ num_curr_seqs: int = 0):
338
+ mock_seq_group = create_dummy_prompt('10', prompt_length=60)[1]
339
+ budget.add_num_batched_tokens(mock_seq_group.request_id,
340
+ num_batched_tokens)
341
+ budget.add_num_seqs(mock_seq_group.request_id, num_curr_seqs)
342
+
343
+
344
def test_prefill_schedule_max_prompt_len():
345
"""
346
Test prompt longer than max_prompt_len is aborted.
@@ -326,7 +391,8 @@ def test_prefill_schedule_token_budget():
391
# Test when current_batched_tokens respected.
392
scheduler = initialize_scheduler()
393
waiting = deque()
329
- budget = create_token_budget(num_batched_tokens=30, token_budget=60)
394
+ budget = create_token_budget(token_budget=60)
395
+ add_token_budget(budget, 30, 0)
396
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
397
# Cannot schedule a prompt that doesn't fit the budget.
398
waiting.append(seq_group)
@@ -337,7 +403,8 @@ def test_prefill_schedule_token_budget():
403
assert budget.num_batched_tokens == 30
404
assert budget.num_curr_seqs == 0
405
assert len(remaining_waiting) == 1
340
- budget = create_token_budget(num_batched_tokens=30, token_budget=90)
406
+ budget = create_token_budget(token_budget=90)
407
+ add_token_budget(budget, 30, 0)
408
remaining_waiting, output = scheduler._schedule_prefills(
409
waiting, budget, None)
410
assert len(output.seq_groups) == 1
@@ -366,7 +433,8 @@ def test_prefill_schedule_max_seqs():
433
434
# Verify curr_num_seqs respected.
435
waiting = deque()
369
- budget = create_token_budget(num_curr_seqs=2, max_num_seqs=2)
436
+ budget = create_token_budget(max_num_seqs=2)
437
+ add_token_budget(budget, 0, 2)
438
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
439
waiting.append(seq_group)
440
remaining_waiting, output = scheduler._schedule_prefills(
@@ -472,7 +540,8 @@ def test_decode_schedule_preempted():
540
curr_loras = None
541
for i in range(3):
542
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
475
- scheduler._allocate_and_set_running(seq_group)
543
+ scheduler._allocate_and_set_running(seq_group, 60)
544
+ append_new_token_seq_group(60, seq_group, 1)
545
running.append(seq_group)
546
scheduler.block_manager.can_append_slots = MagicMock()
547
@@ -484,12 +553,13 @@ def test_decode_schedule_preempted():
553
554
# 1 cannot be scheduled, and the lowest priority (request 2)
555
# should be preempted. 1 will also be preempted.
487
- budget = create_token_budget(num_batched_tokens=3, num_curr_seqs=3)
488
- remainig_running, output = scheduler._schedule_decodes(
556
+ budget = create_token_budget()
557
+ remainig_running, output = scheduler._schedule_running(
558
running, budget, curr_loras, policy)
559
assert len(remainig_running) == 0
491
- assert len(output.seq_groups) == 1
492
- assert output.seq_groups[0].seq_group.request_id == "0"
560
+ assert len(output.decode_seq_groups) == 1
561
+ assert len(output.prefill_seq_groups) == 0
562
+ assert output.decode_seq_groups[0].seq_group.request_id == "0"
563
assert len(output.preempted) == 2
564
# Verify budgets are updated.
565
assert budget.num_batched_tokens == 1
@@ -508,10 +578,16 @@ def test_decode_swap_beam_search():
578
running = deque()
579
policy = PolicyFactory.get_policy(policy_name="fcfs")
580
curr_loras = None
581
+ budget = create_token_budget()
582
for i in range(3):
583
_, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
513
- scheduler._allocate_and_set_running(seq_group)
584
+ scheduler._allocate_and_set_running(seq_group, 60)
585
running.append(seq_group)
586
+ append_new_token_seq_group(60, seq_group, 1)
587
+ budget.add_num_seqs(seq_group.request_id,
588
+ seq_group.get_max_num_running_seqs())
589
+ budget.add_num_batched_tokens(
590
+ seq_group.request_id, seq_group.num_seqs(SequenceStatus.RUNNING))
591
592
# The last request should be swapped out.
593
scheduler.block_manager.can_append_slots = MagicMock()
@@ -525,19 +601,19 @@ def test_decode_swap_beam_search():
601
expected_swap_mapping = {"5": "7"}
602
scheduler.block_manager.swap_out.return_value = expected_swap_mapping
603
528
- budget = create_token_budget(num_batched_tokens=3, num_curr_seqs=3)
529
- remainig_running, output = scheduler._schedule_decodes(
604
+ remainig_running, output = scheduler._schedule_running(
605
running, budget, curr_loras, policy)
606
assert len(remainig_running) == 0
532
- assert len(output.seq_groups) == 2
533
- assert output.seq_groups[0].seq_group.request_id == "0"
534
- assert output.seq_groups[1].seq_group.request_id == "1"
607
+ assert len(output.decode_seq_groups) == 2
608
+ assert len(output.prefill_seq_groups) == 0
609
+ assert output.decode_seq_groups[0].seq_group.request_id == "0"
610
+ assert output.decode_seq_groups[1].seq_group.request_id == "1"
611
assert len(output.preempted) == 0
612
assert len(output.swapped_out) == 1
613
# Budget should refledct preempted requests.
614
assert budget.num_batched_tokens == 2
615
# since there are 2 sequences, 2 should be subtracted.
540
- assert budget.num_curr_seqs == 1
616
+ assert budget.num_curr_seqs == 4
617
# Both should be preempted, not swapped.
618
assert output.blocks_to_swap_out == expected_swap_mapping
619
# Nothing is copied.
@@ -553,7 +629,8 @@ def test_schedule_decode_blocks_to_copy_update():
629
running = deque()
630
policy = PolicyFactory.get_policy(policy_name="fcfs")
631
curr_loras = None
556
- scheduler._allocate_and_set_running(seq_group)
632
+ scheduler._allocate_and_set_running(seq_group, 60)
633
+ append_new_token_seq_group(60, seq_group, 1)
634
running.append(seq_group)
635
636
# The last request should be swapped out.
@@ -561,10 +638,11 @@ def test_schedule_decode_blocks_to_copy_update():
638
scheduler.block_manager.append_slots.return_value = {2: [3]}
639
640
budget = create_token_budget()
564
- remaining_running, output = scheduler._schedule_decodes(
641
+ remaining_running, output = scheduler._schedule_running(
642
running, budget, curr_loras, policy)
643
assert len(remaining_running) == 0
567
- assert len(output.seq_groups) == 1
644
+ assert len(output.decode_seq_groups) == 1
645
+ assert len(output.prefill_seq_groups) == 0
646
assert len(output.preempted) == 0
647
assert len(output.swapped_out) == 0
648
# Nothing is preempted.
@@ -581,7 +659,8 @@ def test_schedule_swapped_simple():
659
curr_loras = None
660
blocks_to_swap_out = {}
661
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
584
- scheduler._allocate_and_set_running(seq_group)
662
+ scheduler._allocate_and_set_running(seq_group, 60)
663
+ append_new_token_seq_group(60, seq_group, 1)
664
scheduler._swap_out(seq_group, blocks_to_swap_out)
665
swapped.append(seq_group)
666
@@ -591,7 +670,8 @@ def test_schedule_swapped_simple():
670
assert len(remaining_swapped) == 0
671
assert budget.num_batched_tokens == 1
672
assert budget.num_curr_seqs == 2
594
- assert len(output.seq_groups) == 1
673
+ assert len(output.decode_seq_groups) == 1
674
+ assert len(output.prefill_seq_groups) == 0
675
# swap in is the reverse of swap out
676
blocks_to_swap_in_reverse = {}
677
for swapin, swapout in output.blocks_to_swap_in.items():
@@ -607,7 +687,8 @@ def test_schedule_swapped_max_token_budget():
687
blocks_to_swap_out = {}
688
for _ in range(2):
689
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
610
- scheduler._allocate_and_set_running(seq_group)
690
+ scheduler._allocate_and_set_running(seq_group, 60)
691
+ append_new_token_seq_group(60, seq_group, 1)
692
scheduler._swap_out(seq_group, blocks_to_swap_out)
693
swapped.append(seq_group)
694
@@ -617,16 +698,19 @@ def test_schedule_swapped_max_token_budget():
698
assert len(remaining_swapped) == 1
699
assert budget.num_batched_tokens == 1
700
assert budget.num_curr_seqs == 2
620
- assert len(output.seq_groups) == 1
701
+ assert len(output.decode_seq_groups) == 1
702
+ assert len(output.prefill_seq_groups) == 0
703
704
# Verify num_batched_tokens are respected.
623
- budget = create_token_budget(num_batched_tokens=1, token_budget=1)
705
+ budget = create_token_budget(token_budget=1)
706
+ add_token_budget(budget, 1, 0)
707
remaining_swapped, output = scheduler._schedule_swapped(
708
remaining_swapped, budget, curr_loras, policy)
709
assert len(remaining_swapped) == 1
710
assert budget.num_batched_tokens == 1
711
assert budget.num_curr_seqs == 0
629
- assert len(output.seq_groups) == 0
712
+ assert len(output.decode_seq_groups) == 0
713
+ assert len(output.prefill_seq_groups) == 0
714
715
716
def test_schedule_swapped_max_seqs():
@@ -635,28 +719,30 @@ def test_schedule_swapped_max_seqs():
719
policy = PolicyFactory.get_policy(policy_name="fcfs")
720
curr_loras = None
721
blocks_to_swap_out = {}
638
- for _ in range(2):
639
- _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
640
- scheduler._allocate_and_set_running(seq_group)
722
+ for i in range(4):
723
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
724
+ scheduler._allocate_and_set_running(seq_group, 60)
725
+ append_new_token_seq_group(60, seq_group, 1)
726
scheduler._swap_out(seq_group, blocks_to_swap_out)
727
swapped.append(seq_group)
728
729
budget = create_token_budget(max_num_seqs=2)
730
remaining_swapped, output = scheduler._schedule_swapped(
731
swapped, budget, curr_loras, policy)
647
- assert len(remaining_swapped) == 1
648
- assert budget.num_batched_tokens == 1
732
+ assert len(remaining_swapped) == 2
733
+ assert budget.num_batched_tokens == 2
734
assert budget.num_curr_seqs == 2
650
- assert len(output.seq_groups) == 1
735
+ assert len(output.decode_seq_groups) == 2
736
+ assert len(output.prefill_seq_groups) == 0
737
738
# Verify num_curr_seqs are respected.
653
- budget = create_token_budget(num_curr_seqs=2, max_num_seqs=2)
739
remaining_swapped, output = scheduler._schedule_swapped(
740
remaining_swapped, budget, curr_loras, policy)
656
- assert len(remaining_swapped) == 1
657
- assert budget.num_batched_tokens == 0
741
+ assert len(remaining_swapped) == 2
742
+ assert budget.num_batched_tokens == 2
743
assert budget.num_curr_seqs == 2
659
- assert len(output.seq_groups) == 0
744
+ assert len(output.decode_seq_groups) == 0
745
+ assert len(output.prefill_seq_groups) == 0
746
747
748
def test_schedule_swapped_max_loras():
@@ -673,7 +759,8 @@ def test_schedule_swapped_max_loras():
759
lora_name=str(i),
760
lora_int_id=i + 1,
761
lora_local_path="abc"))
676
- scheduler._allocate_and_set_running(seq_group)
762
+ scheduler._allocate_and_set_running(seq_group, 60)
763
+ append_new_token_seq_group(60, seq_group, 1)
764
scheduler._swap_out(seq_group, blocks_to_swap_out)
765
swapped.append(seq_group)
766
@@ -683,7 +770,8 @@ def test_schedule_swapped_max_loras():
770
assert len(remaining_swapped) == 1
771
assert budget.num_batched_tokens == 1
772
assert budget.num_curr_seqs == 1
686
- assert len(output.seq_groups) == 1
773
+ assert len(output.decode_seq_groups) == 1
774
+ assert len(output.prefill_seq_groups) == 0
775
assert len(curr_loras) == 1
776
777
@@ -695,7 +783,8 @@ def test_schedule_swapped_cannot_swap_in():
783
blocks_to_swap_out = {}
784
for _ in range(2):
785
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
698
- scheduler._allocate_and_set_running(seq_group)
786
+ scheduler._allocate_and_set_running(seq_group, 60)
787
+ append_new_token_seq_group(60, seq_group, 1)
788
scheduler._swap_out(seq_group, blocks_to_swap_out)
789
swapped.append(seq_group)
790
@@ -709,7 +798,8 @@ def test_schedule_swapped_cannot_swap_in():
798
assert len(remaining_swapped) == 2
799
assert budget.num_batched_tokens == 0
800
assert budget.num_curr_seqs == 0
712
- assert len(output.seq_groups) == 0
801
+ assert len(output.decode_seq_groups) == 0
802
+ assert len(output.prefill_seq_groups) == 0
803
804
805
def test_schedule_swapped_blocks_to_copy():
@@ -718,7 +808,8 @@ def test_schedule_swapped_blocks_to_copy():
808
policy = PolicyFactory.get_policy(policy_name="fcfs")
809
curr_loras = None
810
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
721
- scheduler._allocate_and_set_running(seq_group)
811
+ scheduler._allocate_and_set_running(seq_group, 60)
812
+ append_new_token_seq_group(60, seq_group, 1)
813
blocks_to_swap_out = {}
814
scheduler._swap_out(seq_group, blocks_to_swap_out)
815
swapped.append(seq_group)
@@ -731,5 +822,50 @@ def test_schedule_swapped_blocks_to_copy():
822
remaining_swapped, output = scheduler._schedule_swapped(
823
swapped, budget, curr_loras, policy)
824
assert len(remaining_swapped) == 0
734
- assert len(output.seq_groups) == 1
825
+ assert len(output.decode_seq_groups) == 1
826
+ assert len(output.prefill_seq_groups) == 0
827
assert output.blocks_to_copy == {2: [3]}
828
+
829
+
830
+def test_scheduling_budget():
831
+ TOKEN_BUDGET = 4
832
+ MAX_SEQS = 4
833
+ budget = SchedulingBudget(token_budget=TOKEN_BUDGET, max_num_seqs=MAX_SEQS)
834
+ assert budget.can_schedule(num_new_tokens=1, num_new_seqs=1)
835
+ assert budget.can_schedule(num_new_tokens=4, num_new_seqs=4)
836
+ assert not budget.can_schedule(num_new_tokens=1, num_new_seqs=5)
837
+ assert not budget.can_schedule(num_new_tokens=5, num_new_seqs=1)
838
+ assert not budget.can_schedule(num_new_tokens=5, num_new_seqs=5)
839
+ assert budget.remaining_token_budget() == TOKEN_BUDGET
840
+
841
+ # Verify add/subtract num batched tokens.
842
+ _, seq_group = create_dummy_prompt("1", 3)
843
+ budget.add_num_batched_tokens(seq_group.request_id, 2)
844
+ assert budget.remaining_token_budget() == 2
845
+ assert budget.num_batched_tokens == 2
846
+ assert budget.can_schedule(num_new_tokens=2, num_new_seqs=1)
847
+ assert not budget.can_schedule(num_new_tokens=3, num_new_seqs=1)
848
+ # Verify adding another seq group is no-op.
849
+ budget.add_num_batched_tokens(seq_group.request_id, 2)
850
+ assert budget.remaining_token_budget() == 2
851
+ assert budget.num_batched_tokens == 2
852
+ budget.subtract_num_batched_tokens(seq_group.request_id, 2)
853
+ assert budget.remaining_token_budget() == 4
854
+ assert budget.num_batched_tokens == 0
855
+ budget.subtract_num_batched_tokens(seq_group.request_id, 2)
856
+ assert budget.remaining_token_budget() == 4
857
+ assert budget.num_batched_tokens == 0
858
+
859
+ # Verify add/subtract max seqs.
860
+ _, seq_group = create_dummy_prompt("1", 3)
861
+ budget.add_num_seqs(seq_group.request_id, 2)
862
+ assert budget.can_schedule(num_new_tokens=1, num_new_seqs=2)
863
+ assert not budget.can_schedule(num_new_tokens=1, num_new_seqs=3)
864
+ assert budget.num_curr_seqs == 2
865
+ # Verify adding another seq group is no-op.
866
+ budget.add_num_seqs(seq_group.request_id, 2)
867
+ assert budget.num_curr_seqs == 2
868
+ budget.subtract_num_seqs(seq_group.request_id, 2)
869
+ assert budget.num_curr_seqs == 0
870
+ budget.subtract_num_seqs(seq_group.request_id, 2)
871
+ assert budget.num_curr_seqs == 0
tests/test_sequence.py
+55
-3
@@ -1,7 +1,36 @@
1
+import time
2
+from typing import Optional
3
+
4
import pytest
5
3
-from vllm.sequence import (SamplerOutput, SequenceData, SequenceGroupOutput,
4
- SequenceOutput)
6
+from vllm import SamplingParams
7
+from vllm.lora.request import LoRARequest
8
+from vllm.sequence import (SamplerOutput, Sequence, SequenceData,
9
+ SequenceGroup, SequenceGroupOutput, SequenceOutput)
10
+
11
+
12
+def create_dummy_prompt(
13
+ request_id: str,
14
+ prompt_length: int,
15
+ block_size: Optional[int] = None,
16
+ lora_request: Optional[LoRARequest] = None,
17
+ use_beam_search: bool = False,
18
+ best_of: int = 1,
19
+) -> SequenceGroup:
20
+ if not block_size:
21
+ block_size = prompt_length
22
+
23
+ # Create dummy prompt sequence with tokens 0...block_size-1
24
+ # and prompt "0 ... block_size".
25
+ prompt_tokens = list(range(prompt_length))
26
+ prompt_str = " ".join([str(t) for t in prompt_tokens])
27
+ prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
28
+ seq_group = SequenceGroup(
29
+ request_id, [prompt],
30
+ SamplingParams(use_beam_search=use_beam_search, best_of=best_of),
31
+ time.time(), lora_request)
32
+
33
+ return seq_group
34
35
36
@pytest.fixture
@@ -67,6 +96,29 @@ def test_sequence_data_prefill():
96
97
# append tokens and reset, simulating recompute
98
seq_data.append_token_id(1, logprob=0.0)
70
- seq_data.reset_num_computed_tokens()
99
+ seq_data.reset_state_for_recompute()
100
assert seq_data.get_num_uncomputed_tokens() == 5
101
assert seq_data.get_num_computed_tokens() == 0
102
+
103
+
104
+def test_sequence_group_stage():
105
+ seq_group = create_dummy_prompt("1", 12)
106
+ assert seq_group.is_prefill() is True
107
+ seq_group.update_num_computed_tokens(6)
108
+ assert seq_group.is_prefill() is True
109
+ seq_group.update_num_computed_tokens(5)
110
+ assert seq_group.is_prefill() is True
111
+ seq_group.update_num_computed_tokens(1)
112
+ assert seq_group.is_prefill() is False
113
+ seqs = seq_group.get_seqs()
114
+ assert len(seqs) == 1
115
+ seqs[0].data.append_token_id(1, logprob=0.0)
116
+ for seq in seq_group.get_seqs():
117
+ seq.reset_state_for_recompute()
118
+ assert seq_group.is_prefill() is True
119
+ seq_group.update_num_computed_tokens(5)
120
+ assert seq_group.is_prefill() is True
121
+ seq_group.update_num_computed_tokens(7)
122
+ assert seq_group.is_prefill() is True
123
+ seq_group.update_num_computed_tokens(1)
124
+ assert seq_group.is_prefill() is False
vllm/config.py
+2
-1
@@ -576,7 +576,8 @@ class SchedulerConfig:
576
self._verify_args()
577
578
def _verify_args(self) -> None:
579
- if self.max_num_batched_tokens < self.max_model_len:
579
+ if (self.max_num_batched_tokens < self.max_model_len
580
+ and not self.chunked_prefill_enabled):
581
raise ValueError(
582
f"max_num_batched_tokens ({self.max_num_batched_tokens}) is "
583
f"smaller than max_model_len ({self.max_model_len}). "
vllm/core/policy.py
+1
-3
@@ -38,9 +38,7 @@ class FCFS(Policy):
38
39
class PolicyFactory:
40
41
- _POLICY_REGISTRY = {
42
- 'fcfs': FCFS,
43
- }
41
+ _POLICY_REGISTRY = {'fcfs': FCFS}
42
43
@classmethod
44
def get_policy(cls, policy_name: str, **kwargs) -> Policy:
vllm/core/scheduler.py
+345
-95
@@ -1,7 +1,7 @@
1
import enum
2
import time
3
from collections import deque
4
-from dataclasses import dataclass
4
+from dataclasses import dataclass, field
5
from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
6
7
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
@@ -31,16 +31,64 @@ class PreemptionMode(enum.Enum):
31
32
@dataclass
33
class SchedulingBudget:
34
- """The available slots for scheduling."""
35
- num_batched_tokens: int
36
- num_curr_seqs: int
34
+ """The available slots for scheduling.
35
+
36
+ TODO(sang): Right now, the budget is request_id-aware meaning it can ignore
37
+ budget update from the same request_id. It is because in normal scheduling
38
+ path, we update RUNNING num_seqs ahead of time, meaning it could be
39
+ updated more than once when scheduling RUNNING requests. Since this won't
40
+ happen if we only have chunked prefill scheduling, we can remove this
41
+ feature from the API when chunked prefill is enabled by default.
42
+ """
43
token_budget: int
44
max_num_seqs: int
45
+ _requeset_ids_num_batched_tokens: Set[int] = field(default_factory=set)
46
+ _requeset_ids_num_curr_seqs: Set[int] = field(default_factory=set)
47
+ _num_batched_tokens: int = 0
48
+ _num_curr_seqs: int = 0
49
50
def can_schedule(self, *, num_new_tokens: int, num_new_seqs: int):
51
+ assert num_new_tokens != 0
52
+ assert num_new_seqs != 0
53
return (self.num_batched_tokens + num_new_tokens <= self.token_budget
54
and self.num_curr_seqs + num_new_seqs <= self.max_num_seqs)
55
56
+ def remaining_token_budget(self):
57
+ return self.token_budget - self.num_batched_tokens
58
+
59
+ def add_num_batched_tokens(self, req_id: str, num_batched_tokens: int):
60
+ if req_id in self._requeset_ids_num_batched_tokens:
61
+ return
62
+
63
+ self._requeset_ids_num_batched_tokens.add(req_id)
64
+ self._num_batched_tokens += num_batched_tokens
65
+
66
+ def subtract_num_batched_tokens(self, req_id: str,
67
+ num_batched_tokens: int):
68
+ if req_id in self._requeset_ids_num_batched_tokens:
69
+ self._requeset_ids_num_batched_tokens.remove(req_id)
70
+ self._num_batched_tokens -= num_batched_tokens
71
+
72
+ def add_num_seqs(self, req_id: str, num_curr_seqs: int):
73
+ if req_id in self._requeset_ids_num_curr_seqs:
74
+ return
75
+
76
+ self._requeset_ids_num_curr_seqs.add(req_id)
77
+ self._num_curr_seqs += num_curr_seqs
78
+
79
+ def subtract_num_seqs(self, req_id: str, num_curr_seqs: int):
80
+ if req_id in self._requeset_ids_num_curr_seqs:
81
+ self._requeset_ids_num_curr_seqs.remove(req_id)
82
+ self._num_curr_seqs -= num_curr_seqs
83
+
84
+ @property
85
+ def num_batched_tokens(self):
86
+ return self._num_batched_tokens
87
+
88
+ @property
89
+ def num_curr_seqs(self):
90
+ return self._num_curr_seqs
91
+
92
93
@dataclass
94
class ScheduledSequenceGroup:
@@ -54,6 +102,7 @@ class ScheduledSequenceGroup:
102
103
@dataclass
104
class SchedulerOutputs:
105
+ """The scheduling decision made from a scheduler."""
106
# Scheduled sequence groups.
107
scheduled_seq_groups: Iterable[ScheduledSequenceGroup]
108
# Number of prefill groups scheduled.
@@ -95,10 +144,17 @@ class SchedulerOutputs:
144
145
146
@dataclass
98
-class SchedulerDecodeOutputs:
99
- """Outputs of the decoding phase of the scheduler."""
100
- # Selected sequence groups for decoding.
101
- seq_groups: List[SequenceGroup]
147
+class SchedulerRunningOutputs:
148
+ """The requests that are scheduled from a running queue.
149
+
150
+ Could contain prefill (prefill that's chunked) or decodes. If there's not
151
+ enough memory, it can be preempted (for recompute) or swapped out.
152
+ """
153
+ # Selected sequences that are running and in a decoding phase.
154
+ decode_seq_groups: List[SequenceGroup]
155
+ # Selected sequences that are running and in a prefill phase.
156
+ # I.e., it means the prefill has been chunked.
157
+ prefill_seq_groups: List[SequenceGroup]
158
# The preempted sequences.
159
preempted: List[SequenceGroup]
160
# Sequences that are swapped out.
@@ -107,12 +163,14 @@ class SchedulerDecodeOutputs:
163
blocks_to_swap_out: Dict[int, int]
164
# The blocks to copy.
165
blocks_to_copy: Dict[int, List[int]]
166
+ # The number of slots for lookahead decoding.
167
num_lookahead_slots: int
168
169
@classmethod
113
- def create_empty(cls) -> "SchedulerDecodeOutputs":
114
- return SchedulerDecodeOutputs(
115
- seq_groups=[],
170
+ def create_empty(cls) -> "SchedulerRunningOutputs":
171
+ return SchedulerRunningOutputs(
172
+ decode_seq_groups=[],
173
+ prefill_seq_groups=[],
174
preempted=[],
175
swapped_out=[],
176
blocks_to_swap_out={},
@@ -123,20 +181,28 @@ class SchedulerDecodeOutputs:
181
182
@dataclass
183
class SchedulerSwappedInOutputs:
126
- """Outputs of the decoding phase of the scheduler."""
127
- # Selected sequence groups for decoding.
128
- seq_groups: List[SequenceGroup]
184
+ """The requests that are scheduled from a swap queue.
185
+
186
+ Could contain prefill (prefill that's chunked) or decodes.
187
+ """
188
+ # Selected sequences that are going to be swapped in and is in a
189
+ # decoding phase.
190
+ decode_seq_groups: List[SequenceGroup]
191
+ # Selected sequences that are going to be swapped in and in a prefill
192
+ # phase. I.e., it means the prefill has been chunked.
193
+ prefill_seq_groups: List[SequenceGroup]
194
# The blocks to swap in.
195
blocks_to_swap_in: Dict[int, int]
196
# The blocks to copy.
197
blocks_to_copy: Dict[int, List[int]]
133
- # # The number of batched tokens.
198
+ # The number of slots for lookahead decoding.
199
num_lookahead_slots: int
200
201
@classmethod
202
def create_empty(cls) -> "SchedulerSwappedInOutputs":
203
return SchedulerSwappedInOutputs(
139
- seq_groups=[],
204
+ decode_seq_groups=[],
205
+ prefill_seq_groups=[],
206
blocks_to_swap_in={},
207
blocks_to_copy={},
208
num_lookahead_slots=0,
@@ -145,8 +211,12 @@ class SchedulerSwappedInOutputs:
211
212
@dataclass
213
class SchedulerPrefillOutputs:
148
- """Outputs of the prefill phase of the scheduler."""
149
- # Selected sequence groups for prefill.
214
+ """The requests that are scheduled from a waiting queue.
215
+
216
+ Could contain a fresh prefill requests or preempted requests that need
217
+ to be recomputed from scratch.
218
+ """
219
+ # Selected sequences for prefill.
220
seq_groups: List[SequenceGroup]
221
# Ignored sequence groups.
222
ignored_seq_groups: List[SequenceGroup]
@@ -176,12 +246,12 @@ class Scheduler:
246
# LoRAs. This should be improved in the future.
247
self.lora_config = lora_config
248
179
- # TODO(sang): Fix it after chunked prefill is enabled.
180
- self.prompt_limit = min(self.scheduler_config.max_model_len,
181
- self.scheduler_config.max_num_batched_tokens)
182
-
183
- # Instantiate the scheduling policy.
184
- self.policy = PolicyFactory.get_policy(policy_name="fcfs")
249
+ if self.scheduler_config.chunked_prefill_enabled:
250
+ self.prompt_limit = self.scheduler_config.max_model_len
251
+ else:
252
+ self.prompt_limit = min(
253
+ self.scheduler_config.max_model_len,
254
+ self.scheduler_config.max_num_batched_tokens)
255
256
BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
257
version="v2" if self.scheduler_config.
@@ -268,21 +338,17 @@ class Scheduler:
338
def get_num_unfinished_seq_groups(self) -> int:
339
return len(self.waiting) + len(self.running) + len(self.swapped)
340
271
- def _schedule_decodes(
341
+ def _schedule_running(
342
self,
343
running_queue: deque,
344
budget: SchedulingBudget,
345
curr_loras: Optional[Set[int]],
346
policy: Policy,
277
- ) -> Tuple[deque, SchedulerDecodeOutputs]:
278
- """Schedule sequence groups in a decoding stage.
279
-
280
- NOTE(sang): All the RUNNING num_batched_tokens, num_curr_seqs,
281
- and curr_loras should be already included in `budget` and `curr_loras`.
282
- The API doesn't ADD UP these values.
347
+ enable_chunking: bool = False,
348
+ ) -> Tuple[deque, SchedulerRunningOutputs]:
349
+ """Schedule sequence groups that are running.
350
284
- Note that `budget` and `curr_loras` are still subtracted/popped when
285
- any running requests are preempted from this API.
351
+ Running queue should include decode and chunked prefill requests.
352
353
Args:
354
running_queue: The queue that contains running requests (i.e.,
@@ -292,16 +358,21 @@ class Scheduler:
358
curr_loras: Currently batched lora request ids. The argument is
359
in-place updated when any decodes are preempted.
360
policy: The sorting policy to sort running_queue.
295
-
361
+ enable_chunking: If True, seq group can be chunked and only a
362
+ chunked number of tokens are scheduled if
363
+ `budget.num_batched_tokens` has not enough capacity to schedule
364
+ all tokens.
365
+
366
Returns:
367
A tuple of remaining running queue (should be always 0) after
298
- scheduling and SchedulerDecodeOutputs.
368
+ scheduling and SchedulerRunningOutputs.
369
"""
370
# Blocks that need to be swapped or copied before model execution.
371
blocks_to_swap_out: Dict[int, int] = {}
372
blocks_to_copy: Dict[int, List[int]] = {}
373
304
- seq_groups: List[ScheduledSequenceGroup] = []
374
+ decode_seq_groups: List[ScheduledSequenceGroup] = []
375
+ prefill_seq_groups: List[ScheduledSequenceGroup] = []
376
preempted: List[SequenceGroup] = []
377
swapped_out: List[SequenceGroup] = []
378
@@ -313,18 +384,21 @@ class Scheduler:
384
running_queue = policy.sort_by_priority(now, running_queue)
385
386
while running_queue:
316
- # NOTE: running
387
seq_group = running_queue[0]
318
- num_running_tokens = (
319
- seq_group.num_seqs(status=SequenceStatus.RUNNING) *
320
- self.num_decoding_tokens_per_seq)
388
+ num_running_tokens = self._get_num_new_tokens(
389
+ seq_group, SequenceStatus.RUNNING, enable_chunking, budget)
390
+
391
+ # We can have up to 1 running prefill at any given time in running
392
+ # queue, which means we can guarantee chunk size is at least 1.
393
+ assert num_running_tokens != 0
394
num_running_seqs = seq_group.get_max_num_running_seqs()
395
396
running_queue.popleft()
397
while not self._can_append_slots(seq_group):
325
- # Increase the budget as requests are preempted.
326
- budget.num_batched_tokens -= num_running_tokens
327
- budget.num_curr_seqs -= num_running_seqs
398
+ budget.subtract_num_batched_tokens(seq_group.request_id,
399
+ num_running_tokens)
400
+ budget.subtract_num_seqs(seq_group.request_id,
401
+ num_running_seqs)
402
if curr_loras is not None and seq_group.lora_int_id > 0:
403
curr_loras.pop(seq_group.lora_int_id)
404
@@ -350,14 +424,28 @@ class Scheduler:
424
else:
425
logger.debug(f"append slot for {seq_group}")
426
self._append_slots(seq_group, blocks_to_copy)
353
- seq_groups.append(
354
- ScheduledSequenceGroup(seq_group=seq_group,
355
- token_chunk_size=1))
427
+ is_prefill = seq_group.is_prefill()
428
+ if is_prefill:
429
+ prefill_seq_groups.append(
430
+ ScheduledSequenceGroup(
431
+ seq_group=seq_group,
432
+ token_chunk_size=num_running_tokens))
433
+ else:
434
+ decode_seq_groups.append(
435
+ ScheduledSequenceGroup(seq_group=seq_group,
436
+ token_chunk_size=1))
437
+ budget.add_num_batched_tokens(seq_group.request_id,
438
+ num_running_tokens)
439
+ budget.add_num_seqs(seq_group.request_id, num_running_seqs)
440
+ if curr_loras is not None and seq_group.lora_int_id > 0:
441
+ curr_loras.add(seq_group.lora_int_id)
442
+
443
# Make sure all queues are updated.
444
assert len(running_queue) == 0
445
359
- return running_queue, SchedulerDecodeOutputs(
360
- seq_groups=seq_groups,
446
+ return running_queue, SchedulerRunningOutputs(
447
+ decode_seq_groups=decode_seq_groups,
448
+ prefill_seq_groups=prefill_seq_groups,
449
preempted=preempted,
450
swapped_out=swapped_out,
451
blocks_to_swap_out=blocks_to_swap_out,
@@ -371,6 +459,7 @@ class Scheduler:
459
budget: SchedulingBudget,
460
curr_loras: Optional[Set[int]],
461
policy: Policy,
462
+ enable_chunking: bool = False,
463
) -> Tuple[deque, SchedulerSwappedInOutputs]:
464
"""Schedule sequence groups that are swapped out.
465
@@ -386,7 +475,11 @@ class Scheduler:
475
curr_loras: Currently batched lora request ids. The argument is
476
in-place updated when any requests are swapped in.
477
policy: The sorting policy to sort swapped_queue.
389
-
478
+ enable_chunking: If True, seq group can be chunked and only a
479
+ chunked number of tokens are scheduled if
480
+ `budget.num_batched_tokens` has not enough capacity to schedule
481
+ all tokens.
482
+
483
Returns:
484
A tuple of remaining swapped_queue after scheduling and
485
SchedulerSwappedInOutputs.
@@ -394,7 +487,8 @@ class Scheduler:
487
# Blocks that need to be swapped or copied before model execution.
488
blocks_to_swap_in: Dict[int, int] = {}
489
blocks_to_copy: Dict[int, List[int]] = {}
397
- seq_groups: List[ScheduledSequenceGroup] = []
490
+ decode_seq_groups: List[ScheduledSequenceGroup] = []
491
+ prefill_seq_groups: List[ScheduledSequenceGroup] = []
492
now = time.time()
493
swapped_queue = policy.sort_by_priority(now, swapped_queue)
494
@@ -420,12 +514,13 @@ class Scheduler:
514
# The total number of sequences in the RUNNING state should not
515
# exceed the maximum number of sequences.
516
num_new_seqs = seq_group.get_max_num_running_seqs()
423
- num_new_tokens = (
424
- seq_group.num_seqs(status=SequenceStatus.SWAPPED) *
425
- self.num_decoding_tokens_per_seq)
517
+ num_new_tokens = self._get_num_new_tokens(seq_group,
518
+ SequenceStatus.SWAPPED,
519
+ enable_chunking, budget)
520
427
- if not budget.can_schedule(num_new_tokens=num_new_tokens,
428
- num_new_seqs=num_new_seqs):
521
+ if (num_new_tokens == 0
522
+ or not budget.can_schedule(num_new_tokens=num_new_tokens,
523
+ num_new_seqs=num_new_seqs)):
524
break
525
526
if lora_int_id > 0 and curr_loras is not None:
@@ -433,15 +528,23 @@ class Scheduler:
528
swapped_queue.popleft()
529
self._swap_in(seq_group, blocks_to_swap_in)
530
self._append_slots(seq_group, blocks_to_copy)
436
- seq_groups.append(
437
- ScheduledSequenceGroup(seq_group, token_chunk_size=1))
438
- budget.num_batched_tokens += num_new_tokens
439
- budget.num_curr_seqs += num_new_seqs
531
+ is_prefill = seq_group.is_prefill()
532
+ if is_prefill:
533
+ prefill_seq_groups.append(
534
+ ScheduledSequenceGroup(seq_group,
535
+ token_chunk_size=num_new_tokens))
536
+ else:
537
+ assert num_new_tokens == 1
538
+ decode_seq_groups.append(
539
+ ScheduledSequenceGroup(seq_group, token_chunk_size=1))
540
+ budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
541
+ budget.add_num_seqs(seq_group.request_id, num_new_seqs)
542
543
swapped_queue.extendleft(leftover_swapped)
544
545
return swapped_queue, SchedulerSwappedInOutputs(
444
- seq_groups=seq_groups,
546
+ decode_seq_groups=decode_seq_groups,
547
+ prefill_seq_groups=prefill_seq_groups,
548
blocks_to_swap_in=blocks_to_swap_in,
549
blocks_to_copy=blocks_to_copy,
550
num_lookahead_slots=self._get_num_lookahead_slots(
@@ -452,6 +555,7 @@ class Scheduler:
555
waiting_queue: deque,
556
budget: SchedulingBudget,
557
curr_loras: Optional[Set[int]],
558
+ enable_chunking: bool = False,
559
) -> Tuple[deque, SchedulerPrefillOutputs]:
560
"""Schedule sequence groups that are in prefill stage.
561
@@ -470,6 +574,10 @@ class Scheduler:
574
when any requests are scheduled.
575
curr_loras: Currently batched lora request ids. The argument is
576
in-place updated when any requests are scheduled.
577
+ enable_chunking: If True, seq group can be chunked and only a
578
+ chunked number of tokens are scheduled if
579
+ `budget.num_batched_tokens` has not enough capacity to schedule
580
+ all tokens.
581
582
Returns:
583
A tuple of remaining waiting_queue after scheduling and
@@ -489,11 +597,16 @@ class Scheduler:
597
assert len(waiting_seqs) == 1, (
598
"Waiting sequence group should have only one prompt "
599
"sequence.")
492
-
493
- num_prompt_tokens = waiting_seqs[0].get_len()
494
- if num_prompt_tokens > self.prompt_limit:
600
+ num_new_tokens = self._get_num_new_tokens(seq_group,
601
+ SequenceStatus.WAITING,
602
+ enable_chunking, budget)
603
+ if not enable_chunking:
604
+ num_prompt_tokens = waiting_seqs[0].get_len()
605
+ assert num_new_tokens == num_prompt_tokens
606
+
607
+ if num_new_tokens > self.prompt_limit:
608
logger.warning(
496
- f"Input prompt ({num_prompt_tokens} tokens) is too long"
609
+ f"Input prompt ({num_new_tokens} tokens) is too long"
610
f" and exceeds limit of {self.prompt_limit}")
611
for seq in waiting_seqs:
612
seq.status = SequenceStatus.FINISHED_IGNORED
@@ -507,7 +620,7 @@ class Scheduler:
620
break
621
elif can_allocate == AllocStatus.NEVER:
622
logger.warning(
510
- f"Input prompt ({num_prompt_tokens} tokens) is too long"
623
+ f"Input prompt ({num_new_tokens} tokens) is too long"
624
f" and exceeds the capacity of block_manager")
625
for seq in waiting_seqs:
626
seq.status = SequenceStatus.FINISHED_IGNORED
@@ -528,20 +641,21 @@ class Scheduler:
641
continue
642
643
num_new_seqs = seq_group.get_max_num_running_seqs()
531
- if not budget.can_schedule(num_new_tokens=num_prompt_tokens,
532
- num_new_seqs=num_new_seqs):
644
+ if (num_new_tokens == 0
645
+ or not budget.can_schedule(num_new_tokens=num_new_tokens,
646
+ num_new_seqs=num_new_seqs)):
647
break
648
649
# Can schedule this request.
650
if curr_loras is not None and lora_int_id > 0:
651
curr_loras.add(lora_int_id)
652
waiting_queue.popleft()
539
- self._allocate_and_set_running(seq_group)
653
+ self._allocate_and_set_running(seq_group, num_new_tokens)
654
seq_groups.append(
655
ScheduledSequenceGroup(seq_group=seq_group,
542
- token_chunk_size=num_prompt_tokens))
543
- budget.num_batched_tokens += num_prompt_tokens
544
- budget.num_curr_seqs += num_new_seqs
656
+ token_chunk_size=num_new_tokens))
657
+ budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
658
+ budget.add_num_seqs(seq_group.request_id, num_new_seqs)
659
660
# Queue requests that couldn't be scheduled.
661
waiting_queue.extendleft(leftover_waiting_sequences)
@@ -553,8 +667,8 @@ class Scheduler:
667
ignored_seq_groups=ignored_seq_groups,
668
num_lookahead_slots=self._get_num_lookahead_slots(is_prefill=True))
669
556
- def _schedule(self) -> SchedulerOutputs:
557
- """Batch requests that are queued..
670
+ def _schedule_default(self) -> SchedulerOutputs:
671
+ """Schedule queued requests.
672
673
The current policy is designed to opimimize the throughput. First,
674
it batches as many prefill requests as possible. And it schedules
@@ -563,39 +677,48 @@ class Scheduler:
677
"""
678
# Include running requests to the budget.
679
budget = SchedulingBudget(
566
- num_batched_tokens=sum(
567
- seq_group.num_seqs(status=SequenceStatus.RUNNING)
568
- for seq_group in self.running),
569
- num_curr_seqs=sum(seq_group.get_max_num_running_seqs()
570
- for seq_group in self.running),
680
token_budget=self.scheduler_config.max_num_batched_tokens,
681
max_num_seqs=self.scheduler_config.max_num_seqs,
682
)
683
+ # Make sure we include num running seqs before scheduling prefill,
684
+ # so that we don't schedule beyond max_num_seqs for prefill.
685
+ for seq_group in self.running:
686
+ budget.add_num_seqs(seq_group.request_id,
687
+ seq_group.get_max_num_running_seqs())
688
curr_loras = set(
689
seq_group.lora_int_id
690
for seq_group in self.running) if self.lora_enabled else None
691
692
remaining_waiting, prefills = (self.waiting,
693
SchedulerPrefillOutputs.create_empty())
580
- remaining_running, decodes = (self.running,
581
- SchedulerDecodeOutputs.create_empty())
694
+ remaining_running, running_scheduled = (
695
+ self.running, SchedulerRunningOutputs.create_empty())
696
remaining_swapped, swapped_in = (
697
self.swapped, SchedulerSwappedInOutputs.create_empty())
698
699
# If any requests are swapped, prioritized swapped requests.
700
if not self.swapped:
701
remaining_waiting, prefills = self._schedule_prefills(
588
- self.waiting, budget, curr_loras)
702
+ self.waiting, budget, curr_loras, enable_chunking=False)
703
704
+ fcfs_policy = PolicyFactory.get_policy(policy_name="fcfs")
705
# Don't schedule decodes if prefills are scheduled.
706
+ # NOTE: If `_schedule_prefills` doesn't enable chunking, self.running
707
+ # only contains decode requests, not chunked prefills.
708
if len(prefills.seq_groups) == 0:
592
- remaining_running, decodes = self._schedule_decodes(
593
- self.running, budget, curr_loras, self.policy)
709
+ remaining_running, running_scheduled = self._schedule_running(
710
+ self.running,
711
+ budget,
712
+ curr_loras,
713
+ fcfs_policy,
714
+ enable_chunking=False)
715
+
716
# If any sequence group is preempted, do not swap in any sequence
717
# group. because it means there's no slot for new running requests.
596
- if len(decodes.preempted) + len(decodes.swapped_out) == 0:
718
+ if len(running_scheduled.preempted) + len(
719
+ running_scheduled.swapped_out) == 0:
720
remaining_swapped, swapped_in = self._schedule_swapped(
598
- self.swapped, budget, curr_loras, self.policy)
721
+ self.swapped, budget, curr_loras, fcfs_policy)
722
723
assert (budget.num_batched_tokens <=
724
self.scheduler_config.max_num_batched_tokens)
@@ -603,31 +726,134 @@ class Scheduler:
726
727
# Update waiting requests.
728
self.waiting = remaining_waiting
606
- self.waiting.extendleft(decodes.preempted)
729
+ self.waiting.extendleft(running_scheduled.preempted)
730
# Update new running requests.
731
self.running = remaining_running
732
self.running.extend([s.seq_group for s in prefills.seq_groups])
610
- self.running.extend([s.seq_group for s in decodes.seq_groups])
611
- self.running.extend([s.seq_group for s in swapped_in.seq_groups])
733
+ self.running.extend(
734
+ [s.seq_group for s in running_scheduled.decode_seq_groups])
735
+ self.running.extend(
736
+ [s.seq_group for s in swapped_in.decode_seq_groups])
737
# Update swapped requests.
738
self.swapped = remaining_swapped
614
- self.swapped.extend(decodes.swapped_out)
739
+ self.swapped.extend(running_scheduled.swapped_out)
740
741
+ # There should be no prefill from running queue because this policy
742
+ # doesn't allow chunked prefills.
743
+ assert len(running_scheduled.prefill_seq_groups) == 0
744
+ assert len(swapped_in.prefill_seq_groups) == 0
745
return SchedulerOutputs(
617
- scheduled_seq_groups=prefills.seq_groups + decodes.seq_groups +
618
- swapped_in.seq_groups,
746
+ scheduled_seq_groups=(prefills.seq_groups +
747
+ running_scheduled.decode_seq_groups +
748
+ swapped_in.decode_seq_groups),
749
num_prefill_groups=len(prefills.seq_groups),
750
num_batched_tokens=budget.num_batched_tokens,
751
blocks_to_swap_in=swapped_in.blocks_to_swap_in,
622
- blocks_to_swap_out=decodes.blocks_to_swap_out,
623
- blocks_to_copy=merge_dicts(decodes.blocks_to_copy,
752
+ blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
753
+ blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
754
+ swapped_in.blocks_to_copy),
755
+ ignored_seq_groups=prefills.ignored_seq_groups,
756
+ num_lookahead_slots=(prefills.num_lookahead_slots +
757
+ running_scheduled.num_lookahead_slots +
758
+ swapped_in.num_lookahead_slots),
759
+ )
760
+
761
+ def _schedule_chunked_prefill(self):
762
+ """Schedule queued requests.
763
+
764
+ Chunked prefill allows to chunk prefill requests, batch them together
765
+ with decode requests. This policy 1. schedule as many decoding requests
766
+ as possible. 2. schedule chunked prefill requests that are not
767
+ finished. 3. schedule swapped request. 4. schedule new prefill
768
+ requests.
769
+
770
+ The policy can sustain the high GPU utilization because it can put
771
+ prefill and decodes requests to the same batch, while it improves
772
+ inter token latency because decodes requests don't need to blocked
773
+ by prefill requests.
774
+ """
775
+ budget = SchedulingBudget(
776
+ token_budget=self.scheduler_config.max_num_batched_tokens,
777
+ max_num_seqs=self.scheduler_config.max_num_seqs,
778
+ )
779
+ curr_loras = set()
780
+
781
+ remaining_waiting, prefills = (self.waiting,
782
+ SchedulerPrefillOutputs.create_empty())
783
+ remaining_running, running_scheduled = (
784
+ self.running, SchedulerRunningOutputs.create_empty())
785
+ remaining_swapped, swapped_in = (
786
+ self.swapped, SchedulerSwappedInOutputs.create_empty())
787
+
788
+ # Decoding should be always scheduled first by fcfs.
789
+ fcfs_policy = PolicyFactory.get_policy(policy_name="fcfs")
790
+ remaining_running, running_scheduled = self._schedule_running(
791
+ self.running,
792
+ budget,
793
+ curr_loras,
794
+ fcfs_policy,
795
+ enable_chunking=True)
796
+
797
+ # Schedule swapped out requests.
798
+ # If preemption happens, it means we don't have space for swap-in.
799
+ if len(running_scheduled.preempted) + len(
800
+ running_scheduled.swapped_out) == 0:
801
+ remaining_swapped, swapped_in = self._schedule_swapped(
802
+ self.swapped, budget, curr_loras, fcfs_policy)
803
+
804
+ # Schedule new prefills.
805
+ remaining_waiting, prefills = self._schedule_prefills(
806
+ self.waiting, budget, curr_loras, enable_chunking=True)
807
+
808
+ assert (budget.num_batched_tokens <=
809
+ self.scheduler_config.max_num_batched_tokens)
810
+ assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
811
+
812
+ # Update waiting requests.
813
+ self.waiting = remaining_waiting
814
+ self.waiting.extendleft(running_scheduled.preempted)
815
+ # Update new running requests.
816
+ self.running = remaining_running
817
+ self.running.extend([s.seq_group for s in prefills.seq_groups])
818
+ self.running.extend(
819
+ [s.seq_group for s in running_scheduled.decode_seq_groups])
820
+ self.running.extend(
821
+ [s.seq_group for s in running_scheduled.prefill_seq_groups])
822
+ self.running.extend(
823
+ [s.seq_group for s in swapped_in.decode_seq_groups])
824
+ self.running.extend(
825
+ [s.seq_group for s in swapped_in.prefill_seq_groups])
826
+ # Update swapped requests.
827
+ self.swapped = remaining_swapped
828
+ self.swapped.extend(running_scheduled.swapped_out)
829
+
830
+ return SchedulerOutputs(
831
+ scheduled_seq_groups=(prefills.seq_groups +
832
+ running_scheduled.decode_seq_groups +
833
+ running_scheduled.prefill_seq_groups +
834
+ swapped_in.decode_seq_groups +
835
+ swapped_in.prefill_seq_groups),
836
+ num_prefill_groups=(len(prefills.seq_groups) +
837
+ len(swapped_in.prefill_seq_groups) +
838
+ len(running_scheduled.prefill_seq_groups)),
839
+ num_batched_tokens=budget.num_batched_tokens,
840
+ blocks_to_swap_in=swapped_in.blocks_to_swap_in,
841
+ blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
842
+ blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
843
swapped_in.blocks_to_copy),
844
ignored_seq_groups=prefills.ignored_seq_groups,
845
num_lookahead_slots=(prefills.num_lookahead_slots +
627
- decodes.num_lookahead_slots +
846
+ running_scheduled.num_lookahead_slots +
847
swapped_in.num_lookahead_slots),
848
)
849
850
+ def _schedule(self) -> SchedulerOutputs:
851
+ """Schedule queued requests."""
852
+ if self.scheduler_config.chunked_prefill_enabled:
853
+ return self._schedule_chunked_prefill()
854
+ else:
855
+ return self._schedule_default()
856
+
857
def _can_append_slots(self, seq_group: SequenceGroup) -> bool:
858
"""Determine whether or not we have enough space in the KV cache to
859
continue generation of the sequence group.
@@ -722,7 +948,8 @@ class Scheduler:
948
self.running = deque(seq_group for seq_group in self.running
949
if not seq_group.is_finished())
950
725
- def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
951
+ def _allocate_and_set_running(self, seq_group: SequenceGroup,
952
+ num_new_tokens: int) -> None:
953
self.block_manager.allocate(seq_group)
954
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
955
seq.status = SequenceStatus.RUNNING
@@ -854,3 +1081,26 @@ class Scheduler:
1081
return 0
1082
1083
return self.scheduler_config.num_lookahead_slots
1084
+
1085
+ def _get_num_new_tokens(self, seq_group: SequenceGroup,
1086
+ status: SequenceStatus, enable_chunking: bool,
1087
+ budget: SchedulingBudget) -> Tuple[int, bool]:
1088
+ """Get the next new tokens to compute for a given sequence group
1089
+ that's in a given `status`.
1090
+
1091
+ The API could chunk the number of tokens to compute based on `budget`
1092
+ if `enable_chunking` is True. If a sequence group has multiple
1093
+ sequences (e.g., running beam search), it means it is in decoding
1094
+ phase, so chunking doesn't happen.
1095
+ """
1096
+ num_new_tokens = 0
1097
+ seqs = seq_group.get_seqs(status=status)
1098
+ for seq in seqs:
1099
+ num_new_tokens += seq.get_num_new_tokens()
1100
+ # Chunk if a running request cannot fit in.
1101
+ # If number of seq > 1, it means it is doing beam search in a
1102
+ # decode phase. Do not chunk in that case.
1103
+ if enable_chunking and len(seqs) == 1:
1104
+ num_new_tokens = min(num_new_tokens,
1105
+ budget.remaining_token_budget())
1106
+ return num_new_tokens
vllm/engine/llm_engine.py
+2
-3
@@ -607,11 +607,10 @@ class LLMEngine:
607
now = time.time()
608
# Update the scheduled sequence groups with the model outputs.
609
scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
610
-
610
for scheduled_seq_group, outputs in zip(scheduled_seq_groups, output):
611
seq_group = scheduled_seq_group.seq_group
613
- token_chunk_size = scheduled_seq_group.token_chunk_size
614
- seq_group.update_num_computed_tokens(token_chunk_size)
612
+ seq_group.update_num_computed_tokens(
613
+ scheduled_seq_group.token_chunk_size)
614
self._process_sequence_group_outputs(seq_group, outputs)
615
616
# Free the finished sequence groups.
vllm/sequence.py
+48
-11
@@ -69,6 +69,11 @@ class SequenceStatus(enum.Enum):
69
return finish_reason
70
71
72
+class SequenceStage(enum.Enum):
73
+ PREFILL = enum.auto()
74
+ DECODE = enum.auto()
75
+
76
+
77
@dataclass
78
class RequestMetrics:
79
"""Metrics associated with a request.
@@ -115,6 +120,7 @@ class SequenceData:
120
self.cumulative_logprob = 0.0
121
# The number of tokens that are computed (that run against the model).
122
self._num_computed_tokens = 0
123
+ self._stage: SequenceStage = SequenceStage.PREFILL
124
125
def append_token_id(self, token_id: int, logprob: float) -> None:
126
self.output_token_ids.append(token_id)
@@ -136,16 +142,22 @@ class SequenceData:
142
"""Return the number of prefill tokens that are already computed."""
143
return self._num_computed_tokens
144
139
- def update_num_computed_tokens(self, num_new_computed_tokens: int) -> int:
145
+ def update_num_computed_tokens(self, num_new_computed_tokens: int):
146
"""Update number of tokens computed so far."""
147
self._num_computed_tokens += num_new_computed_tokens
148
+ assert self._num_computed_tokens <= self.get_len(), (
149
+ self._num_computed_tokens, self.get_len())
150
+ # If all tokens are computed, it means it is in decoding phase.
151
+ if self.get_num_uncomputed_tokens() == 0:
152
+ self._stage = SequenceStage.DECODE
153
143
- def reset_num_computed_tokens(self) -> None:
154
+ def reset_state_for_recompute(self) -> None:
155
"""Reset the number of computed tokens from this sequence. It is
156
supposed to be called when a sequence needs to be started from
157
the beginning again (e.g., sequence is preempted).
158
"""
159
self._num_computed_tokens = 0
160
+ self._stage = SequenceStage.PREFILL
161
162
def get_num_uncomputed_tokens(self) -> int:
163
"""Return the number of prefil tokens that are not computed."""
@@ -165,6 +177,10 @@ class SequenceData:
177
def get_output_token_ids(self) -> int:
178
return self.output_token_ids
179
180
+ @property
181
+ def stage(self) -> SequenceStage:
182
+ return self._stage
183
+
184
def __repr__(self) -> str:
185
return (f"SequenceData("
186
f"prompt_token_ids={self.prompt_token_ids}, "
@@ -234,7 +250,7 @@ class Sequence:
250
251
def reset_state_for_recompute(self):
252
"""Reset the sequence states for recomputation."""
237
- self.data.reset_num_computed_tokens()
253
+ self.data.reset_state_for_recompute()
254
255
def _append_logical_block(self) -> None:
256
block = LogicalTokenBlock(
@@ -320,6 +336,23 @@ class Sequence:
336
new_seq.seq_id = new_seq_id
337
return new_seq
338
339
+ def get_num_new_tokens(self) -> int:
340
+ """Get the number of new tokens to be computed.
341
+
342
+ Args:
343
+ remainig_token_budget: The remaining token budgets.
344
+ Returns:
345
+ The new number of tokens to be computed. I.e., 1 for decode, prompt
346
+ size for prefill. If there's not enough remainig_token_budget, it
347
+ can return the chunked number of new tokens.
348
+ """
349
+ if self.data.stage == SequenceStage.DECODE:
350
+ return 1
351
+ return self.data.get_num_uncomputed_tokens()
352
+
353
+ def is_prefill(self) -> bool:
354
+ return self.data.stage == SequenceStage.PREFILL
355
+
356
def __repr__(self) -> str:
357
return (f"Sequence(seq_id={self.seq_id}, "
358
f"status={self.status.name}, "
@@ -461,14 +494,14 @@ class SequenceGroup:
494
def update_num_computed_tokens(self, num_new_computed_tokens: int):
495
"""Update number of tokens computed so far."""
496
for seq in self.seqs_dict.values():
464
- seq.data.update_num_computed_tokens(num_new_computed_tokens)
497
+ if not seq.is_finished():
498
+ seq.data.update_num_computed_tokens(num_new_computed_tokens)
499
500
def get_num_uncomputed_tokens(self) -> int:
467
- # All sequences in the group should have the same prompt, so the
468
- # number of unfinished prefill tokens are the same across all
469
- # sequences.
470
- return list(
471
- self.seqs_dict.values())[0].data.get_num_uncomputed_tokens()
501
+ num_uncomputed_tokens = 0
502
+ for seq in self.get_seqs():
503
+ num_uncomputed_tokens += seq.data.get_num_uncomputed_tokens()
504
+ return num_uncomputed_tokens
505
506
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
507
return len(self.get_seqs(status))
@@ -497,6 +530,10 @@ class SequenceGroup:
530
def is_finished(self) -> bool:
531
return all(seq.is_finished() for seq in self.get_seqs())
532
533
+ def is_prefill(self) -> bool:
534
+ # Every sequences should be in the same stage.
535
+ return self.get_seqs()[0].is_prefill()
536
+
537
def __repr__(self) -> str:
538
return (f"SequenceGroup(request_id={self.request_id}, "
539
f"sampling_params={self.sampling_params}, "
@@ -513,8 +550,8 @@ class SequenceGroupMetadata:
550
sampling_params: The sampling parameters used to generate the outputs.
551
block_tables: The block tables. (Seq id -> list of physical block
552
numbers)
516
- token_chunk_size: The number of tokens to be processed. None if
517
- chunking is not required.
553
+ token_chunk_size: The number of tokens to be processed (per sequence).
554
+ None if chunking is not required.
555
state: Internal state tied to this sequence group.
556
lora_request: LoRA request.
557
multi_modal_data: Multi modal data.
vllm/worker/model_runner.py
-1
@@ -222,7 +222,6 @@ class ModelRunner:
222
# NOTE(woosuk): Here we assume that the first token in the prompt
223
# is always the first token in the sequence.
224
input_positions.extend(list(range(computed_len, prefill_end)))
225
-
225
lora_id = seq_group_metadata.lora_int_id
226
227
if lora_id > 0: