A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
Fix tests in test_scheduler.py that fail with BlockManager V2 (#8728)
sroy745 committed
Sep 23, 2024 at 21:43 UTC
88577ac92808cfd9468e4b54b757d5fcbe9aa486
1 file changed
+260
-89
tests/core/test_scheduler.py
+260
-89
@@ -3,7 +3,8 @@ from collections import deque
3
from typing import List, Set, Tuple
4
from unittest.mock import MagicMock
5
6
-import pytest # noqa
6
+import pytest
7
+from torch import Use # noqa
8
9
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
10
from vllm.core.interfaces import AllocStatus
@@ -16,9 +17,11 @@ from .utils import (append_new_token, append_new_token_seq_group,
17
schedule_and_update_computed_tokens)
18
19
19
-def test_scheduler_add_seq_group():
20
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
21
+def test_scheduler_add_seq_group(use_v2_block_manager: bool):
22
block_size = 4
21
- scheduler_config = SchedulerConfig(100, 64, 1)
23
+ scheduler_config = SchedulerConfig(
24
+ 100, 64, 1, use_v2_block_manager=use_v2_block_manager)
25
cache_config = CacheConfig(block_size, 1.0, 1, cache_dtype="auto")
26
cache_config.num_cpu_blocks = 4
27
cache_config.num_gpu_blocks = 4
@@ -27,14 +30,18 @@ def test_scheduler_add_seq_group():
30
# Add seq group to scheduler.
31
num_seq_group = 4
32
for i in range(num_seq_group):
30
- _, seq_group = create_dummy_prompt(str(i), block_size)
33
+ _, seq_group = create_dummy_prompt(str(i),
34
+ block_size,
35
+ block_size=block_size)
36
scheduler.add_seq_group(seq_group)
37
assert scheduler.get_num_unfinished_seq_groups() == i + 1
38
39
35
-def test_scheduler_abort_seq_group():
40
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
41
+def test_scheduler_abort_seq_group(use_v2_block_manager: bool):
42
block_size = 4
37
- scheduler_config = SchedulerConfig(100, 64, 1)
43
+ scheduler_config = SchedulerConfig(
44
+ 100, 64, 1, use_v2_block_manager=use_v2_block_manager)
45
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
46
cache_config.num_cpu_blocks = 4
47
cache_config.num_gpu_blocks = 4
@@ -54,11 +61,16 @@ def test_scheduler_abort_seq_group():
61
assert scheduler.get_num_unfinished_seq_groups() == 0
62
63
57
-def test_scheduler_schedule_simple():
64
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
65
+def test_scheduler_schedule_simple(use_v2_block_manager: bool):
66
block_size = 4
67
num_seq_group = 4
68
max_model_len = 16
61
- scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len)
69
+ scheduler_config = SchedulerConfig(
70
+ 64,
71
+ num_seq_group,
72
+ max_model_len,
73
+ use_v2_block_manager=use_v2_block_manager)
74
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
75
cache_config.num_cpu_blocks = 8
76
cache_config.num_gpu_blocks = 8
@@ -67,7 +79,9 @@ def test_scheduler_schedule_simple():
79
80
# Add seq groups to scheduler.
81
for i in range(num_seq_group):
70
- _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
82
+ _, seq_group = create_dummy_prompt(str(i),
83
+ prompt_length=block_size,
84
+ block_size=block_size)
85
scheduler.add_seq_group(seq_group)
86
running.append(seq_group)
87
@@ -91,20 +105,24 @@ def test_scheduler_schedule_simple():
105
append_new_token(out, 1)
106
107
94
-def test_scheduler_prefill_prioritized():
108
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
109
+def test_scheduler_prefill_prioritized(use_v2_block_manager: bool):
110
"""Verify running batched tokens are not applied to prefill requests."""
111
block_size = 4
112
max_model_len = 30
113
max_batched_num_tokens = 30
99
- scheduler_config = SchedulerConfig(max_batched_num_tokens, 2,
100
- max_model_len)
114
+ scheduler_config = SchedulerConfig(
115
+ max_batched_num_tokens,
116
+ 2,
117
+ max_model_len,
118
+ use_v2_block_manager=use_v2_block_manager)
119
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
102
- cache_config.num_cpu_blocks = 2
103
- cache_config.num_gpu_blocks = 2
120
+ cache_config.num_cpu_blocks = 16
121
+ cache_config.num_gpu_blocks = 16
122
scheduler = Scheduler(scheduler_config, cache_config, None)
123
124
# Add seq groups to scheduler.
107
- _, seq_group_a = create_dummy_prompt("1", 1)
125
+ _, seq_group_a = create_dummy_prompt("1", 1, block_size=block_size)
126
scheduler.add_seq_group(seq_group_a)
127
128
# Schedule seq groups prompts.
@@ -112,7 +130,7 @@ def test_scheduler_prefill_prioritized():
130
assert get_sequence_groups(out) == [seq_group_a]
131
132
# Add a new prefill request B.
115
- _, seq_group_b = create_dummy_prompt("2", 30)
133
+ _, seq_group_b = create_dummy_prompt("2", 30, block_size=block_size)
134
scheduler.add_seq_group(seq_group_b)
135
136
# Verify prefill requests are prioritized. Since max_batched_num_tokens
@@ -121,18 +139,24 @@ def test_scheduler_prefill_prioritized():
139
assert get_sequence_groups(out) == [seq_group_b]
140
141
124
-def test_scheduler_schedule_preempt_abort():
142
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
143
+def test_scheduler_schedule_preempt_abort(use_v2_block_manager: bool):
144
block_size = 4
145
max_model_len = 16
127
- scheduler_config = SchedulerConfig(64, 2, max_model_len)
146
+ scheduler_config = SchedulerConfig(
147
+ 64, 2, max_model_len, use_v2_block_manager=use_v2_block_manager)
148
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
149
cache_config.num_cpu_blocks = 2
150
cache_config.num_gpu_blocks = 2
151
scheduler = Scheduler(scheduler_config, cache_config, None)
152
153
# Add seq groups to scheduler.
134
- seq_a, seq_group_a = create_dummy_prompt("1", block_size)
135
- seq_b, seq_group_b = create_dummy_prompt("2", block_size)
154
+ seq_a, seq_group_a = create_dummy_prompt("1",
155
+ block_size,
156
+ block_size=block_size)
157
+ seq_b, seq_group_b = create_dummy_prompt("2",
158
+ block_size,
159
+ block_size=block_size)
160
scheduler.add_seq_group(seq_group_a)
161
scheduler.add_seq_group(seq_group_b)
162
@@ -170,12 +194,17 @@ def test_scheduler_schedule_preempt_abort():
194
assert scheduler.get_num_unfinished_seq_groups() == 1
195
196
173
-def test_scheduler_max_seqs():
197
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
198
+def test_scheduler_max_seqs(use_v2_block_manager: bool):
199
block_size = 4
200
num_seq_group = 4
201
max_seq_group = 2
202
max_model_len = 16
178
- scheduler_config = SchedulerConfig(64, max_seq_group, max_model_len)
203
+ scheduler_config = SchedulerConfig(
204
+ 64,
205
+ max_seq_group,
206
+ max_model_len,
207
+ use_v2_block_manager=use_v2_block_manager)
208
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
209
cache_config.num_cpu_blocks = 8
210
cache_config.num_gpu_blocks = 8
@@ -184,7 +213,9 @@ def test_scheduler_max_seqs():
213
all_seq_groups: List[SequenceGroup] = []
214
# Add seq groups to scheduler.
215
for i in range(num_seq_group):
187
- _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
216
+ _, seq_group = create_dummy_prompt(str(i),
217
+ prompt_length=block_size,
218
+ block_size=block_size)
219
all_seq_groups.append(seq_group)
220
221
# Append 1 seq group
@@ -211,9 +242,15 @@ def test_scheduler_max_seqs():
242
assert set(get_sequence_groups(out)) == set([all_seq_groups[1]])
243
244
214
-def test_scheduler_delay_factor():
245
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
246
+def test_scheduler_delay_factor(use_v2_block_manager: bool):
247
block_size = 4
216
- scheduler_config = SchedulerConfig(100, 64, 16, delay_factor=0.5)
248
+ scheduler_config = SchedulerConfig(
249
+ 100,
250
+ 64,
251
+ 16,
252
+ delay_factor=0.5,
253
+ use_v2_block_manager=use_v2_block_manager)
254
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
255
cache_config.num_cpu_blocks = 8
256
cache_config.num_gpu_blocks = 8
@@ -221,7 +258,8 @@ def test_scheduler_delay_factor():
258
259
# schedule first prompt
260
seq_group_meta, seq_group = create_dummy_prompt("0",
224
- prompt_length=block_size)
261
+ prompt_length=block_size,
262
+ block_size=block_size)
263
scheduler.add_seq_group(seq_group)
264
seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
265
assert out.num_prefill_groups > 0
@@ -231,7 +269,8 @@ def test_scheduler_delay_factor():
269
# wait for a second before scheduling next prompt
270
time.sleep(1)
271
seq_group_meta, seq_group = create_dummy_prompt("1",
234
- prompt_length=block_size)
272
+ prompt_length=block_size,
273
+ block_size=block_size)
274
scheduler.add_seq_group(seq_group)
275
276
# second prompt should *not* be scheduled
@@ -248,11 +287,20 @@ def test_scheduler_delay_factor():
287
append_new_token(out, 1)
288
289
251
-def test_swapped_out_prioritized():
252
- scheduler = initialize_scheduler(max_num_seqs=6)
290
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
291
+def test_swapped_out_prioritized(use_v2_block_manager: bool):
292
+ block_size = 4
293
+ scheduler = initialize_scheduler(max_num_seqs=6,
294
+ block_size=block_size,
295
+ use_v2_block_manager=use_v2_block_manager,
296
+ num_cpu_blocks=64,
297
+ num_gpu_blocks=64)
298
# best_of=2 * 3 == 6 sequences.
299
for i in range(3):
255
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
300
+ _, seq_group = create_dummy_prompt(str(i),
301
+ prompt_length=60,
302
+ best_of=2,
303
+ block_size=block_size)
304
scheduler.add_seq_group(seq_group)
305
seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
306
# prefill scheduled now.
@@ -276,7 +324,10 @@ def test_swapped_out_prioritized():
324
append_new_token(out, 1)
325
326
# Add 1 more task. Swap should be prioritized over prefill.
279
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
327
+ _, seq_group = create_dummy_prompt(str(i),
328
+ prompt_length=60,
329
+ best_of=2,
330
+ block_size=block_size)
331
scheduler.add_seq_group(seq_group)
332
seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
333
append_new_token(out, 1)
@@ -287,17 +338,26 @@ def test_swapped_out_prioritized():
338
assert out.blocks_to_swap_out == []
339
340
290
-def initialize_scheduler(*,
291
- max_num_seqs=1000,
292
- max_token_budget=1000,
293
- max_model_len=1000,
294
- lora_config=None):
295
- block_size = 4
296
- scheduler_config = SchedulerConfig(max_token_budget, max_num_seqs,
297
- max_model_len)
341
+def initialize_scheduler(
342
+ *,
343
+ max_num_seqs=1000,
344
+ max_token_budget=1000,
345
+ max_model_len=1000,
346
+ lora_config=None,
347
+ use_v2_block_manager=False,
348
+ block_size=4,
349
+ num_cpu_blocks=8,
350
+ num_gpu_blocks=8,
351
+):
352
+ block_size = block_size
353
+ scheduler_config = SchedulerConfig(
354
+ max_token_budget,
355
+ max_num_seqs,
356
+ max_model_len,
357
+ use_v2_block_manager=use_v2_block_manager)
358
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
299
- cache_config.num_cpu_blocks = 8
300
- cache_config.num_gpu_blocks = 8
359
+ cache_config.num_cpu_blocks = num_cpu_blocks
360
+ cache_config.num_gpu_blocks = num_gpu_blocks
361
scheduler = Scheduler(scheduler_config, cache_config, lora_config)
362
return scheduler
363
@@ -319,12 +379,18 @@ def add_token_budget(budget: SchedulingBudget,
379
budget.add_num_seqs(mock_seq_group.request_id, num_curr_seqs)
380
381
322
-def test_prefill_schedule_max_prompt_len():
382
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
383
+def test_prefill_schedule_max_prompt_len(use_v2_block_manager: bool):
384
"""
385
Test prompt longer than max_prompt_len is aborted.
386
"""
326
- scheduler = initialize_scheduler(max_model_len=30)
327
- _, seq_group = create_dummy_prompt("0", prompt_length=60)
387
+ block_size = 4
388
+ scheduler = initialize_scheduler(max_model_len=30,
389
+ use_v2_block_manager=use_v2_block_manager,
390
+ block_size=block_size)
391
+ _, seq_group = create_dummy_prompt("0",
392
+ prompt_length=60,
393
+ block_size=block_size)
394
scheduler.add_seq_group(seq_group)
395
budget = create_token_budget()
396
output = scheduler._schedule_prefills(budget, None)
@@ -336,14 +402,21 @@ def test_prefill_schedule_max_prompt_len():
402
assert len(remaining_waiting) == 0
403
404
339
-def test_prefill_schedule_token_budget():
405
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
406
+def test_prefill_schedule_token_budget(use_v2_block_manager: bool):
407
"""
408
Test token budget respected.
409
"""
343
- scheduler = initialize_scheduler()
410
+ block_size = 4
411
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
412
+ block_size=block_size,
413
+ num_cpu_blocks=64,
414
+ num_gpu_blocks=64)
415
budget = create_token_budget(token_budget=0)
416
for i in range(2):
346
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
417
+ _, seq_group = create_dummy_prompt(str(i),
418
+ prompt_length=60,
419
+ block_size=block_size)
420
scheduler.add_seq_group(seq_group)
421
422
# 0 token budget == nothing is scheduled.
@@ -366,10 +439,15 @@ def test_prefill_schedule_token_budget():
439
assert len(remaining_waiting) == 1
440
441
# Test when current_batched_tokens respected.
369
- scheduler = initialize_scheduler()
442
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
443
+ block_size=block_size,
444
+ num_cpu_blocks=16,
445
+ num_gpu_blocks=16)
446
budget = create_token_budget(token_budget=60)
447
add_token_budget(budget, 30, 0)
372
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
448
+ _, seq_group = create_dummy_prompt(str(i),
449
+ prompt_length=60,
450
+ block_size=block_size)
451
# Cannot schedule a prompt that doesn't fit the budget.
452
scheduler.add_seq_group(seq_group)
453
output = scheduler._schedule_prefills(budget, None)
@@ -389,14 +467,21 @@ def test_prefill_schedule_token_budget():
467
assert len(remaining_waiting) == 0
468
469
392
-def test_prefill_schedule_max_seqs():
470
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
471
+def test_prefill_schedule_max_seqs(use_v2_block_manager: bool):
472
"""
473
Test max seq respected.
474
"""
396
- scheduler = initialize_scheduler()
475
+ block_size = 4
476
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
477
+ block_size=block_size,
478
+ num_cpu_blocks=64,
479
+ num_gpu_blocks=64)
480
budget = create_token_budget(max_num_seqs=2)
481
for i in range(3):
399
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
482
+ _, seq_group = create_dummy_prompt(str(i),
483
+ prompt_length=60,
484
+ block_size=block_size)
485
scheduler.add_seq_group(seq_group)
486
output = scheduler._schedule_prefills(budget, None)
487
remaining_waiting = scheduler.waiting
@@ -410,7 +495,9 @@ def test_prefill_schedule_max_seqs():
495
scheduler.waiting = deque()
496
budget = create_token_budget(max_num_seqs=2)
497
add_token_budget(budget, 0, 2)
413
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
498
+ _, seq_group = create_dummy_prompt(str(i),
499
+ prompt_length=60,
500
+ block_size=block_size)
501
scheduler.add_seq_group(seq_group)
502
output = scheduler._schedule_prefills(budget, None)
503
remaining_waiting = scheduler.waiting
@@ -421,17 +508,24 @@ def test_prefill_schedule_max_seqs():
508
assert len(remaining_waiting) == 1
509
510
424
-def test_prefill_schedule_max_lora():
511
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
512
+def test_prefill_schedule_max_lora(use_v2_block_manager: bool):
513
"""
514
Test max lora is respected and prioritized.
515
"""
516
+ block_size = 4
517
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
429
- scheduler = initialize_scheduler(lora_config=lora_config)
518
+ scheduler = initialize_scheduler(lora_config=lora_config,
519
+ use_v2_block_manager=use_v2_block_manager,
520
+ block_size=block_size,
521
+ num_cpu_blocks=64,
522
+ num_gpu_blocks=64)
523
budget = create_token_budget(token_budget=120)
524
curr_loras: Set[int] = set()
525
for i in range(2):
526
_, seq_group = create_dummy_prompt(str(i),
527
prompt_length=60,
528
+ block_size=block_size,
529
lora_request=LoRARequest(
530
lora_name=str(i),
531
lora_int_id=i + 1,
@@ -443,7 +537,9 @@ def test_prefill_schedule_max_lora():
537
# If a request is not scheduled because it hits max lora, it is
538
# prioritized. Verify that.
539
for i in range(2, 4):
446
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
540
+ _, seq_group = create_dummy_prompt(str(i),
541
+ prompt_length=60,
542
+ block_size=block_size)
543
scheduler.add_seq_group(seq_group)
544
# Schedule 2 requests (0 and 2)
545
output = scheduler._schedule_prefills(budget, curr_loras)
@@ -467,14 +563,21 @@ def test_prefill_schedule_max_lora():
563
assert budget.num_batched_tokens == 60
564
565
470
-def test_prefill_schedule_no_block_manager_capacity():
566
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
567
+def test_prefill_schedule_no_block_manager_capacity(use_v2_block_manager):
568
"""
569
Test sequence cannot be scheduled due to block manager has no capacity.
570
"""
474
- scheduler = initialize_scheduler()
571
+ block_size = 4
572
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
573
+ block_size=block_size,
574
+ num_gpu_blocks=128,
575
+ num_cpu_blocks=128)
576
budget = create_token_budget()
577
for i in range(3):
477
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
578
+ _, seq_group = create_dummy_prompt(str(i),
579
+ prompt_length=60,
580
+ block_size=block_size)
581
scheduler.add_seq_group(seq_group)
582
scheduler.block_manager.can_allocate = MagicMock()
583
scheduler.block_manager.can_allocate.return_value = AllocStatus.LATER
@@ -489,7 +592,9 @@ def test_prefill_schedule_no_block_manager_capacity():
592
scheduler = initialize_scheduler()
593
budget = create_token_budget()
594
for i in range(3):
492
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
595
+ _, seq_group = create_dummy_prompt(str(i),
596
+ prompt_length=60,
597
+ block_size=block_size)
598
scheduler.add_seq_group(seq_group)
599
scheduler.block_manager.can_allocate = MagicMock()
600
scheduler.block_manager.can_allocate.return_value = AllocStatus.NEVER
@@ -502,14 +607,21 @@ def test_prefill_schedule_no_block_manager_capacity():
607
assert len(remaining_waiting) == 0
608
609
505
-def test_decode_schedule_preempted():
610
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
611
+def test_decode_schedule_preempted(use_v2_block_manager: bool):
612
"""
613
Test decodes cannot be scheduled and preempted.
614
"""
509
- scheduler = initialize_scheduler()
615
+ block_size = 4
616
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
617
+ block_size=block_size,
618
+ num_cpu_blocks=64,
619
+ num_gpu_blocks=64)
620
curr_loras = None
621
for i in range(3):
512
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
622
+ _, seq_group = create_dummy_prompt(str(i),
623
+ prompt_length=60,
624
+ block_size=block_size)
625
scheduler._allocate_and_set_running(seq_group)
626
append_new_token_seq_group(60, seq_group, 1)
627
scheduler._add_seq_group_to_running(seq_group)
@@ -541,15 +653,23 @@ def test_decode_schedule_preempted():
653
assert output.blocks_to_copy == []
654
655
544
-def test_decode_swap_beam_search():
656
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
657
+def test_decode_swap_beam_search(use_v2_block_manager: bool):
658
"""
659
Test best_of > 1 swap out blocks
660
"""
548
- scheduler = initialize_scheduler()
661
+ block_size = 4
662
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
663
+ block_size=block_size,
664
+ num_gpu_blocks=64,
665
+ num_cpu_blocks=64)
666
curr_loras = None
667
budget = create_token_budget()
668
for i in range(3):
552
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
669
+ _, seq_group = create_dummy_prompt(str(i),
670
+ prompt_length=60,
671
+ best_of=2,
672
+ block_size=block_size)
673
scheduler._allocate_and_set_running(seq_group)
674
scheduler._add_seq_group_to_running(seq_group)
675
append_new_token_seq_group(60, seq_group, 1)
@@ -589,12 +709,20 @@ def test_decode_swap_beam_search():
709
assert output.blocks_to_copy == []
710
711
592
-def test_schedule_decode_blocks_to_copy_update():
712
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
713
+def test_schedule_decode_blocks_to_copy_update(use_v2_block_manager: bool):
714
"""
715
Verify blocks_to_copy is updated.
716
"""
596
- scheduler = initialize_scheduler()
597
- _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
717
+ block_size = 4
718
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
719
+ block_size=4,
720
+ num_cpu_blocks=16,
721
+ num_gpu_blocks=16)
722
+ _, seq_group = create_dummy_prompt("1",
723
+ prompt_length=60,
724
+ best_of=2,
725
+ block_size=block_size)
726
curr_loras = None
727
scheduler._allocate_and_set_running(seq_group)
728
append_new_token_seq_group(60, seq_group, 1)
@@ -644,12 +772,17 @@ def test_schedule_swapped_simple():
772
assert blocks_to_swap_out == blocks_to_swap_in_reverse
773
774
647
-def test_schedule_swapped_max_token_budget():
648
- scheduler = initialize_scheduler()
775
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
776
+def test_schedule_swapped_max_token_budget(use_v2_block_manager: bool):
777
+ block_size = 4
778
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
779
+ block_size=block_size,
780
+ num_cpu_blocks=32,
781
+ num_gpu_blocks=32)
782
curr_loras = None
783
blocks_to_swap_out: List[Tuple[int, int]] = []
651
- for _ in range(2):
652
- _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
784
+ for i in range(2):
785
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
786
scheduler._allocate_and_set_running(seq_group)
787
append_new_token_seq_group(60, seq_group, 1)
788
scheduler._swap_out(seq_group, blocks_to_swap_out)
@@ -676,12 +809,19 @@ def test_schedule_swapped_max_token_budget():
809
assert len(output.prefill_seq_groups) == 0
810
811
679
-def test_schedule_swapped_max_seqs():
680
- scheduler = initialize_scheduler()
812
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
813
+def test_schedule_swapped_max_seqs(use_v2_block_manager: bool):
814
+ block_size = 4
815
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
816
+ block_size=block_size,
817
+ num_cpu_blocks=64,
818
+ num_gpu_blocks=64)
819
curr_loras = None
820
blocks_to_swap_out: List[Tuple[int, int]] = []
821
for i in range(4):
684
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
822
+ _, seq_group = create_dummy_prompt(str(i),
823
+ prompt_length=60,
824
+ block_size=4)
825
scheduler._allocate_and_set_running(seq_group)
826
append_new_token_seq_group(60, seq_group, 1)
827
scheduler._swap_out(seq_group, blocks_to_swap_out)
@@ -706,14 +846,21 @@ def test_schedule_swapped_max_seqs():
846
assert len(output.prefill_seq_groups) == 0
847
848
709
-def test_schedule_swapped_max_loras():
849
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
850
+def test_schedule_swapped_max_loras(use_v2_block_manager: bool):
851
+ block_size = 4
852
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
711
- scheduler = initialize_scheduler(lora_config=lora_config)
853
+ scheduler = initialize_scheduler(lora_config=lora_config,
854
+ use_v2_block_manager=use_v2_block_manager,
855
+ block_size=block_size,
856
+ num_cpu_blocks=32,
857
+ num_gpu_blocks=32)
858
curr_loras: Set[int] = set()
859
blocks_to_swap_out: List[Tuple[int, int]] = []
860
for i in range(2):
861
_, seq_group = create_dummy_prompt(str(i),
862
prompt_length=60,
863
+ block_size=block_size,
864
lora_request=LoRARequest(
865
lora_name=str(i),
866
lora_int_id=i + 1,
@@ -734,12 +881,20 @@ def test_schedule_swapped_max_loras():
881
assert len(curr_loras) == 1
882
883
737
-def test_schedule_swapped_cannot_swap_in():
738
- scheduler = initialize_scheduler()
884
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
885
+def test_schedule_swapped_cannot_swap_in(use_v2_block_manager: bool):
886
+ block_size = 4
887
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
888
+ block_size=block_size,
889
+ num_cpu_blocks=32,
890
+ num_gpu_blocks=32)
891
curr_loras = None
892
blocks_to_swap_out: List[Tuple[int, int]] = []
741
- for _ in range(2):
742
- _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
893
+ for i in range(2):
894
+ _, seq_group = create_dummy_prompt(str(i),
895
+ prompt_length=60,
896
+ best_of=2,
897
+ block_size=block_size)
898
scheduler._allocate_and_set_running(seq_group)
899
append_new_token_seq_group(60, seq_group, 1)
900
scheduler._swap_out(seq_group, blocks_to_swap_out)
@@ -759,12 +914,20 @@ def test_schedule_swapped_cannot_swap_in():
914
assert len(output.prefill_seq_groups) == 0
915
916
762
-def test_infeasible_swap():
763
- scheduler = initialize_scheduler()
917
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
918
+def test_infeasible_swap(use_v2_block_manager: bool):
919
+ block_size = 4
920
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
921
+ block_size=block_size,
922
+ num_cpu_blocks=32,
923
+ num_gpu_blocks=32)
924
curr_loras = None
925
blocks_to_swap_out: List[Tuple[int, int]] = []
766
- for _ in range(2):
767
- _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
926
+ for i in range(2):
927
+ _, seq_group = create_dummy_prompt(str(i),
928
+ prompt_length=60,
929
+ best_of=2,
930
+ block_size=block_size)
931
scheduler._allocate_and_set_running(seq_group)
932
append_new_token_seq_group(60, seq_group, 1)
933
scheduler._swap_out(seq_group, blocks_to_swap_out)
@@ -785,10 +948,18 @@ def test_infeasible_swap():
948
assert len(output.prefill_seq_groups) == 0
949
950
788
-def test_schedule_swapped_blocks_to_copy():
789
- scheduler = initialize_scheduler()
951
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
952
+def test_schedule_swapped_blocks_to_copy(use_v2_block_manager: bool):
953
+ block_size = 4
954
+ scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
955
+ block_size=block_size,
956
+ num_cpu_blocks=32,
957
+ num_gpu_blocks=32)
958
curr_loras = None
791
- _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
959
+ _, seq_group = create_dummy_prompt("1",
960
+ prompt_length=60,
961
+ best_of=2,
962
+ block_size=block_size)
963
scheduler._allocate_and_set_running(seq_group)
964
append_new_token_seq_group(60, seq_group, 1)
965
blocks_to_swap_out: List[Tuple[int, int]] = []