@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 88577ac928
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

Fix tests in test_scheduler.py that fail with BlockManager V2 (#8728)

sroy745 committed Sep 23, 2024 at 21:43 UTC 88577ac92808cfd9468e4b54b757d5fcbe9aa486
1 file changed +260 -89
tests/core/test_scheduler.py
+260 -89
@@ -3,7 +3,8 @@ from collections import deque
3 from typing import List, Set, Tuple
4 from unittest.mock import MagicMock
5
6 -import pytest # noqa
6 +import pytest
7 +from torch import Use # noqa
8
9 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
10 from vllm.core.interfaces import AllocStatus
@@ -16,9 +17,11 @@ from .utils import (append_new_token, append_new_token_seq_group,
17 schedule_and_update_computed_tokens)
18
19
19 -def test_scheduler_add_seq_group():
20 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
21 +def test_scheduler_add_seq_group(use_v2_block_manager: bool):
22 block_size = 4
21 - scheduler_config = SchedulerConfig(100, 64, 1)
23 + scheduler_config = SchedulerConfig(
24 + 100, 64, 1, use_v2_block_manager=use_v2_block_manager)
25 cache_config = CacheConfig(block_size, 1.0, 1, cache_dtype="auto")
26 cache_config.num_cpu_blocks = 4
27 cache_config.num_gpu_blocks = 4
@@ -27,14 +30,18 @@ def test_scheduler_add_seq_group():
30 # Add seq group to scheduler.
31 num_seq_group = 4
32 for i in range(num_seq_group):
30 - _, seq_group = create_dummy_prompt(str(i), block_size)
33 + _, seq_group = create_dummy_prompt(str(i),
34 + block_size,
35 + block_size=block_size)
36 scheduler.add_seq_group(seq_group)
37 assert scheduler.get_num_unfinished_seq_groups() == i + 1
38
39
35 -def test_scheduler_abort_seq_group():
40 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
41 +def test_scheduler_abort_seq_group(use_v2_block_manager: bool):
42 block_size = 4
37 - scheduler_config = SchedulerConfig(100, 64, 1)
43 + scheduler_config = SchedulerConfig(
44 + 100, 64, 1, use_v2_block_manager=use_v2_block_manager)
45 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
46 cache_config.num_cpu_blocks = 4
47 cache_config.num_gpu_blocks = 4
@@ -54,11 +61,16 @@ def test_scheduler_abort_seq_group():
61 assert scheduler.get_num_unfinished_seq_groups() == 0
62
63
57 -def test_scheduler_schedule_simple():
64 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
65 +def test_scheduler_schedule_simple(use_v2_block_manager: bool):
66 block_size = 4
67 num_seq_group = 4
68 max_model_len = 16
61 - scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len)
69 + scheduler_config = SchedulerConfig(
70 + 64,
71 + num_seq_group,
72 + max_model_len,
73 + use_v2_block_manager=use_v2_block_manager)
74 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
75 cache_config.num_cpu_blocks = 8
76 cache_config.num_gpu_blocks = 8
@@ -67,7 +79,9 @@ def test_scheduler_schedule_simple():
79
80 # Add seq groups to scheduler.
81 for i in range(num_seq_group):
70 - _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
82 + _, seq_group = create_dummy_prompt(str(i),
83 + prompt_length=block_size,
84 + block_size=block_size)
85 scheduler.add_seq_group(seq_group)
86 running.append(seq_group)
87
@@ -91,20 +105,24 @@ def test_scheduler_schedule_simple():
105 append_new_token(out, 1)
106
107
94 -def test_scheduler_prefill_prioritized():
108 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
109 +def test_scheduler_prefill_prioritized(use_v2_block_manager: bool):
110 """Verify running batched tokens are not applied to prefill requests."""
111 block_size = 4
112 max_model_len = 30
113 max_batched_num_tokens = 30
99 - scheduler_config = SchedulerConfig(max_batched_num_tokens, 2,
100 - max_model_len)
114 + scheduler_config = SchedulerConfig(
115 + max_batched_num_tokens,
116 + 2,
117 + max_model_len,
118 + use_v2_block_manager=use_v2_block_manager)
119 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
102 - cache_config.num_cpu_blocks = 2
103 - cache_config.num_gpu_blocks = 2
120 + cache_config.num_cpu_blocks = 16
121 + cache_config.num_gpu_blocks = 16
122 scheduler = Scheduler(scheduler_config, cache_config, None)
123
124 # Add seq groups to scheduler.
107 - _, seq_group_a = create_dummy_prompt("1", 1)
125 + _, seq_group_a = create_dummy_prompt("1", 1, block_size=block_size)
126 scheduler.add_seq_group(seq_group_a)
127
128 # Schedule seq groups prompts.
@@ -112,7 +130,7 @@ def test_scheduler_prefill_prioritized():
130 assert get_sequence_groups(out) == [seq_group_a]
131
132 # Add a new prefill request B.
115 - _, seq_group_b = create_dummy_prompt("2", 30)
133 + _, seq_group_b = create_dummy_prompt("2", 30, block_size=block_size)
134 scheduler.add_seq_group(seq_group_b)
135
136 # Verify prefill requests are prioritized. Since max_batched_num_tokens
@@ -121,18 +139,24 @@ def test_scheduler_prefill_prioritized():
139 assert get_sequence_groups(out) == [seq_group_b]
140
141
124 -def test_scheduler_schedule_preempt_abort():
142 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
143 +def test_scheduler_schedule_preempt_abort(use_v2_block_manager: bool):
144 block_size = 4
145 max_model_len = 16
127 - scheduler_config = SchedulerConfig(64, 2, max_model_len)
146 + scheduler_config = SchedulerConfig(
147 + 64, 2, max_model_len, use_v2_block_manager=use_v2_block_manager)
148 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
149 cache_config.num_cpu_blocks = 2
150 cache_config.num_gpu_blocks = 2
151 scheduler = Scheduler(scheduler_config, cache_config, None)
152
153 # Add seq groups to scheduler.
134 - seq_a, seq_group_a = create_dummy_prompt("1", block_size)
135 - seq_b, seq_group_b = create_dummy_prompt("2", block_size)
154 + seq_a, seq_group_a = create_dummy_prompt("1",
155 + block_size,
156 + block_size=block_size)
157 + seq_b, seq_group_b = create_dummy_prompt("2",
158 + block_size,
159 + block_size=block_size)
160 scheduler.add_seq_group(seq_group_a)
161 scheduler.add_seq_group(seq_group_b)
162
@@ -170,12 +194,17 @@ def test_scheduler_schedule_preempt_abort():
194 assert scheduler.get_num_unfinished_seq_groups() == 1
195
196
173 -def test_scheduler_max_seqs():
197 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
198 +def test_scheduler_max_seqs(use_v2_block_manager: bool):
199 block_size = 4
200 num_seq_group = 4
201 max_seq_group = 2
202 max_model_len = 16
178 - scheduler_config = SchedulerConfig(64, max_seq_group, max_model_len)
203 + scheduler_config = SchedulerConfig(
204 + 64,
205 + max_seq_group,
206 + max_model_len,
207 + use_v2_block_manager=use_v2_block_manager)
208 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
209 cache_config.num_cpu_blocks = 8
210 cache_config.num_gpu_blocks = 8
@@ -184,7 +213,9 @@ def test_scheduler_max_seqs():
213 all_seq_groups: List[SequenceGroup] = []
214 # Add seq groups to scheduler.
215 for i in range(num_seq_group):
187 - _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
216 + _, seq_group = create_dummy_prompt(str(i),
217 + prompt_length=block_size,
218 + block_size=block_size)
219 all_seq_groups.append(seq_group)
220
221 # Append 1 seq group
@@ -211,9 +242,15 @@ def test_scheduler_max_seqs():
242 assert set(get_sequence_groups(out)) == set([all_seq_groups[1]])
243
244
214 -def test_scheduler_delay_factor():
245 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
246 +def test_scheduler_delay_factor(use_v2_block_manager: bool):
247 block_size = 4
216 - scheduler_config = SchedulerConfig(100, 64, 16, delay_factor=0.5)
248 + scheduler_config = SchedulerConfig(
249 + 100,
250 + 64,
251 + 16,
252 + delay_factor=0.5,
253 + use_v2_block_manager=use_v2_block_manager)
254 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
255 cache_config.num_cpu_blocks = 8
256 cache_config.num_gpu_blocks = 8
@@ -221,7 +258,8 @@ def test_scheduler_delay_factor():
258
259 # schedule first prompt
260 seq_group_meta, seq_group = create_dummy_prompt("0",
224 - prompt_length=block_size)
261 + prompt_length=block_size,
262 + block_size=block_size)
263 scheduler.add_seq_group(seq_group)
264 seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
265 assert out.num_prefill_groups > 0
@@ -231,7 +269,8 @@ def test_scheduler_delay_factor():
269 # wait for a second before scheduling next prompt
270 time.sleep(1)
271 seq_group_meta, seq_group = create_dummy_prompt("1",
234 - prompt_length=block_size)
272 + prompt_length=block_size,
273 + block_size=block_size)
274 scheduler.add_seq_group(seq_group)
275
276 # second prompt should *not* be scheduled
@@ -248,11 +287,20 @@ def test_scheduler_delay_factor():
287 append_new_token(out, 1)
288
289
251 -def test_swapped_out_prioritized():
252 - scheduler = initialize_scheduler(max_num_seqs=6)
290 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
291 +def test_swapped_out_prioritized(use_v2_block_manager: bool):
292 + block_size = 4
293 + scheduler = initialize_scheduler(max_num_seqs=6,
294 + block_size=block_size,
295 + use_v2_block_manager=use_v2_block_manager,
296 + num_cpu_blocks=64,
297 + num_gpu_blocks=64)
298 # best_of=2 * 3 == 6 sequences.
299 for i in range(3):
255 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
300 + _, seq_group = create_dummy_prompt(str(i),
301 + prompt_length=60,
302 + best_of=2,
303 + block_size=block_size)
304 scheduler.add_seq_group(seq_group)
305 seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
306 # prefill scheduled now.
@@ -276,7 +324,10 @@ def test_swapped_out_prioritized():
324 append_new_token(out, 1)
325
326 # Add 1 more task. Swap should be prioritized over prefill.
279 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
327 + _, seq_group = create_dummy_prompt(str(i),
328 + prompt_length=60,
329 + best_of=2,
330 + block_size=block_size)
331 scheduler.add_seq_group(seq_group)
332 seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
333 append_new_token(out, 1)
@@ -287,17 +338,26 @@ def test_swapped_out_prioritized():
338 assert out.blocks_to_swap_out == []
339
340
290 -def initialize_scheduler(*,
291 - max_num_seqs=1000,
292 - max_token_budget=1000,
293 - max_model_len=1000,
294 - lora_config=None):
295 - block_size = 4
296 - scheduler_config = SchedulerConfig(max_token_budget, max_num_seqs,
297 - max_model_len)
341 +def initialize_scheduler(
342 + *,
343 + max_num_seqs=1000,
344 + max_token_budget=1000,
345 + max_model_len=1000,
346 + lora_config=None,
347 + use_v2_block_manager=False,
348 + block_size=4,
349 + num_cpu_blocks=8,
350 + num_gpu_blocks=8,
351 +):
352 + block_size = block_size
353 + scheduler_config = SchedulerConfig(
354 + max_token_budget,
355 + max_num_seqs,
356 + max_model_len,
357 + use_v2_block_manager=use_v2_block_manager)
358 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
299 - cache_config.num_cpu_blocks = 8
300 - cache_config.num_gpu_blocks = 8
359 + cache_config.num_cpu_blocks = num_cpu_blocks
360 + cache_config.num_gpu_blocks = num_gpu_blocks
361 scheduler = Scheduler(scheduler_config, cache_config, lora_config)
362 return scheduler
363
@@ -319,12 +379,18 @@ def add_token_budget(budget: SchedulingBudget,
379 budget.add_num_seqs(mock_seq_group.request_id, num_curr_seqs)
380
381
322 -def test_prefill_schedule_max_prompt_len():
382 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
383 +def test_prefill_schedule_max_prompt_len(use_v2_block_manager: bool):
384 """
385 Test prompt longer than max_prompt_len is aborted.
386 """
326 - scheduler = initialize_scheduler(max_model_len=30)
327 - _, seq_group = create_dummy_prompt("0", prompt_length=60)
387 + block_size = 4
388 + scheduler = initialize_scheduler(max_model_len=30,
389 + use_v2_block_manager=use_v2_block_manager,
390 + block_size=block_size)
391 + _, seq_group = create_dummy_prompt("0",
392 + prompt_length=60,
393 + block_size=block_size)
394 scheduler.add_seq_group(seq_group)
395 budget = create_token_budget()
396 output = scheduler._schedule_prefills(budget, None)
@@ -336,14 +402,21 @@ def test_prefill_schedule_max_prompt_len():
402 assert len(remaining_waiting) == 0
403
404
339 -def test_prefill_schedule_token_budget():
405 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
406 +def test_prefill_schedule_token_budget(use_v2_block_manager: bool):
407 """
408 Test token budget respected.
409 """
343 - scheduler = initialize_scheduler()
410 + block_size = 4
411 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
412 + block_size=block_size,
413 + num_cpu_blocks=64,
414 + num_gpu_blocks=64)
415 budget = create_token_budget(token_budget=0)
416 for i in range(2):
346 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
417 + _, seq_group = create_dummy_prompt(str(i),
418 + prompt_length=60,
419 + block_size=block_size)
420 scheduler.add_seq_group(seq_group)
421
422 # 0 token budget == nothing is scheduled.
@@ -366,10 +439,15 @@ def test_prefill_schedule_token_budget():
439 assert len(remaining_waiting) == 1
440
441 # Test when current_batched_tokens respected.
369 - scheduler = initialize_scheduler()
442 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
443 + block_size=block_size,
444 + num_cpu_blocks=16,
445 + num_gpu_blocks=16)
446 budget = create_token_budget(token_budget=60)
447 add_token_budget(budget, 30, 0)
372 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
448 + _, seq_group = create_dummy_prompt(str(i),
449 + prompt_length=60,
450 + block_size=block_size)
451 # Cannot schedule a prompt that doesn't fit the budget.
452 scheduler.add_seq_group(seq_group)
453 output = scheduler._schedule_prefills(budget, None)
@@ -389,14 +467,21 @@ def test_prefill_schedule_token_budget():
467 assert len(remaining_waiting) == 0
468
469
392 -def test_prefill_schedule_max_seqs():
470 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
471 +def test_prefill_schedule_max_seqs(use_v2_block_manager: bool):
472 """
473 Test max seq respected.
474 """
396 - scheduler = initialize_scheduler()
475 + block_size = 4
476 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
477 + block_size=block_size,
478 + num_cpu_blocks=64,
479 + num_gpu_blocks=64)
480 budget = create_token_budget(max_num_seqs=2)
481 for i in range(3):
399 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
482 + _, seq_group = create_dummy_prompt(str(i),
483 + prompt_length=60,
484 + block_size=block_size)
485 scheduler.add_seq_group(seq_group)
486 output = scheduler._schedule_prefills(budget, None)
487 remaining_waiting = scheduler.waiting
@@ -410,7 +495,9 @@ def test_prefill_schedule_max_seqs():
495 scheduler.waiting = deque()
496 budget = create_token_budget(max_num_seqs=2)
497 add_token_budget(budget, 0, 2)
413 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
498 + _, seq_group = create_dummy_prompt(str(i),
499 + prompt_length=60,
500 + block_size=block_size)
501 scheduler.add_seq_group(seq_group)
502 output = scheduler._schedule_prefills(budget, None)
503 remaining_waiting = scheduler.waiting
@@ -421,17 +508,24 @@ def test_prefill_schedule_max_seqs():
508 assert len(remaining_waiting) == 1
509
510
424 -def test_prefill_schedule_max_lora():
511 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
512 +def test_prefill_schedule_max_lora(use_v2_block_manager: bool):
513 """
514 Test max lora is respected and prioritized.
515 """
516 + block_size = 4
517 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
429 - scheduler = initialize_scheduler(lora_config=lora_config)
518 + scheduler = initialize_scheduler(lora_config=lora_config,
519 + use_v2_block_manager=use_v2_block_manager,
520 + block_size=block_size,
521 + num_cpu_blocks=64,
522 + num_gpu_blocks=64)
523 budget = create_token_budget(token_budget=120)
524 curr_loras: Set[int] = set()
525 for i in range(2):
526 _, seq_group = create_dummy_prompt(str(i),
527 prompt_length=60,
528 + block_size=block_size,
529 lora_request=LoRARequest(
530 lora_name=str(i),
531 lora_int_id=i + 1,
@@ -443,7 +537,9 @@ def test_prefill_schedule_max_lora():
537 # If a request is not scheduled because it hits max lora, it is
538 # prioritized. Verify that.
539 for i in range(2, 4):
446 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
540 + _, seq_group = create_dummy_prompt(str(i),
541 + prompt_length=60,
542 + block_size=block_size)
543 scheduler.add_seq_group(seq_group)
544 # Schedule 2 requests (0 and 2)
545 output = scheduler._schedule_prefills(budget, curr_loras)
@@ -467,14 +563,21 @@ def test_prefill_schedule_max_lora():
563 assert budget.num_batched_tokens == 60
564
565
470 -def test_prefill_schedule_no_block_manager_capacity():
566 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
567 +def test_prefill_schedule_no_block_manager_capacity(use_v2_block_manager):
568 """
569 Test sequence cannot be scheduled due to block manager has no capacity.
570 """
474 - scheduler = initialize_scheduler()
571 + block_size = 4
572 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
573 + block_size=block_size,
574 + num_gpu_blocks=128,
575 + num_cpu_blocks=128)
576 budget = create_token_budget()
577 for i in range(3):
477 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
578 + _, seq_group = create_dummy_prompt(str(i),
579 + prompt_length=60,
580 + block_size=block_size)
581 scheduler.add_seq_group(seq_group)
582 scheduler.block_manager.can_allocate = MagicMock()
583 scheduler.block_manager.can_allocate.return_value = AllocStatus.LATER
@@ -489,7 +592,9 @@ def test_prefill_schedule_no_block_manager_capacity():
592 scheduler = initialize_scheduler()
593 budget = create_token_budget()
594 for i in range(3):
492 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
595 + _, seq_group = create_dummy_prompt(str(i),
596 + prompt_length=60,
597 + block_size=block_size)
598 scheduler.add_seq_group(seq_group)
599 scheduler.block_manager.can_allocate = MagicMock()
600 scheduler.block_manager.can_allocate.return_value = AllocStatus.NEVER
@@ -502,14 +607,21 @@ def test_prefill_schedule_no_block_manager_capacity():
607 assert len(remaining_waiting) == 0
608
609
505 -def test_decode_schedule_preempted():
610 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
611 +def test_decode_schedule_preempted(use_v2_block_manager: bool):
612 """
613 Test decodes cannot be scheduled and preempted.
614 """
509 - scheduler = initialize_scheduler()
615 + block_size = 4
616 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
617 + block_size=block_size,
618 + num_cpu_blocks=64,
619 + num_gpu_blocks=64)
620 curr_loras = None
621 for i in range(3):
512 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
622 + _, seq_group = create_dummy_prompt(str(i),
623 + prompt_length=60,
624 + block_size=block_size)
625 scheduler._allocate_and_set_running(seq_group)
626 append_new_token_seq_group(60, seq_group, 1)
627 scheduler._add_seq_group_to_running(seq_group)
@@ -541,15 +653,23 @@ def test_decode_schedule_preempted():
653 assert output.blocks_to_copy == []
654
655
544 -def test_decode_swap_beam_search():
656 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
657 +def test_decode_swap_beam_search(use_v2_block_manager: bool):
658 """
659 Test best_of > 1 swap out blocks
660 """
548 - scheduler = initialize_scheduler()
661 + block_size = 4
662 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
663 + block_size=block_size,
664 + num_gpu_blocks=64,
665 + num_cpu_blocks=64)
666 curr_loras = None
667 budget = create_token_budget()
668 for i in range(3):
552 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
669 + _, seq_group = create_dummy_prompt(str(i),
670 + prompt_length=60,
671 + best_of=2,
672 + block_size=block_size)
673 scheduler._allocate_and_set_running(seq_group)
674 scheduler._add_seq_group_to_running(seq_group)
675 append_new_token_seq_group(60, seq_group, 1)
@@ -589,12 +709,20 @@ def test_decode_swap_beam_search():
709 assert output.blocks_to_copy == []
710
711
592 -def test_schedule_decode_blocks_to_copy_update():
712 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
713 +def test_schedule_decode_blocks_to_copy_update(use_v2_block_manager: bool):
714 """
715 Verify blocks_to_copy is updated.
716 """
596 - scheduler = initialize_scheduler()
597 - _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
717 + block_size = 4
718 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
719 + block_size=4,
720 + num_cpu_blocks=16,
721 + num_gpu_blocks=16)
722 + _, seq_group = create_dummy_prompt("1",
723 + prompt_length=60,
724 + best_of=2,
725 + block_size=block_size)
726 curr_loras = None
727 scheduler._allocate_and_set_running(seq_group)
728 append_new_token_seq_group(60, seq_group, 1)
@@ -644,12 +772,17 @@ def test_schedule_swapped_simple():
772 assert blocks_to_swap_out == blocks_to_swap_in_reverse
773
774
647 -def test_schedule_swapped_max_token_budget():
648 - scheduler = initialize_scheduler()
775 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
776 +def test_schedule_swapped_max_token_budget(use_v2_block_manager: bool):
777 + block_size = 4
778 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
779 + block_size=block_size,
780 + num_cpu_blocks=32,
781 + num_gpu_blocks=32)
782 curr_loras = None
783 blocks_to_swap_out: List[Tuple[int, int]] = []
651 - for _ in range(2):
652 - _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
784 + for i in range(2):
785 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
786 scheduler._allocate_and_set_running(seq_group)
787 append_new_token_seq_group(60, seq_group, 1)
788 scheduler._swap_out(seq_group, blocks_to_swap_out)
@@ -676,12 +809,19 @@ def test_schedule_swapped_max_token_budget():
809 assert len(output.prefill_seq_groups) == 0
810
811
679 -def test_schedule_swapped_max_seqs():
680 - scheduler = initialize_scheduler()
812 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
813 +def test_schedule_swapped_max_seqs(use_v2_block_manager: bool):
814 + block_size = 4
815 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
816 + block_size=block_size,
817 + num_cpu_blocks=64,
818 + num_gpu_blocks=64)
819 curr_loras = None
820 blocks_to_swap_out: List[Tuple[int, int]] = []
821 for i in range(4):
684 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
822 + _, seq_group = create_dummy_prompt(str(i),
823 + prompt_length=60,
824 + block_size=4)
825 scheduler._allocate_and_set_running(seq_group)
826 append_new_token_seq_group(60, seq_group, 1)
827 scheduler._swap_out(seq_group, blocks_to_swap_out)
@@ -706,14 +846,21 @@ def test_schedule_swapped_max_seqs():
846 assert len(output.prefill_seq_groups) == 0
847
848
709 -def test_schedule_swapped_max_loras():
849 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
850 +def test_schedule_swapped_max_loras(use_v2_block_manager: bool):
851 + block_size = 4
852 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
711 - scheduler = initialize_scheduler(lora_config=lora_config)
853 + scheduler = initialize_scheduler(lora_config=lora_config,
854 + use_v2_block_manager=use_v2_block_manager,
855 + block_size=block_size,
856 + num_cpu_blocks=32,
857 + num_gpu_blocks=32)
858 curr_loras: Set[int] = set()
859 blocks_to_swap_out: List[Tuple[int, int]] = []
860 for i in range(2):
861 _, seq_group = create_dummy_prompt(str(i),
862 prompt_length=60,
863 + block_size=block_size,
864 lora_request=LoRARequest(
865 lora_name=str(i),
866 lora_int_id=i + 1,
@@ -734,12 +881,20 @@ def test_schedule_swapped_max_loras():
881 assert len(curr_loras) == 1
882
883
737 -def test_schedule_swapped_cannot_swap_in():
738 - scheduler = initialize_scheduler()
884 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
885 +def test_schedule_swapped_cannot_swap_in(use_v2_block_manager: bool):
886 + block_size = 4
887 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
888 + block_size=block_size,
889 + num_cpu_blocks=32,
890 + num_gpu_blocks=32)
891 curr_loras = None
892 blocks_to_swap_out: List[Tuple[int, int]] = []
741 - for _ in range(2):
742 - _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
893 + for i in range(2):
894 + _, seq_group = create_dummy_prompt(str(i),
895 + prompt_length=60,
896 + best_of=2,
897 + block_size=block_size)
898 scheduler._allocate_and_set_running(seq_group)
899 append_new_token_seq_group(60, seq_group, 1)
900 scheduler._swap_out(seq_group, blocks_to_swap_out)
@@ -759,12 +914,20 @@ def test_schedule_swapped_cannot_swap_in():
914 assert len(output.prefill_seq_groups) == 0
915
916
762 -def test_infeasible_swap():
763 - scheduler = initialize_scheduler()
917 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
918 +def test_infeasible_swap(use_v2_block_manager: bool):
919 + block_size = 4
920 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
921 + block_size=block_size,
922 + num_cpu_blocks=32,
923 + num_gpu_blocks=32)
924 curr_loras = None
925 blocks_to_swap_out: List[Tuple[int, int]] = []
766 - for _ in range(2):
767 - _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
926 + for i in range(2):
927 + _, seq_group = create_dummy_prompt(str(i),
928 + prompt_length=60,
929 + best_of=2,
930 + block_size=block_size)
931 scheduler._allocate_and_set_running(seq_group)
932 append_new_token_seq_group(60, seq_group, 1)
933 scheduler._swap_out(seq_group, blocks_to_swap_out)
@@ -785,10 +948,18 @@ def test_infeasible_swap():
948 assert len(output.prefill_seq_groups) == 0
949
950
788 -def test_schedule_swapped_blocks_to_copy():
789 - scheduler = initialize_scheduler()
951 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
952 +def test_schedule_swapped_blocks_to_copy(use_v2_block_manager: bool):
953 + block_size = 4
954 + scheduler = initialize_scheduler(use_v2_block_manager=use_v2_block_manager,
955 + block_size=block_size,
956 + num_cpu_blocks=32,
957 + num_gpu_blocks=32)
958 curr_loras = None
791 - _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
959 + _, seq_group = create_dummy_prompt("1",
960 + prompt_length=60,
961 + best_of=2,
962 + block_size=block_size)
963 scheduler._allocate_and_set_running(seq_group)
964 append_new_token_seq_group(60, seq_group, 1)
965 blocks_to_swap_out: List[Tuple[int, int]] = []