@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc3afc20df
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

Fix tests in test_chunked_prefill_scheduler which fail with BlockManager V2 (#8752)

sroy745 committed Sep 24, 2024 at 21:26 UTC fc3afc20df410dd523f94967b98836084f561ab7
1 file changed +143 -82
tests/core/test_chunked_prefill_scheduler.py
+143 -82
@@ -27,16 +27,19 @@ def schedule_and_update_computed_tokens(scheduler):
27 return metas, out
28
29
30 -def test_simple():
30 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
31 +def test_simple(use_v2_block_manager: bool):
32 """Verify basic scheduling works."""
33 block_size = 4
34 num_seq_group = 4
35 max_model_len = 16
36 max_num_batched_tokens = 64
36 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
37 - num_seq_group,
38 - max_model_len,
39 - enable_chunked_prefill=True)
37 + scheduler_config = SchedulerConfig(
38 + max_num_batched_tokens,
39 + num_seq_group,
40 + max_model_len,
41 + enable_chunked_prefill=True,
42 + use_v2_block_manager=use_v2_block_manager)
43 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
44 cache_config.num_cpu_blocks = 8
45 cache_config.num_gpu_blocks = 8
@@ -45,7 +48,9 @@ def test_simple():
48
49 # Add seq groups to scheduler.
50 for i in range(num_seq_group):
48 - _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
51 + _, seq_group = create_dummy_prompt(str(i),
52 + prompt_length=block_size,
53 + block_size=block_size)
54 scheduler.add_seq_group(seq_group)
55 running.append(seq_group)
56
@@ -69,30 +74,36 @@ def test_simple():
74 assert len(seq_group_meta) == num_seq_group
75
76
72 -def test_chunk():
77 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
78 +def test_chunk(use_v2_block_manager: bool):
79 """Verify prefills are chunked properly."""
80 block_size = 4
81 max_seqs = 60
82 max_model_len = 80
83 max_num_batched_tokens = 64
78 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
79 - max_seqs,
80 - max_model_len,
81 - enable_chunked_prefill=True)
84 + scheduler_config = SchedulerConfig(
85 + max_num_batched_tokens,
86 + max_seqs,
87 + max_model_len,
88 + enable_chunked_prefill=True,
89 + use_v2_block_manager=use_v2_block_manager)
90 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
83 - cache_config.num_cpu_blocks = 8
84 - cache_config.num_gpu_blocks = 8
91 + cache_config.num_cpu_blocks = 32
92 + cache_config.num_gpu_blocks = 32
93 scheduler = Scheduler(scheduler_config, cache_config, None)
94 running: List[SequenceGroup] = []
95
96 # Add seq groups to scheduler.
97 for i in range(2):
90 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
98 + _, seq_group = create_dummy_prompt(str(i),
99 + prompt_length=60,
100 + block_size=block_size)
101 scheduler.add_seq_group(seq_group)
102 running.append(seq_group)
103
104 # Verify the second request is chunked.
105 seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
106 + print()
107 assert set(get_sequence_groups(out)) == set(running)
108 assert seq_group_meta[0].token_chunk_size == 60
109 # Verify it is chunked.
@@ -113,24 +124,29 @@ def test_chunk():
124 assert out.num_batched_tokens == 57
125
126
116 -def test_complex():
127 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
128 +def test_complex(use_v2_block_manager: bool):
129 block_size = 4
130 max_seqs = 60
131 max_model_len = 80
132 max_num_batched_tokens = 64
121 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
122 - max_seqs,
123 - max_model_len,
124 - enable_chunked_prefill=True)
133 + scheduler_config = SchedulerConfig(
134 + max_num_batched_tokens,
135 + max_seqs,
136 + max_model_len,
137 + enable_chunked_prefill=True,
138 + use_v2_block_manager=use_v2_block_manager)
139 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
126 - cache_config.num_cpu_blocks = 8
127 - cache_config.num_gpu_blocks = 8
140 + cache_config.num_cpu_blocks = 64
141 + cache_config.num_gpu_blocks = 64
142 scheduler = Scheduler(scheduler_config, cache_config, None)
143 running: List[SequenceGroup] = []
144
145 # Add seq groups to scheduler.
146 for i in range(2):
133 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
147 + _, seq_group = create_dummy_prompt(str(i),
148 + prompt_length=60,
149 + block_size=block_size)
150 scheduler.add_seq_group(seq_group)
151 running.append(seq_group)
152 assert seq_group.is_prefill()
@@ -151,7 +167,9 @@ def test_complex():
167
168 # Add 2 more requests.
169 for i in range(2, 4):
154 - _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
170 + _, seq_group = create_dummy_prompt(str(i),
171 + prompt_length=60,
172 + block_size=block_size)
173 scheduler.add_seq_group(seq_group)
174 running.append(seq_group)
175
@@ -176,16 +194,19 @@ def test_complex():
194 assert running[2].is_prefill()
195
196
179 -def test_maximal_decoding():
197 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
198 +def test_maximal_decoding(use_v2_block_manager: bool):
199 """Verify decoding requests are prioritized."""
200 block_size = 4
201 max_seqs = 2
202 max_model_len = 8
203 max_num_batched_tokens = 2
185 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
186 - max_seqs,
187 - max_model_len,
188 - enable_chunked_prefill=True)
204 + scheduler_config = SchedulerConfig(
205 + max_num_batched_tokens,
206 + max_seqs,
207 + max_model_len,
208 + enable_chunked_prefill=True,
209 + use_v2_block_manager=use_v2_block_manager)
210 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
211 cache_config.num_cpu_blocks = 8
212 cache_config.num_gpu_blocks = 8
@@ -194,7 +215,9 @@ def test_maximal_decoding():
215
216 # Add seq groups to scheduler.
217 for i in range(2):
197 - _, seq_group = create_dummy_prompt(str(i), prompt_length=2)
218 + _, seq_group = create_dummy_prompt(str(i),
219 + prompt_length=2,
220 + block_size=block_size)
221 scheduler.add_seq_group(seq_group)
222 running.append(seq_group)
223 assert seq_group.is_prefill()
@@ -211,7 +234,9 @@ def test_maximal_decoding():
234 append_new_token(running[0], 1)
235
236 # Create one more seq_group.
214 - _, seq_group = create_dummy_prompt("3", prompt_length=2)
237 + _, seq_group = create_dummy_prompt("3",
238 + prompt_length=2,
239 + block_size=block_size)
240 scheduler.add_seq_group(seq_group)
241 running.append(seq_group)
242 assert seq_group.is_prefill()
@@ -263,23 +288,28 @@ def test_maximal_decoding():
288 assert out.num_batched_tokens == 2
289
290
266 -def test_prompt_limit():
291 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
292 +def test_prompt_limit(use_v2_block_manager: bool):
293 """Verify max_num_batched_tokens < max_model_len is possible."""
294 block_size = 4
295 max_seqs = 32
296 max_model_len = 64
297 max_num_batched_tokens = 32
272 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
273 - max_seqs,
274 - max_model_len,
275 - enable_chunked_prefill=True)
298 + scheduler_config = SchedulerConfig(
299 + max_num_batched_tokens,
300 + max_seqs,
301 + max_model_len,
302 + enable_chunked_prefill=True,
303 + use_v2_block_manager=use_v2_block_manager)
304 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
277 - cache_config.num_cpu_blocks = 8
278 - cache_config.num_gpu_blocks = 8
305 + cache_config.num_cpu_blocks = 16
306 + cache_config.num_gpu_blocks = 16
307 scheduler = Scheduler(scheduler_config, cache_config, None)
308 running: List[SequenceGroup] = []
309
282 - _, seq_group = create_dummy_prompt("1", prompt_length=48)
310 + _, seq_group = create_dummy_prompt("1",
311 + prompt_length=48,
312 + block_size=block_size)
313 scheduler.add_seq_group(seq_group)
314 running.append(seq_group)
315 assert seq_group.is_prefill()
@@ -293,7 +323,8 @@ def test_prompt_limit():
323 assert out.num_batched_tokens == 32
324
325
296 -def test_prompt_limit_exceed():
326 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
327 +def test_prompt_limit_exceed(use_v2_block_manager: bool):
328 block_size = 4
329 max_seqs = 64
330 max_model_len = 32
@@ -303,12 +334,13 @@ def test_prompt_limit_exceed():
334 max_model_len,
335 enable_chunked_prefill=True)
336 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
306 - cache_config.num_cpu_blocks = 8
307 - cache_config.num_gpu_blocks = 8
337 + cache_config.num_cpu_blocks = 16
338 + cache_config.num_gpu_blocks = 16
339 scheduler = Scheduler(scheduler_config, cache_config, None)
340 running: List[SequenceGroup] = []
310 -
311 - _, seq_group = create_dummy_prompt("2", prompt_length=48)
341 + _, seq_group = create_dummy_prompt("2",
342 + prompt_length=48,
343 + block_size=block_size)
344 scheduler.add_seq_group(seq_group)
345 running.append(seq_group)
346 assert seq_group.is_prefill()
@@ -317,22 +349,28 @@ def test_prompt_limit_exceed():
349 assert out.ignored_seq_groups[0] == seq_group
350
351
320 -def test_swap():
352 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
353 +def test_swap(use_v2_block_manager: bool):
354 """Verify swapping works with chunked prefill requests"""
355 block_size = 4
356 max_seqs = 30
357 max_model_len = 200
358 max_num_batched_tokens = 30
326 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
327 - max_seqs,
328 - max_model_len,
329 - enable_chunked_prefill=True)
359 + scheduler_config = SchedulerConfig(
360 + max_num_batched_tokens,
361 + max_seqs,
362 + max_model_len,
363 + enable_chunked_prefill=True,
364 + use_v2_block_manager=use_v2_block_manager)
365 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
331 - cache_config.num_cpu_blocks = 8
332 - cache_config.num_gpu_blocks = 8
366 + cache_config.num_cpu_blocks = 16
367 + cache_config.num_gpu_blocks = 16
368 scheduler = Scheduler(scheduler_config, cache_config, None)
369
335 - _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
370 + _, seq_group = create_dummy_prompt("1",
371 + prompt_length=60,
372 + best_of=2,
373 + block_size=block_size)
374 scheduler.add_seq_group(seq_group)
375 _, out = schedule_and_update_computed_tokens(scheduler)
376 # The request is chunked.
@@ -369,21 +407,27 @@ def test_swap():
407 assert out.blocks_to_swap_out == []
408
409
372 -def test_running_prefill_prioritized_over_swap():
410 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
411 +def test_running_prefill_prioritized_over_swap(use_v2_block_manager: bool):
412 block_size = 4
413 max_seqs = 30
414 max_model_len = 200
415 max_num_batched_tokens = 30
377 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
378 - max_seqs,
379 - max_model_len,
380 - enable_chunked_prefill=True)
416 + scheduler_config = SchedulerConfig(
417 + max_num_batched_tokens,
418 + max_seqs,
419 + max_model_len,
420 + enable_chunked_prefill=True,
421 + use_v2_block_manager=use_v2_block_manager)
422 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
382 - cache_config.num_cpu_blocks = 8
383 - cache_config.num_gpu_blocks = 8
423 + cache_config.num_cpu_blocks = 32
424 + cache_config.num_gpu_blocks = 32
425 scheduler = Scheduler(scheduler_config, cache_config, None)
426
386 - _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
427 + _, seq_group = create_dummy_prompt("1",
428 + prompt_length=60,
429 + best_of=2,
430 + block_size=block_size)
431 scheduler.add_seq_group(seq_group)
432 _, out = schedule_and_update_computed_tokens(scheduler)
433 # The request is chunked.
@@ -413,7 +457,9 @@ def test_running_prefill_prioritized_over_swap():
457 scheduler.block_manager.can_swap_in = MagicMock()
458 scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
459
416 - _, seq_group2 = create_dummy_prompt("2", prompt_length=60)
460 + _, seq_group2 = create_dummy_prompt("2",
461 + prompt_length=60,
462 + block_size=block_size)
463 scheduler.add_seq_group(seq_group2)
464 _, out = schedule_and_update_computed_tokens(scheduler)
465 assert len(out.scheduled_seq_groups) == 1
@@ -455,22 +501,27 @@ def test_running_prefill_prioritized_over_swap():
501 assert out.blocks_to_swap_out == []
502
503
458 -def test_chunked_prefill_preempt():
504 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
505 +def test_chunked_prefill_preempt(use_v2_block_manager: bool):
506 """Verify preempt works with chunked prefill requests"""
507 block_size = 4
508 max_seqs = 30
509 max_model_len = 200
510 max_num_batched_tokens = 30
464 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
465 - max_seqs,
466 - max_model_len,
467 - enable_chunked_prefill=True)
511 + scheduler_config = SchedulerConfig(
512 + max_num_batched_tokens,
513 + max_seqs,
514 + max_model_len,
515 + enable_chunked_prefill=True,
516 + use_v2_block_manager=use_v2_block_manager)
517 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
469 - cache_config.num_cpu_blocks = 8
470 - cache_config.num_gpu_blocks = 8
518 + cache_config.num_cpu_blocks = 16
519 + cache_config.num_gpu_blocks = 16
520 scheduler = Scheduler(scheduler_config, cache_config, None)
521
473 - _, seq_group = create_dummy_prompt("1", prompt_length=60)
522 + _, seq_group = create_dummy_prompt("1",
523 + prompt_length=60,
524 + block_size=block_size)
525 scheduler.add_seq_group(seq_group)
526 _, out = schedule_and_update_computed_tokens(scheduler)
527 # The request is chunked.
@@ -517,22 +568,27 @@ def test_chunked_prefill_preempt():
568 assert out.num_batched_tokens == max_num_batched_tokens
569
570
520 -def test_chunked_prefill_max_seqs():
571 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
572 +def test_chunked_prefill_max_seqs(use_v2_block_manager: bool):
573 block_size = 4
574 max_seqs = 2
575 max_model_len = 80
576 max_num_batched_tokens = 64
525 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
526 - max_seqs,
527 - max_model_len,
528 - enable_chunked_prefill=True)
577 + scheduler_config = SchedulerConfig(
578 + max_num_batched_tokens,
579 + max_seqs,
580 + max_model_len,
581 + enable_chunked_prefill=True,
582 + use_v2_block_manager=use_v2_block_manager)
583 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
530 - cache_config.num_cpu_blocks = 8
531 - cache_config.num_gpu_blocks = 8
584 + cache_config.num_cpu_blocks = 128
585 + cache_config.num_gpu_blocks = 128
586 scheduler = Scheduler(scheduler_config, cache_config, None)
587 running: List[SequenceGroup] = []
588
535 - _, seq_group = create_dummy_prompt("1", prompt_length=65)
589 + _, seq_group = create_dummy_prompt("1",
590 + prompt_length=65,
591 + block_size=block_size)
592 scheduler.add_seq_group(seq_group)
593 running.append(seq_group)
594 # The first prefill is chunked.
@@ -542,7 +598,9 @@ def test_chunked_prefill_max_seqs():
598
599 # Add new requests.
600 for i in range(4):
545 - _, seq_group = create_dummy_prompt(str(i), prompt_length=65)
601 + _, seq_group = create_dummy_prompt(str(i),
602 + prompt_length=65,
603 + block_size=block_size)
604 scheduler.add_seq_group(seq_group)
605 running.append(seq_group)
606
@@ -564,16 +622,19 @@ def test_chunked_prefill_max_seqs():
622 assert not running[1].is_prefill()
623
624
567 -def test_perfix_caching():
625 +@pytest.mark.parametrize('use_v2_block_manager', [True, False])
626 +def test_perfix_caching(use_v2_block_manager: bool):
627 """Verify allocating full blocks when prefix caching is enabled."""
628 block_size = 4
629 max_seqs = 10
630 max_model_len = 80
631 max_num_batched_tokens = 64
573 - scheduler_config = SchedulerConfig(max_num_batched_tokens,
574 - max_seqs,
575 - max_model_len,
576 - enable_chunked_prefill=True)
632 + scheduler_config = SchedulerConfig(
633 + max_num_batched_tokens,
634 + max_seqs,
635 + max_model_len,
636 + enable_chunked_prefill=True,
637 + use_v2_block_manager=use_v2_block_manager)
638 cache_config = CacheConfig(block_size,
639 1.0,
640 1,