A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
Fix tests in test_chunked_prefill_scheduler which fail with BlockManager V2 (#8752)
sroy745 committed
Sep 24, 2024 at 21:26 UTC
fc3afc20df410dd523f94967b98836084f561ab7
1 file changed
+143
-82
tests/core/test_chunked_prefill_scheduler.py
+143
-82
@@ -27,16 +27,19 @@ def schedule_and_update_computed_tokens(scheduler):
27
return metas, out
28
29
30
-def test_simple():
30
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
31
+def test_simple(use_v2_block_manager: bool):
32
"""Verify basic scheduling works."""
33
block_size = 4
34
num_seq_group = 4
35
max_model_len = 16
36
max_num_batched_tokens = 64
36
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
37
- num_seq_group,
38
- max_model_len,
39
- enable_chunked_prefill=True)
37
+ scheduler_config = SchedulerConfig(
38
+ max_num_batched_tokens,
39
+ num_seq_group,
40
+ max_model_len,
41
+ enable_chunked_prefill=True,
42
+ use_v2_block_manager=use_v2_block_manager)
43
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
44
cache_config.num_cpu_blocks = 8
45
cache_config.num_gpu_blocks = 8
@@ -45,7 +48,9 @@ def test_simple():
48
49
# Add seq groups to scheduler.
50
for i in range(num_seq_group):
48
- _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
51
+ _, seq_group = create_dummy_prompt(str(i),
52
+ prompt_length=block_size,
53
+ block_size=block_size)
54
scheduler.add_seq_group(seq_group)
55
running.append(seq_group)
56
@@ -69,30 +74,36 @@ def test_simple():
74
assert len(seq_group_meta) == num_seq_group
75
76
72
-def test_chunk():
77
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
78
+def test_chunk(use_v2_block_manager: bool):
79
"""Verify prefills are chunked properly."""
80
block_size = 4
81
max_seqs = 60
82
max_model_len = 80
83
max_num_batched_tokens = 64
78
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
79
- max_seqs,
80
- max_model_len,
81
- enable_chunked_prefill=True)
84
+ scheduler_config = SchedulerConfig(
85
+ max_num_batched_tokens,
86
+ max_seqs,
87
+ max_model_len,
88
+ enable_chunked_prefill=True,
89
+ use_v2_block_manager=use_v2_block_manager)
90
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
83
- cache_config.num_cpu_blocks = 8
84
- cache_config.num_gpu_blocks = 8
91
+ cache_config.num_cpu_blocks = 32
92
+ cache_config.num_gpu_blocks = 32
93
scheduler = Scheduler(scheduler_config, cache_config, None)
94
running: List[SequenceGroup] = []
95
96
# Add seq groups to scheduler.
97
for i in range(2):
90
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
98
+ _, seq_group = create_dummy_prompt(str(i),
99
+ prompt_length=60,
100
+ block_size=block_size)
101
scheduler.add_seq_group(seq_group)
102
running.append(seq_group)
103
104
# Verify the second request is chunked.
105
seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
106
+ print()
107
assert set(get_sequence_groups(out)) == set(running)
108
assert seq_group_meta[0].token_chunk_size == 60
109
# Verify it is chunked.
@@ -113,24 +124,29 @@ def test_chunk():
124
assert out.num_batched_tokens == 57
125
126
116
-def test_complex():
127
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
128
+def test_complex(use_v2_block_manager: bool):
129
block_size = 4
130
max_seqs = 60
131
max_model_len = 80
132
max_num_batched_tokens = 64
121
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
122
- max_seqs,
123
- max_model_len,
124
- enable_chunked_prefill=True)
133
+ scheduler_config = SchedulerConfig(
134
+ max_num_batched_tokens,
135
+ max_seqs,
136
+ max_model_len,
137
+ enable_chunked_prefill=True,
138
+ use_v2_block_manager=use_v2_block_manager)
139
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
126
- cache_config.num_cpu_blocks = 8
127
- cache_config.num_gpu_blocks = 8
140
+ cache_config.num_cpu_blocks = 64
141
+ cache_config.num_gpu_blocks = 64
142
scheduler = Scheduler(scheduler_config, cache_config, None)
143
running: List[SequenceGroup] = []
144
145
# Add seq groups to scheduler.
146
for i in range(2):
133
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
147
+ _, seq_group = create_dummy_prompt(str(i),
148
+ prompt_length=60,
149
+ block_size=block_size)
150
scheduler.add_seq_group(seq_group)
151
running.append(seq_group)
152
assert seq_group.is_prefill()
@@ -151,7 +167,9 @@ def test_complex():
167
168
# Add 2 more requests.
169
for i in range(2, 4):
154
- _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
170
+ _, seq_group = create_dummy_prompt(str(i),
171
+ prompt_length=60,
172
+ block_size=block_size)
173
scheduler.add_seq_group(seq_group)
174
running.append(seq_group)
175
@@ -176,16 +194,19 @@ def test_complex():
194
assert running[2].is_prefill()
195
196
179
-def test_maximal_decoding():
197
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
198
+def test_maximal_decoding(use_v2_block_manager: bool):
199
"""Verify decoding requests are prioritized."""
200
block_size = 4
201
max_seqs = 2
202
max_model_len = 8
203
max_num_batched_tokens = 2
185
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
186
- max_seqs,
187
- max_model_len,
188
- enable_chunked_prefill=True)
204
+ scheduler_config = SchedulerConfig(
205
+ max_num_batched_tokens,
206
+ max_seqs,
207
+ max_model_len,
208
+ enable_chunked_prefill=True,
209
+ use_v2_block_manager=use_v2_block_manager)
210
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
211
cache_config.num_cpu_blocks = 8
212
cache_config.num_gpu_blocks = 8
@@ -194,7 +215,9 @@ def test_maximal_decoding():
215
216
# Add seq groups to scheduler.
217
for i in range(2):
197
- _, seq_group = create_dummy_prompt(str(i), prompt_length=2)
218
+ _, seq_group = create_dummy_prompt(str(i),
219
+ prompt_length=2,
220
+ block_size=block_size)
221
scheduler.add_seq_group(seq_group)
222
running.append(seq_group)
223
assert seq_group.is_prefill()
@@ -211,7 +234,9 @@ def test_maximal_decoding():
234
append_new_token(running[0], 1)
235
236
# Create one more seq_group.
214
- _, seq_group = create_dummy_prompt("3", prompt_length=2)
237
+ _, seq_group = create_dummy_prompt("3",
238
+ prompt_length=2,
239
+ block_size=block_size)
240
scheduler.add_seq_group(seq_group)
241
running.append(seq_group)
242
assert seq_group.is_prefill()
@@ -263,23 +288,28 @@ def test_maximal_decoding():
288
assert out.num_batched_tokens == 2
289
290
266
-def test_prompt_limit():
291
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
292
+def test_prompt_limit(use_v2_block_manager: bool):
293
"""Verify max_num_batched_tokens < max_model_len is possible."""
294
block_size = 4
295
max_seqs = 32
296
max_model_len = 64
297
max_num_batched_tokens = 32
272
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
273
- max_seqs,
274
- max_model_len,
275
- enable_chunked_prefill=True)
298
+ scheduler_config = SchedulerConfig(
299
+ max_num_batched_tokens,
300
+ max_seqs,
301
+ max_model_len,
302
+ enable_chunked_prefill=True,
303
+ use_v2_block_manager=use_v2_block_manager)
304
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
277
- cache_config.num_cpu_blocks = 8
278
- cache_config.num_gpu_blocks = 8
305
+ cache_config.num_cpu_blocks = 16
306
+ cache_config.num_gpu_blocks = 16
307
scheduler = Scheduler(scheduler_config, cache_config, None)
308
running: List[SequenceGroup] = []
309
282
- _, seq_group = create_dummy_prompt("1", prompt_length=48)
310
+ _, seq_group = create_dummy_prompt("1",
311
+ prompt_length=48,
312
+ block_size=block_size)
313
scheduler.add_seq_group(seq_group)
314
running.append(seq_group)
315
assert seq_group.is_prefill()
@@ -293,7 +323,8 @@ def test_prompt_limit():
323
assert out.num_batched_tokens == 32
324
325
296
-def test_prompt_limit_exceed():
326
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
327
+def test_prompt_limit_exceed(use_v2_block_manager: bool):
328
block_size = 4
329
max_seqs = 64
330
max_model_len = 32
@@ -303,12 +334,13 @@ def test_prompt_limit_exceed():
334
max_model_len,
335
enable_chunked_prefill=True)
336
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
306
- cache_config.num_cpu_blocks = 8
307
- cache_config.num_gpu_blocks = 8
337
+ cache_config.num_cpu_blocks = 16
338
+ cache_config.num_gpu_blocks = 16
339
scheduler = Scheduler(scheduler_config, cache_config, None)
340
running: List[SequenceGroup] = []
310
-
311
- _, seq_group = create_dummy_prompt("2", prompt_length=48)
341
+ _, seq_group = create_dummy_prompt("2",
342
+ prompt_length=48,
343
+ block_size=block_size)
344
scheduler.add_seq_group(seq_group)
345
running.append(seq_group)
346
assert seq_group.is_prefill()
@@ -317,22 +349,28 @@ def test_prompt_limit_exceed():
349
assert out.ignored_seq_groups[0] == seq_group
350
351
320
-def test_swap():
352
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
353
+def test_swap(use_v2_block_manager: bool):
354
"""Verify swapping works with chunked prefill requests"""
355
block_size = 4
356
max_seqs = 30
357
max_model_len = 200
358
max_num_batched_tokens = 30
326
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
327
- max_seqs,
328
- max_model_len,
329
- enable_chunked_prefill=True)
359
+ scheduler_config = SchedulerConfig(
360
+ max_num_batched_tokens,
361
+ max_seqs,
362
+ max_model_len,
363
+ enable_chunked_prefill=True,
364
+ use_v2_block_manager=use_v2_block_manager)
365
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
331
- cache_config.num_cpu_blocks = 8
332
- cache_config.num_gpu_blocks = 8
366
+ cache_config.num_cpu_blocks = 16
367
+ cache_config.num_gpu_blocks = 16
368
scheduler = Scheduler(scheduler_config, cache_config, None)
369
335
- _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
370
+ _, seq_group = create_dummy_prompt("1",
371
+ prompt_length=60,
372
+ best_of=2,
373
+ block_size=block_size)
374
scheduler.add_seq_group(seq_group)
375
_, out = schedule_and_update_computed_tokens(scheduler)
376
# The request is chunked.
@@ -369,21 +407,27 @@ def test_swap():
407
assert out.blocks_to_swap_out == []
408
409
372
-def test_running_prefill_prioritized_over_swap():
410
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
411
+def test_running_prefill_prioritized_over_swap(use_v2_block_manager: bool):
412
block_size = 4
413
max_seqs = 30
414
max_model_len = 200
415
max_num_batched_tokens = 30
377
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
378
- max_seqs,
379
- max_model_len,
380
- enable_chunked_prefill=True)
416
+ scheduler_config = SchedulerConfig(
417
+ max_num_batched_tokens,
418
+ max_seqs,
419
+ max_model_len,
420
+ enable_chunked_prefill=True,
421
+ use_v2_block_manager=use_v2_block_manager)
422
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
382
- cache_config.num_cpu_blocks = 8
383
- cache_config.num_gpu_blocks = 8
423
+ cache_config.num_cpu_blocks = 32
424
+ cache_config.num_gpu_blocks = 32
425
scheduler = Scheduler(scheduler_config, cache_config, None)
426
386
- _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
427
+ _, seq_group = create_dummy_prompt("1",
428
+ prompt_length=60,
429
+ best_of=2,
430
+ block_size=block_size)
431
scheduler.add_seq_group(seq_group)
432
_, out = schedule_and_update_computed_tokens(scheduler)
433
# The request is chunked.
@@ -413,7 +457,9 @@ def test_running_prefill_prioritized_over_swap():
457
scheduler.block_manager.can_swap_in = MagicMock()
458
scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
459
416
- _, seq_group2 = create_dummy_prompt("2", prompt_length=60)
460
+ _, seq_group2 = create_dummy_prompt("2",
461
+ prompt_length=60,
462
+ block_size=block_size)
463
scheduler.add_seq_group(seq_group2)
464
_, out = schedule_and_update_computed_tokens(scheduler)
465
assert len(out.scheduled_seq_groups) == 1
@@ -455,22 +501,27 @@ def test_running_prefill_prioritized_over_swap():
501
assert out.blocks_to_swap_out == []
502
503
458
-def test_chunked_prefill_preempt():
504
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
505
+def test_chunked_prefill_preempt(use_v2_block_manager: bool):
506
"""Verify preempt works with chunked prefill requests"""
507
block_size = 4
508
max_seqs = 30
509
max_model_len = 200
510
max_num_batched_tokens = 30
464
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
465
- max_seqs,
466
- max_model_len,
467
- enable_chunked_prefill=True)
511
+ scheduler_config = SchedulerConfig(
512
+ max_num_batched_tokens,
513
+ max_seqs,
514
+ max_model_len,
515
+ enable_chunked_prefill=True,
516
+ use_v2_block_manager=use_v2_block_manager)
517
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
469
- cache_config.num_cpu_blocks = 8
470
- cache_config.num_gpu_blocks = 8
518
+ cache_config.num_cpu_blocks = 16
519
+ cache_config.num_gpu_blocks = 16
520
scheduler = Scheduler(scheduler_config, cache_config, None)
521
473
- _, seq_group = create_dummy_prompt("1", prompt_length=60)
522
+ _, seq_group = create_dummy_prompt("1",
523
+ prompt_length=60,
524
+ block_size=block_size)
525
scheduler.add_seq_group(seq_group)
526
_, out = schedule_and_update_computed_tokens(scheduler)
527
# The request is chunked.
@@ -517,22 +568,27 @@ def test_chunked_prefill_preempt():
568
assert out.num_batched_tokens == max_num_batched_tokens
569
570
520
-def test_chunked_prefill_max_seqs():
571
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
572
+def test_chunked_prefill_max_seqs(use_v2_block_manager: bool):
573
block_size = 4
574
max_seqs = 2
575
max_model_len = 80
576
max_num_batched_tokens = 64
525
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
526
- max_seqs,
527
- max_model_len,
528
- enable_chunked_prefill=True)
577
+ scheduler_config = SchedulerConfig(
578
+ max_num_batched_tokens,
579
+ max_seqs,
580
+ max_model_len,
581
+ enable_chunked_prefill=True,
582
+ use_v2_block_manager=use_v2_block_manager)
583
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
530
- cache_config.num_cpu_blocks = 8
531
- cache_config.num_gpu_blocks = 8
584
+ cache_config.num_cpu_blocks = 128
585
+ cache_config.num_gpu_blocks = 128
586
scheduler = Scheduler(scheduler_config, cache_config, None)
587
running: List[SequenceGroup] = []
588
535
- _, seq_group = create_dummy_prompt("1", prompt_length=65)
589
+ _, seq_group = create_dummy_prompt("1",
590
+ prompt_length=65,
591
+ block_size=block_size)
592
scheduler.add_seq_group(seq_group)
593
running.append(seq_group)
594
# The first prefill is chunked.
@@ -542,7 +598,9 @@ def test_chunked_prefill_max_seqs():
598
599
# Add new requests.
600
for i in range(4):
545
- _, seq_group = create_dummy_prompt(str(i), prompt_length=65)
601
+ _, seq_group = create_dummy_prompt(str(i),
602
+ prompt_length=65,
603
+ block_size=block_size)
604
scheduler.add_seq_group(seq_group)
605
running.append(seq_group)
606
@@ -564,16 +622,19 @@ def test_chunked_prefill_max_seqs():
622
assert not running[1].is_prefill()
623
624
567
-def test_perfix_caching():
625
+@pytest.mark.parametrize('use_v2_block_manager', [True, False])
626
+def test_perfix_caching(use_v2_block_manager: bool):
627
"""Verify allocating full blocks when prefix caching is enabled."""
628
block_size = 4
629
max_seqs = 10
630
max_model_len = 80
631
max_num_batched_tokens = 64
573
- scheduler_config = SchedulerConfig(max_num_batched_tokens,
574
- max_seqs,
575
- max_model_len,
576
- enable_chunked_prefill=True)
632
+ scheduler_config = SchedulerConfig(
633
+ max_num_batched_tokens,
634
+ max_seqs,
635
+ max_model_len,
636
+ enable_chunked_prefill=True,
637
+ use_v2_block_manager=use_v2_block_manager)
638
cache_config = CacheConfig(block_size,
639
1.0,
640
1,