@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e3580537a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Performance] Enable chunked prefill and prefix caching together (#7753)

Cody Yu committed Aug 28, 2024 at 00:36 UTC e3580537a41a46b0f3cd750b86b633c1857a8c90
9 files changed +225 -27
tests/basic_correctness/test_chunked_prefill.py
+66
@@ -6,6 +6,7 @@ prefill requests are chunked.
6
7 Run `pytest tests/models/test_chunked_prefill.py`.
8 """
9 +from contextlib import nullcontext
10
11 import pytest
12
@@ -156,3 +157,68 @@ def test_models_with_fp8_kv_cache(
157 name_0="no_chunked_prefill",
158 name_1="chunked_prefill",
159 )
160 +
161 +
162 +@pytest.mark.parametrize("max_tokens", [16])
163 +@pytest.mark.parametrize("enforce_eager", [False])
164 +@pytest.mark.parametrize("chunk_size", [30, 32])
165 +@pytest.mark.parametrize("use_v2_block_manager", [False, True])
166 +# NOTE: Increasing this in this suite will fail CI because we currently cannot
167 +# reset distributed env properly. Use a value > 1 just when you test.
168 +@pytest.mark.parametrize("tensor_parallel_size", [1])
169 +def test_with_prefix_caching(
170 + vllm_runner,
171 + max_tokens: int,
172 + enforce_eager: bool,
173 + chunk_size: int,
174 + use_v2_block_manager: bool,
175 + tensor_parallel_size: int,
176 +) -> None:
177 + """
178 + Checks exact match decode with and without prefix caching
179 + with chunked prefill enabled.
180 + """
181 + model = "meta-llama/Llama-2-7b-chat-hf"
182 + # The common prompt has 142 tokens with Llama-2 tokenizer.
183 + common_prompt = "You are a helpful AI assistant " * 20
184 + unique_prompts = [
185 + "Question", # Warmup
186 + "Question", # Fully cached
187 + "Another question", # Partial cached
188 + ]
189 + full_prompts = [f"{common_prompt}\n{p}" for p in unique_prompts]
190 +
191 + max_num_batched_tokens = max_num_seqs = chunk_size
192 + outputs = {} # type: ignore
193 + check_result = True
194 + for enable in (True, False):
195 + with vllm_runner(
196 + model,
197 + dtype="half",
198 + max_num_batched_tokens=max_num_batched_tokens,
199 + enable_chunked_prefill=True,
200 + enable_prefix_caching=enable,
201 + tensor_parallel_size=tensor_parallel_size,
202 + use_v2_block_manager=use_v2_block_manager,
203 + enforce_eager=enforce_eager,
204 + max_num_seqs=max_num_seqs,
205 + ) as vllm_model:
206 + # It should fail when prefix caching is enable and chunk
207 + # size is not a multiple of block size (16).
208 + should_fail = chunk_size % 16 != 0 and enable
209 + check_result &= not should_fail
210 + outputs[enable] = []
211 + # Send the request one-by-one to ensure the cache is populated.
212 + with pytest.raises(ValueError) if should_fail else nullcontext():
213 + for prompt in full_prompts:
214 + outputs[enable] += vllm_model.generate_greedy([prompt],
215 + max_tokens)
216 +
217 + # Check results only if we did not expect a failure.
218 + if check_result:
219 + check_outputs_equal(
220 + outputs_0_lst=outputs[False],
221 + outputs_1_lst=outputs[True],
222 + name_0="w/o prefix caching",
223 + name_1="with prefix caching",
224 + )
tests/core/test_block_manager.py
+40
@@ -595,3 +595,43 @@ def test_sliding_window_multi_seq():
595
596 # assert all blocks are free now
597 assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
598 +
599 +
600 +def test_mark_blocks_as_computed_with_prefix_cache_and_chunked_prefill():
601 + """When prefix cache and chunked prefill are enabled, the block manager
602 + should only mark a chunk of blocks as computed instead of all blocks.
603 + """
604 +
605 + block_size = 4
606 + num_cpu_blocks = 0
607 + num_gpu_blocks = 16
608 + block_manager = BlockSpaceManagerV1(block_size,
609 + num_gpu_blocks,
610 + num_cpu_blocks,
611 + watermark=0,
612 + enable_caching=True)
613 +
614 + # Set prompt size to have num_gpu_blocks - 1 full blocks.
615 + prompt_length = block_size * num_gpu_blocks - 1
616 +
617 + # Allocate (reserve) all blocks.
618 + _, seq_group = create_dummy_prompt("0",
619 + prompt_length,
620 + block_size=block_size)
621 + block_manager.allocate(seq_group)
622 + assert seq_group.seqs[0].n_blocks == num_gpu_blocks
623 +
624 + # 1st chunk: Compute 2 and half blocks. Should mark 2 blocks as computed.
625 + token_chunk_size = int(block_size * 2.5)
626 + block_manager.mark_blocks_as_computed(seq_group, token_chunk_size)
627 + computed_blocks = block_manager.get_all_computed_blocks(seq_group.seqs[0])
628 + assert len(computed_blocks) == 2
629 +
630 + # Actual computed tokens.
631 + seq_group.seqs[0].data.update_num_computed_tokens(token_chunk_size)
632 +
633 + # 2nd chunk: Complete 3rd block and additional 4 blocks.
634 + token_chunk_size = int(block_size * 4.5)
635 + block_manager.mark_blocks_as_computed(seq_group, token_chunk_size)
636 + computed_blocks = block_manager.get_all_computed_blocks(seq_group.seqs[0])
637 + assert len(computed_blocks) == 7
tests/core/test_chunked_prefill_scheduler.py
+39
@@ -562,3 +562,42 @@ def test_chunked_prefill_max_seqs():
562 assert len(get_sequence_groups(out)) == max_seqs
563 assert not running[0].is_prefill()
564 assert not running[1].is_prefill()
565 +
566 +
567 +def test_perfix_caching():
568 + """Verify allocating full blocks when prefix caching is enabled."""
569 + block_size = 4
570 + max_seqs = 10
571 + max_model_len = 80
572 + max_num_batched_tokens = 64
573 + scheduler_config = SchedulerConfig(max_num_batched_tokens,
574 + max_seqs,
575 + max_model_len,
576 + enable_chunked_prefill=True)
577 + cache_config = CacheConfig(block_size,
578 + 1.0,
579 + 1,
580 + "auto",
581 + enable_prefix_caching=True)
582 + cache_config.num_cpu_blocks = 0
583 + cache_config.num_gpu_blocks = 32
584 + scheduler = Scheduler(scheduler_config, cache_config, None)
585 + running: List[SequenceGroup] = []
586 +
587 + # Add seq groups to scheduler.
588 + for i in range(2):
589 + _, seq_group = create_dummy_prompt(str(i),
590 + block_size=block_size,
591 + prompt_length=50)
592 + scheduler.add_seq_group(seq_group)
593 + running.append(seq_group)
594 +
595 + seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
596 + assert set(get_sequence_groups(out)) == set(running)
597 + assert seq_group_meta[0].token_chunk_size == 50
598 + # Verify it is chunked. Note that although the budget is 64-50=14,
599 + # we only allocate full blocks for prefix caching, so only 4*(14//4)=12
600 + # tokens are allocated.
601 + assert seq_group_meta[1].token_chunk_size == 12
602 + assert out.num_prefill_groups == 2
603 + assert out.num_batched_tokens == 62
vllm/core/block_manager_v1.py
+13 -6
@@ -681,14 +681,20 @@ class BlockSpaceManagerV1(BlockSpaceManager):
681 for block in block_table:
682 block.last_accessed = access_time
683
684 - def compute_full_blocks_in_seq(self, seq: Sequence):
684 + def compute_full_blocks_in_seq(self, seq: Sequence, token_chunk_size: int):
685 if seq.seq_id not in self.block_tables:
686 return
687 - max_full_block = seq.get_len() // self.block_size - 1
687 +
688 + # When chunked prefill is enabled, the computed full blocks
689 + # should be calculated based on the number of computed tokens.
690 + max_computed_tokens = (seq.data.get_num_computed_tokens() +
691 + token_chunk_size)
692 + computed_full_blocks = max_computed_tokens // self.block_size
693 +
694 block_table = self.block_tables[seq.seq_id]
689 - if max_full_block == -1:
695 + if computed_full_blocks == 0:
696 return
691 - for i in reversed(range(max_full_block)):
697 + for i in reversed(range(computed_full_blocks)):
698 if block_table[i].computed:
699 break
700 block_table[i].computed = True
@@ -718,10 +724,11 @@ class BlockSpaceManagerV1(BlockSpaceManager):
724 ids_list = [self.get_all_computed_blocks(seq) for seq in seqs]
725 return commonprefix([ids for ids in ids_list if ids != []])
726
721 - def mark_blocks_as_computed(self, seq_group: SequenceGroup):
727 + def mark_blocks_as_computed(self, seq_group: SequenceGroup,
728 + token_chunk_size: int):
729 if self.enable_caching:
730 for seq in seq_group.get_seqs():
724 - self.compute_full_blocks_in_seq(seq)
731 + self.compute_full_blocks_in_seq(seq, token_chunk_size)
732
733 def get_prefix_cache_hit_rate(self, device: Device) -> float:
734 if device == Device.GPU:
vllm/core/block_manager_v2.py
+2 -1
@@ -290,7 +290,8 @@ class BlockSpaceManagerV2(BlockSpaceManager):
290 self._last_access_blocks_tracker.update_last_access(
291 seq.seq_id, now)
292
293 - def mark_blocks_as_computed(self, seq_group: SequenceGroup):
293 + def mark_blocks_as_computed(self, seq_group: SequenceGroup,
294 + token_chunk_size: int):
295 # If prefix caching is enabled, mark immutable blocks as computed
296 # right after they have been scheduled (for prefill). This assumes
297 # the scheduler is synchronous so blocks are actually computed when
vllm/core/embedding_model_block_manager.py
+2 -1
@@ -80,7 +80,8 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
80 seq_group: List[Sequence]) -> List[int]:
81 return []
82
83 - def mark_blocks_as_computed(self, seq_group: SequenceGroup):
83 + def mark_blocks_as_computed(self, seq_group: SequenceGroup,
84 + token_chunk_size: int):
85 pass
86
87 def get_prefix_cache_hit_rate(self, device: Device) -> float:
vllm/core/interfaces.py
+2 -1
@@ -115,7 +115,8 @@ class BlockSpaceManager(ABC):
115 pass
116
117 @abstractmethod
118 - def mark_blocks_as_computed(self, seq_group: SequenceGroup):
118 + def mark_blocks_as_computed(self, seq_group: SequenceGroup,
119 + token_chunk_size: int):
120 pass
121
122 @abstractmethod
vllm/core/scheduler.py
+24 -6
@@ -1226,7 +1226,8 @@ class Scheduler:
1226 # will crash the vLLM instance / will not retry.
1227 for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
1228 self.block_manager.mark_blocks_as_computed(
1229 - scheduled_seq_group.seq_group)
1229 + scheduled_seq_group.seq_group,
1230 + scheduled_seq_group.token_chunk_size)
1231
1232 self._seq_group_metadata_cache[self.next_cache_id].reset()
1233
@@ -1457,10 +1458,27 @@ class Scheduler:
1458 for seq in seqs:
1459 num_new_tokens += seq.get_num_new_tokens()
1460 assert num_new_tokens > 0
1460 - # Chunk if a running request cannot fit in.
1461 - # If number of seq > 1, it means it is doing beam search in a
1462 - # decode phase. Do not chunk in that case.
1461 + # Chunk if a running request cannot fit in the given budget.
1462 + # If number of seq > 1, it means it is doing beam search
1463 + # in a decode phase. Do not chunk.
1464 if enable_chunking and len(seqs) == 1:
1464 - num_new_tokens = min(num_new_tokens,
1465 - budget.remaining_token_budget())
1465 + remaining_token_budget = budget.remaining_token_budget()
1466 + if self.cache_config.enable_prefix_caching:
1467 + # When prefix caching is enabled, we always allocate
1468 + # the number of new tokens that is dividable by the block size
1469 + # to avoid partial block matching.
1470 + block_size = self.cache_config.block_size
1471 + reminder = budget.token_budget % block_size
1472 + if reminder != 0:
1473 + raise ValueError("When enabling chunked prefill and "
1474 + "prefix caching, max_num_batched_tokens "
1475 + "(chunk size) must be dividable by "
1476 + "block size, but got chunk_size "
1477 + f"({budget.token_budget}) % block_size "
1478 + f"({block_size}) = {reminder}")
1479 + if remaining_token_budget < num_new_tokens:
1480 + num_new_tokens = (remaining_token_budget //
1481 + block_size) * block_size
1482 + else:
1483 + num_new_tokens = min(num_new_tokens, remaining_token_budget)
1484 return num_new_tokens
vllm/worker/model_runner.py
+37 -12
@@ -501,23 +501,48 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
501 and self.sliding_window is None
502 and inter_data.is_prompt)
503 inter_data.prefix_cache_hit = prefix_cache_hit
504 - if self.chunked_prefill_enabled and prefix_cache_hit:
505 - raise RuntimeError(
506 - "chunked prefill cannot be used with prefix caching now.")
507 -
508 - # If prefix cache is hit, advance context length to bypass
509 - # hit blocks. Accordingly, input tokens, position and query length
510 - # have to be updated.
511 - if prefix_cache_hit:
512 - assert computed_block_nums is not None
513 - context_len = len(computed_block_nums) * self.block_size
504 +
505 + if not prefix_cache_hit:
506 + return
507 +
508 + assert computed_block_nums is not None
509 + # The cache hit prompt tokens in this sequence. Note that
510 + # this may be larger than the sequence length if chunked
511 + # prefill is enabled.
512 + prefix_cache_len = len(computed_block_nums) * self.block_size
513 + # The number of so far computed prompt tokens in this sequence.
514 + context_len = inter_data.context_lens[seq_idx]
515 + # The total number of prompt tokens in this sequence.
516 + # When chunked prefill is enabled, this is the token number of
517 + # computed chunks + current chunk.
518 + seq_len = inter_data.seq_lens[seq_idx]
519 + if prefix_cache_len <= context_len:
520 + # We already passed the cache hit region,
521 + # so do normal computation.
522 + pass
523 + elif context_len < prefix_cache_len < seq_len:
524 + # Partial hit. Compute the missing part.
525 + uncomputed_start = prefix_cache_len - context_len
526 inter_data.input_tokens[seq_idx] = inter_data.input_tokens[
515 - seq_idx][context_len:]
527 + seq_idx][uncomputed_start:]
528 inter_data.input_positions[seq_idx] = inter_data.input_positions[
517 - seq_idx][context_len:]
529 + seq_idx][uncomputed_start:]
530 + context_len = prefix_cache_len
531 +
532 inter_data.context_lens[seq_idx] = context_len
533 inter_data.query_lens[
534 seq_idx] = inter_data.seq_lens[seq_idx] - context_len
535 + elif seq_len <= prefix_cache_len:
536 + # Full hit. Only compute the last token to avoid
537 + # erroneous behavior. FIXME: Ideally we should directly
538 + # mark all tokens as computed in the scheduler and do not
539 + # schedule this sequence, so this case should not happen.
540 + inter_data.input_tokens[seq_idx] = inter_data.input_tokens[
541 + seq_idx][-1:]
542 + inter_data.input_positions[seq_idx] = inter_data.input_positions[
543 + seq_idx][-1:]
544 + inter_data.query_lens[seq_idx] = 1
545 + inter_data.context_lens[seq_idx] = inter_data.seq_lens[seq_idx] - 1
546
547 def _compute_for_sliding_window(self, inter_data: InterDataForSeqGroup,
548 seq_idx: int,