@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fb2c1c86c1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix block table for seqs that have prefix cache hits (#7018)

Zach Zheng committed Aug 2, 2024 at 22:38 UTC fb2c1c86c196aa1531435d0c445fbea4c9dd4aa5
2 files changed +65 -3
tests/prefix_caching/test_prefix_caching.py
+56
@@ -6,10 +6,17 @@ from typing import List
6
7 import pytest
8
9 +from tests.kernels.utils import override_backend_env_variable
10 from vllm.block import PhysicalTokenBlock
11 from vllm.core.block_manager_v1 import CachedBlockAllocator
12 from vllm.utils import Device
13
14 +from ..models.utils import check_outputs_equal
15 +
16 +MODELS = [
17 + "facebook/opt-125m",
18 +]
19 +
20
21 @pytest.mark.parametrize("block_size", [16])
22 @pytest.mark.parametrize("num_blocks", [16])
@@ -76,3 +83,52 @@ def test_eviction(num_blocks: int, ):
83 assert (realloc_block != new_block)
84 assert (new_block.block_hash == new_block_hash)
85 assert (new_block.block_number == 2)
86 +
87 +
88 +@pytest.mark.parametrize("model", MODELS)
89 +@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
90 +@pytest.mark.parametrize("dtype", ["half"])
91 +@pytest.mark.parametrize("max_tokens", [5])
92 +@pytest.mark.parametrize("cached_position", [0, 1])
93 +@pytest.mark.parametrize("use_v2_block_manager", [False, True])
94 +def test_mixed_requests(
95 + hf_runner,
96 + vllm_runner,
97 + example_prompts,
98 + model: str,
99 + backend: str,
100 + dtype: str,
101 + max_tokens: int,
102 + cached_position: int,
103 + use_v2_block_manager: bool,
104 + monkeypatch,
105 +) -> None:
106 + """
107 + Test the case when some sequences have the prefix cache hit
108 + and the others don't. The cached position determines where
109 + the sequence is at among the batch of prefills.
110 + """
111 + override_backend_env_variable(monkeypatch, backend)
112 +
113 + with hf_runner(model, dtype=dtype) as hf_model:
114 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
115 +
116 + cached_prompt = example_prompts[cached_position]
117 + with vllm_runner(
118 + model,
119 + dtype=dtype,
120 + enable_prefix_caching=True,
121 + use_v2_block_manager=use_v2_block_manager,
122 + ) as vllm_model:
123 + # Run the first prompt so the cache is populated
124 + vllm_outputs = vllm_model.generate_greedy([cached_prompt], max_tokens)
125 +
126 + # Run all the promopts
127 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
128 +
129 + check_outputs_equal(
130 + outputs_0_lst=hf_outputs,
131 + outputs_1_lst=vllm_outputs,
132 + name_0="hf",
133 + name_1="vllm",
134 + )
vllm/attention/backends/flash_attn.py
+9 -3
@@ -209,6 +209,7 @@ class FlashAttentionMetadataBuilder(
209 self.num_prefills = 0
210 self.num_prefill_tokens = 0
211 self.num_decode_tokens = 0
212 + self.has_prefix_cache_hit = False
213
214 self.input_builder = input_builder
215 self.runner = input_builder.runner
@@ -219,7 +220,7 @@ class FlashAttentionMetadataBuilder(
220
221 def _add_seq_group(
222 self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
222 - chunked_prefill_enabled: bool):
223 + chunked_prefill_enabled: bool, prefix_cache_hit: bool):
224 """Add a sequence group to the metadata. Specifically update/append
225 1. context length.
226 2. block table.
@@ -252,7 +253,7 @@ class FlashAttentionMetadataBuilder(
253 # only allowing multiple of block_size chunk size.
254 # NOTE: This only works for oooooooxxx style attention.
255 block_table = []
255 - if inter_data.prefix_cache_hit:
256 + if prefix_cache_hit:
257 # NOTE(woosuk): For flash-attn, the block table should
258 # include the entries for the incoming prefill tokens.
259 block_table = block_tables[seq_id]
@@ -281,9 +282,14 @@ class FlashAttentionMetadataBuilder(
282 -1 if cuda graph is not used.
283 batch_size: The maybe padded batch size.
284 """
285 + prefix_cache_hit = any([
286 + inter_data.prefix_cache_hit
287 + for inter_data in self.input_builder.inter_data_list
288 + ])
289 for inter_data in self.input_builder.inter_data_list:
290 self._add_seq_group(inter_data,
286 - self.input_builder.chunked_prefill_enabled)
291 + self.input_builder.chunked_prefill_enabled,
292 + prefix_cache_hit)
293
294 device = self.runner.device
295 use_captured_graph = cuda_graph_pad_size != -1