A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix block table for seqs that have prefix cache hits (#7018)
Zach Zheng committed
Aug 2, 2024 at 22:38 UTC
fb2c1c86c196aa1531435d0c445fbea4c9dd4aa5
2 files changed
+65
-3
tests/prefix_caching/test_prefix_caching.py
+56
@@ -6,10 +6,17 @@ from typing import List
6
7
import pytest
8
9
+from tests.kernels.utils import override_backend_env_variable
10
from vllm.block import PhysicalTokenBlock
11
from vllm.core.block_manager_v1 import CachedBlockAllocator
12
from vllm.utils import Device
13
14
+from ..models.utils import check_outputs_equal
15
+
16
+MODELS = [
17
+ "facebook/opt-125m",
18
+]
19
+
20
21
@pytest.mark.parametrize("block_size", [16])
22
@pytest.mark.parametrize("num_blocks", [16])
@@ -76,3 +83,52 @@ def test_eviction(num_blocks: int, ):
83
assert (realloc_block != new_block)
84
assert (new_block.block_hash == new_block_hash)
85
assert (new_block.block_number == 2)
86
+
87
+
88
+@pytest.mark.parametrize("model", MODELS)
89
+@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
90
+@pytest.mark.parametrize("dtype", ["half"])
91
+@pytest.mark.parametrize("max_tokens", [5])
92
+@pytest.mark.parametrize("cached_position", [0, 1])
93
+@pytest.mark.parametrize("use_v2_block_manager", [False, True])
94
+def test_mixed_requests(
95
+ hf_runner,
96
+ vllm_runner,
97
+ example_prompts,
98
+ model: str,
99
+ backend: str,
100
+ dtype: str,
101
+ max_tokens: int,
102
+ cached_position: int,
103
+ use_v2_block_manager: bool,
104
+ monkeypatch,
105
+) -> None:
106
+ """
107
+ Test the case when some sequences have the prefix cache hit
108
+ and the others don't. The cached position determines where
109
+ the sequence is at among the batch of prefills.
110
+ """
111
+ override_backend_env_variable(monkeypatch, backend)
112
+
113
+ with hf_runner(model, dtype=dtype) as hf_model:
114
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
115
+
116
+ cached_prompt = example_prompts[cached_position]
117
+ with vllm_runner(
118
+ model,
119
+ dtype=dtype,
120
+ enable_prefix_caching=True,
121
+ use_v2_block_manager=use_v2_block_manager,
122
+ ) as vllm_model:
123
+ # Run the first prompt so the cache is populated
124
+ vllm_outputs = vllm_model.generate_greedy([cached_prompt], max_tokens)
125
+
126
+ # Run all the promopts
127
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
128
+
129
+ check_outputs_equal(
130
+ outputs_0_lst=hf_outputs,
131
+ outputs_1_lst=vllm_outputs,
132
+ name_0="hf",
133
+ name_1="vllm",
134
+ )
vllm/attention/backends/flash_attn.py
+9
-3
@@ -209,6 +209,7 @@ class FlashAttentionMetadataBuilder(
209
self.num_prefills = 0
210
self.num_prefill_tokens = 0
211
self.num_decode_tokens = 0
212
+ self.has_prefix_cache_hit = False
213
214
self.input_builder = input_builder
215
self.runner = input_builder.runner
@@ -219,7 +220,7 @@ class FlashAttentionMetadataBuilder(
220
221
def _add_seq_group(
222
self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
222
- chunked_prefill_enabled: bool):
223
+ chunked_prefill_enabled: bool, prefix_cache_hit: bool):
224
"""Add a sequence group to the metadata. Specifically update/append
225
1. context length.
226
2. block table.
@@ -252,7 +253,7 @@ class FlashAttentionMetadataBuilder(
253
# only allowing multiple of block_size chunk size.
254
# NOTE: This only works for oooooooxxx style attention.
255
block_table = []
255
- if inter_data.prefix_cache_hit:
256
+ if prefix_cache_hit:
257
# NOTE(woosuk): For flash-attn, the block table should
258
# include the entries for the incoming prefill tokens.
259
block_table = block_tables[seq_id]
@@ -281,9 +282,14 @@ class FlashAttentionMetadataBuilder(
282
-1 if cuda graph is not used.
283
batch_size: The maybe padded batch size.
284
"""
285
+ prefix_cache_hit = any([
286
+ inter_data.prefix_cache_hit
287
+ for inter_data in self.input_builder.inter_data_list
288
+ ])
289
for inter_data in self.input_builder.inter_data_list:
290
self._add_seq_group(inter_data,
286
- self.input_builder.chunked_prefill_enabled)
291
+ self.input_builder.chunked_prefill_enabled,
292
+ prefix_cache_hit)
293
294
device = self.runner.device
295
use_captured_graph = cuda_graph_pad_size != -1