A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix `illegal memory access` error with chunked prefill, prefix caching, block manager v2 and xformers enabled together (#9532)
Signed-off-by: sasha0552 <admin@sasha0552.org>
sasha0552 committed
Oct 31, 2024 at 18:46 UTC
55650c83a0c386526ed04912a0c60eccca202f3e
2 files changed
+34
-3
tests/prefix_caching/test_prefix_caching.py
+28
@@ -5,6 +5,7 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
5
import pytest
6
7
from tests.kernels.utils import override_backend_env_variable
8
+from vllm import SamplingParams, TokensPrompt
9
10
from ..models.utils import check_outputs_equal
11
@@ -12,6 +13,14 @@ MODELS = [
13
"facebook/opt-125m",
14
]
15
16
+UNSTABLE_PROMPT_SEQUENCE = [
17
+ ([0] * 588) + ([1] * 1332) + ([2] * 30) + ([3] * 1),
18
+ ([0] * 588) + ([1] * 1332) + ([4] * 3) + ([5] * 50),
19
+ ([0] * 588) + ([1] * 1332) + ([2] * 30) + ([6] * 95),
20
+ ([0] * 588) + ([1] * 1332) + ([4] * 3) + ([7] * 174),
21
+ ([0] * 588) + ([8] * 1539),
22
+]
23
+
24
25
@pytest.mark.parametrize("model", MODELS)
26
@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
@@ -57,3 +66,22 @@ def test_mixed_requests(
66
name_0="hf",
67
name_1="vllm",
68
)
69
+
70
+
71
+@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
72
+def test_unstable_prompt_sequence(
73
+ vllm_runner,
74
+ backend: str,
75
+ monkeypatch,
76
+) -> None:
77
+ override_backend_env_variable(monkeypatch, backend)
78
+
79
+ with vllm_runner(
80
+ "Qwen/Qwen2.5-0.5B-Instruct",
81
+ enable_chunked_prefill=True,
82
+ enable_prefix_caching=True,
83
+ max_model_len=4096,
84
+ ) as vllm_model:
85
+ for prompt in UNSTABLE_PROMPT_SEQUENCE:
86
+ vllm_model.generate(TokensPrompt(prompt_token_ids=prompt),
87
+ SamplingParams(max_tokens=1))
vllm/attention/backends/utils.py
+6
-3
@@ -138,7 +138,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
138
chunked_prefill_enabled: bool):
139
is_prompt = inter_data.is_prompt
140
block_tables = inter_data.block_tables
141
- computed_block_nums = inter_data.computed_block_nums
141
142
for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
143
curr_sliding_window_block) in zip(
@@ -164,10 +163,14 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
163
# NOTE: This only works for oooooooxxx style attention.
164
block_table = []
165
if inter_data.prefix_cache_hit:
167
- block_table = computed_block_nums
166
+ block_table = block_tables[seq_id]
167
elif ((chunked_prefill_enabled or not is_prompt)
168
and block_tables is not None):
170
- block_table = block_tables[seq_id][-curr_sliding_window_block:]
169
+ if curr_sliding_window_block == 0:
170
+ block_table = block_tables[seq_id]
171
+ else:
172
+ block_table = block_tables[seq_id][
173
+ -curr_sliding_window_block:]
174
self.block_tables.append(block_table)
175
176
# Compute slot mapping.