@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55650c83a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix `illegal memory access` error with chunked prefill, prefix caching, block manager v2 and xformers enabled together (#9532)

Signed-off-by: sasha0552 <admin@sasha0552.org>

sasha0552 committed Oct 31, 2024 at 18:46 UTC 55650c83a0c386526ed04912a0c60eccca202f3e
2 files changed +34 -3
tests/prefix_caching/test_prefix_caching.py
+28
@@ -5,6 +5,7 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
5 import pytest
6
7 from tests.kernels.utils import override_backend_env_variable
8 +from vllm import SamplingParams, TokensPrompt
9
10 from ..models.utils import check_outputs_equal
11
@@ -12,6 +13,14 @@ MODELS = [
13 "facebook/opt-125m",
14 ]
15
16 +UNSTABLE_PROMPT_SEQUENCE = [
17 + ([0] * 588) + ([1] * 1332) + ([2] * 30) + ([3] * 1),
18 + ([0] * 588) + ([1] * 1332) + ([4] * 3) + ([5] * 50),
19 + ([0] * 588) + ([1] * 1332) + ([2] * 30) + ([6] * 95),
20 + ([0] * 588) + ([1] * 1332) + ([4] * 3) + ([7] * 174),
21 + ([0] * 588) + ([8] * 1539),
22 +]
23 +
24
25 @pytest.mark.parametrize("model", MODELS)
26 @pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
@@ -57,3 +66,22 @@ def test_mixed_requests(
66 name_0="hf",
67 name_1="vllm",
68 )
69 +
70 +
71 +@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
72 +def test_unstable_prompt_sequence(
73 + vllm_runner,
74 + backend: str,
75 + monkeypatch,
76 +) -> None:
77 + override_backend_env_variable(monkeypatch, backend)
78 +
79 + with vllm_runner(
80 + "Qwen/Qwen2.5-0.5B-Instruct",
81 + enable_chunked_prefill=True,
82 + enable_prefix_caching=True,
83 + max_model_len=4096,
84 + ) as vllm_model:
85 + for prompt in UNSTABLE_PROMPT_SEQUENCE:
86 + vllm_model.generate(TokensPrompt(prompt_token_ids=prompt),
87 + SamplingParams(max_tokens=1))
vllm/attention/backends/utils.py
+6 -3
@@ -138,7 +138,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
138 chunked_prefill_enabled: bool):
139 is_prompt = inter_data.is_prompt
140 block_tables = inter_data.block_tables
141 - computed_block_nums = inter_data.computed_block_nums
141
142 for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
143 curr_sliding_window_block) in zip(
@@ -164,10 +163,14 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
163 # NOTE: This only works for oooooooxxx style attention.
164 block_table = []
165 if inter_data.prefix_cache_hit:
167 - block_table = computed_block_nums
166 + block_table = block_tables[seq_id]
167 elif ((chunked_prefill_enabled or not is_prompt)
168 and block_tables is not None):
170 - block_table = block_tables[seq_id][-curr_sliding_window_block:]
169 + if curr_sliding_window_block == 0:
170 + block_table = block_tables[seq_id]
171 + else:
172 + block_table = block_tables[seq_id][
173 + -curr_sliding_window_block:]
174 self.block_tables.append(block_table)
175
176 # Compute slot mapping.