A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Test] Test multiple attn backend for chunked prefill. (#4023)
SangBin Cho committed
Apr 13, 2024 at 01:56 UTC
36729bac1303b655b816b77f45b17237bfafd692
4 files changed
+13
-23
.buildkite/test-pipeline.yaml
+7
-1
@@ -12,7 +12,13 @@ steps:
12
command: pytest -v -s async_engine
13
14
- label: Basic Correctness Test
15
- command: pytest -v -s basic_correctness
15
+ commands:
16
+ - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
17
+ - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
18
+ - VLLM_ATTENTION_BACKEND=ROCM_FLASH pytest -v -s basic_correctness/test_basic_correctness.py
19
+ - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
20
+ - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
21
+ - VLLM_ATTENTION_BACKEND=ROCM_FLASH pytest -v -s basic_correctness/test_chunked_prefill.py
22
23
- label: Core Test
24
command: pytest -v -s core
tests/basic_correctness/test_basic_correctness.py
-6
@@ -4,8 +4,6 @@ Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4
"""
5
import pytest
6
7
-from vllm.attention.selector import VLLM_ATTENTION_BACKEND
8
-
7
MODELS = [
8
"facebook/opt-125m",
9
"meta-llama/Llama-2-7b-hf",
@@ -16,7 +14,6 @@ MODELS = [
14
@pytest.mark.parametrize("dtype", ["half"])
15
@pytest.mark.parametrize("max_tokens", [5])
16
@pytest.mark.parametrize("enforce_eager", [False, True])
19
-@pytest.mark.parametrize("attn_backend", ["XFORMERS", "FLASH_ATTN"])
17
def test_models(
18
hf_runner,
19
vllm_runner,
@@ -25,10 +22,7 @@ def test_models(
22
dtype: str,
23
max_tokens: int,
24
enforce_eager: bool,
28
- attn_backend: str,
29
- monkeypatch,
25
) -> None:
31
- monkeypatch.setenv(VLLM_ATTENTION_BACKEND, attn_backend)
26
hf_model = hf_runner(model, dtype=dtype)
27
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
28
del hf_model
tests/basic_correctness/test_chunked_prefill.py
-4
@@ -33,10 +33,6 @@ def test_models(
33
enforce_eager: bool,
34
tensor_parallel_size: int,
35
) -> None:
36
- if (tensor_parallel_size == 2 and chunked_prefill_token_size != 16
37
- and not enforce_eager):
38
- pytest.skip(f"Skip {chunked_prefill_token_size=} and {enforce_eager=} "
39
- "for high TP to save testing time.")
36
max_num_seqs = min(chunked_prefill_token_size, 256)
37
enable_chunked_prefill = False
38
max_num_batched_tokens = None
vllm/attention/backends/rocm_flash_attn.py
+6
-12
@@ -162,7 +162,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
162
# AMD Radeon 7900 series (gfx1100) currently does not support
163
# xFormers nor FlashAttention. As a temporary workaround, we use
164
# naive PyTorch implementation of attention.
165
- self.attn_fuc = _naive_attention()
165
+ self.attn_fuc = _naive_attention
166
logger.debug("Using naive attention in ROCmBackend")
167
elif self.use_triton_flash_attn:
168
from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
@@ -334,26 +334,21 @@ def _naive_attention(
334
prompt_lens: List[int],
335
scale: float,
336
) -> torch.Tensor:
337
- num_tokens = query.shape[0]
337
output = torch.empty_like(query)
338
start = 0
339
for _, prompt_len in enumerate(prompt_lens):
340
end = start + prompt_len
341
out = _naive_masked_attention(
343
- query[None, start:end],
344
- key[None, start:end],
345
- value[None, start:end],
342
+ query[start:end],
343
+ key[start:end],
344
+ value[start:end],
345
scale,
346
)
347
# TODO(woosuk): Unnecessary copy. Optimize.
348
output[start:end].copy_(out)
349
start += prompt_len
350
352
- # Using view got RuntimeError: view size is not compatible
353
- # with input tensor's size and stride (at least one
354
- # dimension spans across two contiguous subspaces).
355
- # Use reshape instead.
356
- return output.reshape(num_tokens, -1)
351
+ return output
352
353
354
def _naive_masked_attention(
@@ -362,14 +357,13 @@ def _naive_masked_attention(
357
value: torch.Tensor,
358
scale: float,
359
) -> torch.Tensor:
365
- seq_len, _, _ = query.shape
360
+ seq_len, head_size, head_dim = query.shape
361
attn_mask = torch.triu(torch.ones(seq_len,
362
seq_len,
363
dtype=query.dtype,
364
device=query.device),
365
diagonal=1)
366
attn_mask = attn_mask * torch.finfo(query.dtype).min
372
-
367
attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
368
attn_weights = attn_weights + attn_mask.float()
369
attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)