@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 36729bac13
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Test] Test multiple attn backend for chunked prefill. (#4023)

SangBin Cho committed Apr 13, 2024 at 01:56 UTC 36729bac1303b655b816b77f45b17237bfafd692
4 files changed +13 -23
.buildkite/test-pipeline.yaml
+7 -1
@@ -12,7 +12,13 @@ steps:
12 command: pytest -v -s async_engine
13
14 - label: Basic Correctness Test
15 - command: pytest -v -s basic_correctness
15 + commands:
16 + - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
17 + - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
18 + - VLLM_ATTENTION_BACKEND=ROCM_FLASH pytest -v -s basic_correctness/test_basic_correctness.py
19 + - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
20 + - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
21 + - VLLM_ATTENTION_BACKEND=ROCM_FLASH pytest -v -s basic_correctness/test_chunked_prefill.py
22
23 - label: Core Test
24 command: pytest -v -s core
tests/basic_correctness/test_basic_correctness.py
-6
@@ -4,8 +4,6 @@ Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4 """
5 import pytest
6
7 -from vllm.attention.selector import VLLM_ATTENTION_BACKEND
8 -
7 MODELS = [
8 "facebook/opt-125m",
9 "meta-llama/Llama-2-7b-hf",
@@ -16,7 +14,6 @@ MODELS = [
14 @pytest.mark.parametrize("dtype", ["half"])
15 @pytest.mark.parametrize("max_tokens", [5])
16 @pytest.mark.parametrize("enforce_eager", [False, True])
19 -@pytest.mark.parametrize("attn_backend", ["XFORMERS", "FLASH_ATTN"])
17 def test_models(
18 hf_runner,
19 vllm_runner,
@@ -25,10 +22,7 @@ def test_models(
22 dtype: str,
23 max_tokens: int,
24 enforce_eager: bool,
28 - attn_backend: str,
29 - monkeypatch,
25 ) -> None:
31 - monkeypatch.setenv(VLLM_ATTENTION_BACKEND, attn_backend)
26 hf_model = hf_runner(model, dtype=dtype)
27 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
28 del hf_model
tests/basic_correctness/test_chunked_prefill.py
-4
@@ -33,10 +33,6 @@ def test_models(
33 enforce_eager: bool,
34 tensor_parallel_size: int,
35 ) -> None:
36 - if (tensor_parallel_size == 2 and chunked_prefill_token_size != 16
37 - and not enforce_eager):
38 - pytest.skip(f"Skip {chunked_prefill_token_size=} and {enforce_eager=} "
39 - "for high TP to save testing time.")
36 max_num_seqs = min(chunked_prefill_token_size, 256)
37 enable_chunked_prefill = False
38 max_num_batched_tokens = None
vllm/attention/backends/rocm_flash_attn.py
+6 -12
@@ -162,7 +162,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
162 # AMD Radeon 7900 series (gfx1100) currently does not support
163 # xFormers nor FlashAttention. As a temporary workaround, we use
164 # naive PyTorch implementation of attention.
165 - self.attn_fuc = _naive_attention()
165 + self.attn_fuc = _naive_attention
166 logger.debug("Using naive attention in ROCmBackend")
167 elif self.use_triton_flash_attn:
168 from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
@@ -334,26 +334,21 @@ def _naive_attention(
334 prompt_lens: List[int],
335 scale: float,
336 ) -> torch.Tensor:
337 - num_tokens = query.shape[0]
337 output = torch.empty_like(query)
338 start = 0
339 for _, prompt_len in enumerate(prompt_lens):
340 end = start + prompt_len
341 out = _naive_masked_attention(
343 - query[None, start:end],
344 - key[None, start:end],
345 - value[None, start:end],
342 + query[start:end],
343 + key[start:end],
344 + value[start:end],
345 scale,
346 )
347 # TODO(woosuk): Unnecessary copy. Optimize.
348 output[start:end].copy_(out)
349 start += prompt_len
350
352 - # Using view got RuntimeError: view size is not compatible
353 - # with input tensor's size and stride (at least one
354 - # dimension spans across two contiguous subspaces).
355 - # Use reshape instead.
356 - return output.reshape(num_tokens, -1)
351 + return output
352
353
354 def _naive_masked_attention(
@@ -362,14 +357,13 @@ def _naive_masked_attention(
357 value: torch.Tensor,
358 scale: float,
359 ) -> torch.Tensor:
365 - seq_len, _, _ = query.shape
360 + seq_len, head_size, head_dim = query.shape
361 attn_mask = torch.triu(torch.ones(seq_len,
362 seq_len,
363 dtype=query.dtype,
364 device=query.device),
365 diagonal=1)
366 attn_mask = attn_mask * torch.finfo(query.dtype).min
372 -
367 attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
368 attn_weights = attn_weights + attn_mask.float()
369 attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)