@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e42df7227d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Test] Add xformer and flash attn tests (#3961)

Co-authored-by: Simon Mo <simon.mo@hey.com>

SangBin Cho committed Apr 11, 2024 at 12:09 UTC e42df7227d18e2b96785f8ee52053663ade05b63
2 files changed +15
tests/basic_correctness/test_basic_correctness.py
+6
@@ -4,6 +4,8 @@ Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4 """
5 import pytest
6
7 +from vllm.attention.selector import VLLM_ATTENTION_BACKEND
8 +
9 MODELS = [
10 "facebook/opt-125m",
11 "meta-llama/Llama-2-7b-hf",
@@ -14,6 +16,7 @@ MODELS = [
16 @pytest.mark.parametrize("dtype", ["half"])
17 @pytest.mark.parametrize("max_tokens", [5])
18 @pytest.mark.parametrize("enforce_eager", [False, True])
19 +@pytest.mark.parametrize("attn_backend", ["XFORMERS", "FLASH_ATTN"])
20 def test_models(
21 hf_runner,
22 vllm_runner,
@@ -22,7 +25,10 @@ def test_models(
25 dtype: str,
26 max_tokens: int,
27 enforce_eager: bool,
28 + attn_backend: str,
29 + monkeypatch,
30 ) -> None:
31 + monkeypatch.setenv(VLLM_ATTENTION_BACKEND, attn_backend)
32 hf_model = hf_runner(model, dtype=dtype)
33 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
34 del hf_model
vllm/attention/selector.py
+9
@@ -1,4 +1,5 @@
1 import enum
2 +import os
3 from functools import lru_cache
4 from typing import Type
5
@@ -10,6 +11,8 @@ from vllm.utils import is_cpu, is_hip
11
12 logger = init_logger(__name__)
13
14 +VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
15 +
16
17 class _Backend(enum.Enum):
18 FLASH_ATTN = enum.auto()
@@ -75,4 +78,10 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
78 "Cannot use FlashAttention backend because the flash_attn package "
79 "is not found. Please install it for better performance.")
80 return _Backend.XFORMERS
81 +
82 + backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
83 + if backend_by_env_var is not None:
84 + return _Backend[backend_by_env_var]
85 +
86 + # Default case.
87 return _Backend.FLASH_ATTN