A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix]: During testing, use pytest monkeypatch for safely overriding the env var that indicates the vLLM backend (#5210)
afeldman-nm committed
Jun 3, 2024 at 23:32 UTC
f42a006b15e876a2a7cb24614ece780bde6d1e3d
2 files changed
+32
-17
tests/kernels/test_attention_selector.py
+10
-17
@@ -1,21 +1,22 @@
1
-import os
1
from unittest.mock import patch
2
3
import pytest
4
import torch
5
6
+from tests.kernels.utils import (STR_FLASH_ATTN_VAL, STR_INVALID_VAL,
7
+ override_backend_env_variable)
8
from vllm.attention.selector import which_attn_to_use
9
10
11
@pytest.mark.parametrize(
12
"name", ["TORCH_SDPA", "ROCM_FLASH", "XFORMERS", "FLASHINFER"])
13
@pytest.mark.parametrize("device", ["cpu", "hip", "cuda"])
13
-def test_env(name: str, device: str):
14
+def test_env(name: str, device: str, monkeypatch):
15
"""Test that the attention selector can be set via environment variable.
16
Note that we do not test FlashAttn because it is the default backend.
17
"""
17
- name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
18
- os.environ["VLLM_ATTENTION_BACKEND"] = name
18
+
19
+ override_backend_env_variable(monkeypatch, name)
20
21
if device == "cpu":
22
with patch("vllm.attention.selector.is_cpu", return_value=True):
@@ -32,14 +33,11 @@ def test_env(name: str, device: str):
33
torch.float16, 16)
34
assert backend.name == name
35
35
- if name_backup is not None:
36
- os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
37
-
36
39
-def test_flash_attn():
37
+def test_flash_attn(monkeypatch):
38
"""Test FlashAttn validation."""
41
- name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
42
- os.environ["VLLM_ATTENTION_BACKEND"] = "FLASH_ATTN"
39
+
40
+ override_backend_env_variable(monkeypatch, STR_FLASH_ATTN_VAL)
41
42
# Unsupported CUDA arch
43
with patch("torch.cuda.get_device_capability", return_value=[7, 5]):
@@ -71,14 +69,9 @@ def test_flash_attn():
69
backend = which_attn_to_use(8, 17, 8, None, torch.float16, None, 16)
70
assert backend.name != "FLASH_ATTN"
71
74
- if name_backup is not None:
75
- os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
76
-
72
78
-def test_invalid_env():
73
+def test_invalid_env(monkeypatch):
74
"""Throw an exception if the backend name is invalid."""
80
- name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
81
- os.environ["VLLM_ATTENTION_BACKEND"] = "INVALID"
75
+ override_backend_env_variable(monkeypatch, STR_INVALID_VAL)
76
with pytest.raises(ValueError):
77
which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
84
- os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
tests/kernels/utils.py
new
+22
@@ -0,0 +1,22 @@
1
+"""Kernel test utils"""
2
+
3
+import pytest
4
+
5
+STR_BACKEND_ENV_VAR: str = "VLLM_ATTENTION_BACKEND"
6
+STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
7
+STR_INVALID_VAL: str = "INVALID"
8
+
9
+
10
+def override_backend_env_variable(mpatch: pytest.MonkeyPatch,
11
+ backend_name: str) -> None:
12
+ '''
13
+ Override the environment variable indicating the vLLM backend temporarily,
14
+ using pytest monkeypatch to ensure that the env vars get
15
+ reset once the test context exits.
16
+
17
+ Arguments:
18
+
19
+ * mpatch: pytest monkeypatch instance
20
+ * backend_name: attention backend name to force
21
+ '''
22
+ mpatch.setenv(STR_BACKEND_ENV_VAR, backend_name)