@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f42a006b15
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix]: During testing, use pytest monkeypatch for safely overriding the env var that indicates the vLLM backend (#5210)

afeldman-nm committed Jun 3, 2024 at 23:32 UTC f42a006b15e876a2a7cb24614ece780bde6d1e3d
2 files changed +32 -17
tests/kernels/test_attention_selector.py
+10 -17
@@ -1,21 +1,22 @@
1 -import os
1 from unittest.mock import patch
2
3 import pytest
4 import torch
5
6 +from tests.kernels.utils import (STR_FLASH_ATTN_VAL, STR_INVALID_VAL,
7 + override_backend_env_variable)
8 from vllm.attention.selector import which_attn_to_use
9
10
11 @pytest.mark.parametrize(
12 "name", ["TORCH_SDPA", "ROCM_FLASH", "XFORMERS", "FLASHINFER"])
13 @pytest.mark.parametrize("device", ["cpu", "hip", "cuda"])
13 -def test_env(name: str, device: str):
14 +def test_env(name: str, device: str, monkeypatch):
15 """Test that the attention selector can be set via environment variable.
16 Note that we do not test FlashAttn because it is the default backend.
17 """
17 - name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
18 - os.environ["VLLM_ATTENTION_BACKEND"] = name
18 +
19 + override_backend_env_variable(monkeypatch, name)
20
21 if device == "cpu":
22 with patch("vllm.attention.selector.is_cpu", return_value=True):
@@ -32,14 +33,11 @@ def test_env(name: str, device: str):
33 torch.float16, 16)
34 assert backend.name == name
35
35 - if name_backup is not None:
36 - os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
37 -
36
39 -def test_flash_attn():
37 +def test_flash_attn(monkeypatch):
38 """Test FlashAttn validation."""
41 - name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
42 - os.environ["VLLM_ATTENTION_BACKEND"] = "FLASH_ATTN"
39 +
40 + override_backend_env_variable(monkeypatch, STR_FLASH_ATTN_VAL)
41
42 # Unsupported CUDA arch
43 with patch("torch.cuda.get_device_capability", return_value=[7, 5]):
@@ -71,14 +69,9 @@ def test_flash_attn():
69 backend = which_attn_to_use(8, 17, 8, None, torch.float16, None, 16)
70 assert backend.name != "FLASH_ATTN"
71
74 - if name_backup is not None:
75 - os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
76 -
72
78 -def test_invalid_env():
73 +def test_invalid_env(monkeypatch):
74 """Throw an exception if the backend name is invalid."""
80 - name_backup = os.environ.get("VLLM_ATTENTION_BACKEND", None)
81 - os.environ["VLLM_ATTENTION_BACKEND"] = "INVALID"
75 + override_backend_env_variable(monkeypatch, STR_INVALID_VAL)
76 with pytest.raises(ValueError):
77 which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
84 - os.environ["VLLM_ATTENTION_BACKEND"] = name_backup
tests/kernels/utils.py new
+22
@@ -0,0 +1,22 @@
1 +"""Kernel test utils"""
2 +
3 +import pytest
4 +
5 +STR_BACKEND_ENV_VAR: str = "VLLM_ATTENTION_BACKEND"
6 +STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
7 +STR_INVALID_VAL: str = "INVALID"
8 +
9 +
10 +def override_backend_env_variable(mpatch: pytest.MonkeyPatch,
11 + backend_name: str) -> None:
12 + '''
13 + Override the environment variable indicating the vLLM backend temporarily,
14 + using pytest monkeypatch to ensure that the env vars get
15 + reset once the test context exits.
16 +
17 + Arguments:
18 +
19 + * mpatch: pytest monkeypatch instance
20 + * backend_name: attention backend name to force
21 + '''
22 + mpatch.setenv(STR_BACKEND_ENV_VAR, backend_name)