A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Misc] Remove flashinfer warning, add flashinfer tests to CI (#6351)
Lily Liu committed
Jul 11, 2024 at 18:32 UTC
d6ab5289976fe219f943a7df4fb3a0ba1cb31a00
3 files changed
+10
-6
.buildkite/test-pipeline.yaml
+5
-3
@@ -19,8 +19,10 @@ steps:
19
- label: Basic Correctness Test
20
mirror_hardwares: [amd]
21
commands:
22
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
23
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
24
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
25
+ - VLLM_ATTENTION_BACKEND=FLASHINFER pytest -v -s basic_correctness/test_basic_correctness.py
26
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
27
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
28
- VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
@@ -122,14 +124,14 @@ steps:
124
- label: Kernels Test %N
125
#mirror_hardwares: [amd]
126
commands:
125
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.7/flashinfer-0.0.7+cu121torch2.3-cp310-cp310-linux_x86_64.whl
127
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
128
- pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
129
parallelism: 4
130
131
- label: Models Test
132
#mirror_hardwares: [amd]
133
commands:
132
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.7/flashinfer-0.0.7+cu121torch2.3-cp310-cp310-linux_x86_64.whl
134
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
135
- pytest -v -s models -m \"not vlm\"
136
137
- label: Vision Language Models Test
@@ -240,7 +242,7 @@ steps:
242
- pytest -v -s distributed/test_custom_all_reduce.py
243
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
244
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
243
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.7/flashinfer-0.0.7+cu121torch2.3-cp310-cp310-linux_x86_64.whl
245
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
246
- VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
247
- VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
248
- pytest -v -s -x lora/test_mixtral.py
tests/basic_correctness/test_basic_correctness.py
+5
@@ -2,11 +2,13 @@
2
3
Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4
"""
5
+import os
6
import weakref
7
8
import pytest
9
10
from vllm import LLM
11
+from vllm.utils import is_hip
12
13
from ..models.utils import check_outputs_equal
14
@@ -26,6 +28,9 @@ def test_vllm_gc_ed():
28
assert weak_llm() is None
29
30
31
+@pytest.mark.skipif(is_hip()
32
+ and os.getenv("VLLM_ATTENTION_BACKEND") == "FLASHINFER",
33
+ reason="Flashinfer does not support ROCm/HIP.")
34
@pytest.mark.parametrize("model", MODELS)
35
@pytest.mark.parametrize("dtype", ["half"])
36
@pytest.mark.parametrize("max_tokens", [5])
vllm/attention/selector.py
-3
@@ -77,9 +77,6 @@ def get_attn_backend(
77
return IpexAttnBackend
78
elif backend == _Backend.FLASHINFER:
79
logger.info("Using Flashinfer backend.")
80
- logger.warning(("Flashinfer will be stuck on llama-2-7b,"
81
- " please avoid using Flashinfer as the "
82
- "backend when running on llama-2-7b."))
80
from vllm.attention.backends.flashinfer import FlashInferBackend
81
return FlashInferBackend
82
elif backend == _Backend.PALLAS: