@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6ab528997
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Misc] Remove flashinfer warning, add flashinfer tests to CI (#6351)

Lily Liu committed Jul 11, 2024 at 18:32 UTC d6ab5289976fe219f943a7df4fb3a0ba1cb31a00
3 files changed +10 -6
.buildkite/test-pipeline.yaml
+5 -3
@@ -19,8 +19,10 @@ steps:
19 - label: Basic Correctness Test
20 mirror_hardwares: [amd]
21 commands:
22 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
23 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
24 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
25 + - VLLM_ATTENTION_BACKEND=FLASHINFER pytest -v -s basic_correctness/test_basic_correctness.py
26 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
27 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
28 - VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
@@ -122,14 +124,14 @@ steps:
124 - label: Kernels Test %N
125 #mirror_hardwares: [amd]
126 commands:
125 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.7/flashinfer-0.0.7+cu121torch2.3-cp310-cp310-linux_x86_64.whl
127 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
128 - pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
129 parallelism: 4
130
131 - label: Models Test
132 #mirror_hardwares: [amd]
133 commands:
132 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.7/flashinfer-0.0.7+cu121torch2.3-cp310-cp310-linux_x86_64.whl
134 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
135 - pytest -v -s models -m \"not vlm\"
136
137 - label: Vision Language Models Test
@@ -240,7 +242,7 @@ steps:
242 - pytest -v -s distributed/test_custom_all_reduce.py
243 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
244 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
243 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.7/flashinfer-0.0.7+cu121torch2.3-cp310-cp310-linux_x86_64.whl
245 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
246 - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
247 - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
248 - pytest -v -s -x lora/test_mixtral.py
tests/basic_correctness/test_basic_correctness.py
+5
@@ -2,11 +2,13 @@
2
3 Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4 """
5 +import os
6 import weakref
7
8 import pytest
9
10 from vllm import LLM
11 +from vllm.utils import is_hip
12
13 from ..models.utils import check_outputs_equal
14
@@ -26,6 +28,9 @@ def test_vllm_gc_ed():
28 assert weak_llm() is None
29
30
31 +@pytest.mark.skipif(is_hip()
32 + and os.getenv("VLLM_ATTENTION_BACKEND") == "FLASHINFER",
33 + reason="Flashinfer does not support ROCm/HIP.")
34 @pytest.mark.parametrize("model", MODELS)
35 @pytest.mark.parametrize("dtype", ["half"])
36 @pytest.mark.parametrize("max_tokens", [5])
vllm/attention/selector.py
-3
@@ -77,9 +77,6 @@ def get_attn_backend(
77 return IpexAttnBackend
78 elif backend == _Backend.FLASHINFER:
79 logger.info("Using Flashinfer backend.")
80 - logger.warning(("Flashinfer will be stuck on llama-2-7b,"
81 - " please avoid using Flashinfer as the "
82 - "backend when running on llama-2-7b."))
80 from vllm.attention.backends.flashinfer import FlashInferBackend
81 return FlashInferBackend
82 elif backend == _Backend.PALLAS: