@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1cb0cc2975
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[FIX] Make `flash_attn` optional (#3269)

Woosuk Kwon committed Mar 8, 2024 at 10:52 UTC 1cb0cc2975d1c42c445c795f955b783e78919502
5 files changed +41 -78
.gitignore
-3
@@ -184,6 +184,3 @@ _build/
184
185 # Benchmark dataset
186 *.json
187 -
188 -# Third-party Python packages.
189 -vllm/thirdparty_files/
setup.py
+3 -45
@@ -3,7 +3,6 @@ import io
3 import os
4 import re
5 import subprocess
6 -import sys
6 import warnings
7 from pathlib import Path
8 from typing import List, Set
@@ -15,8 +14,6 @@ import torch.utils.cpp_extension as torch_cpp_ext
14 from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME
15
16 ROOT_DIR = os.path.dirname(__file__)
18 -# This is a temporary directory to store third-party packages.
19 -THIRDPARTY_SUBDIR = "vllm/thirdparty_files"
17
18 # If you are developing the C++ backend of vLLM, consider building vLLM with
19 # `python setup.py develop` since it will give you incremental builds.
@@ -327,46 +324,8 @@ if _is_cuda():
324 "nvcc": NVCC_FLAGS_PUNICA,
325 },
326 ))
330 -
331 - # Download the FlashAttention package.
332 - # Adapted from https://github.com/ray-project/ray/blob/f92928c9cfcbbf80c3a8534ca4911de1b44069c0/python/setup.py#L518-L530
333 - flash_attn_version = "2.5.6"
334 - install_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
335 - subprocess.check_call(
336 - [
337 - sys.executable,
338 - "-m",
339 - "pip",
340 - "install",
341 - "-q",
342 - f"--target={install_dir}",
343 - "einops", # Dependency of flash-attn.
344 - f"flash-attn=={flash_attn_version}",
345 - "--no-dependencies", # Required to avoid re-installing torch.
346 - ],
347 - env=dict(os.environ, CC="gcc"),
348 - )
349 -
350 - # Copy the FlashAttention package into the vLLM package after build.
351 - class build_ext(BuildExtension):
352 -
353 - def run(self):
354 - super().run()
355 - target_dir = os.path.join(self.build_lib, THIRDPARTY_SUBDIR)
356 - if not os.path.exists(target_dir):
357 - os.makedirs(target_dir)
358 - self.copy_tree(install_dir, target_dir)
359 -
360 - class BinaryDistribution(setuptools.Distribution):
361 -
362 - def has_ext_modules(self):
363 - return True
364 -
365 -else:
366 - build_ext = BuildExtension
367 - BinaryDistribution = setuptools.Distribution
368 - if _is_neuron():
369 - neuronxcc_version = get_neuronxcc_version()
327 +elif _is_neuron():
328 + neuronxcc_version = get_neuronxcc_version()
329
330 vllm_extension_sources = [
331 "csrc/cache_kernels.cu",
@@ -509,7 +468,6 @@ setuptools.setup(
468 python_requires=">=3.8",
469 install_requires=get_requirements(),
470 ext_modules=ext_modules,
512 - cmdclass={"build_ext": build_ext} if not _is_neuron() else {},
513 - distclass=BinaryDistribution,
471 + cmdclass={"build_ext": BuildExtension} if not _is_neuron() else {},
472 package_data=package_data,
473 )
vllm/__init__.py
+7 -23
@@ -1,28 +1,12 @@
1 """vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
2
3 -
4 -# Adapted from https://github.com/ray-project/ray/blob/f92928c9cfcbbf80c3a8534ca4911de1b44069c0/python/ray/__init__.py#L11
5 -def _configure_system():
6 - import os
7 - import sys
8 -
9 - # Importing flash-attn.
10 - thirdparty_files = os.path.join(os.path.abspath(os.path.dirname(__file__)),
11 - "thirdparty_files")
12 - sys.path.insert(0, thirdparty_files)
13 -
14 -
15 -_configure_system()
16 -# Delete configuration function.
17 -del _configure_system
18 -
19 -from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs # noqa: E402
20 -from vllm.engine.async_llm_engine import AsyncLLMEngine # noqa: E402
21 -from vllm.engine.llm_engine import LLMEngine # noqa: E402
22 -from vllm.engine.ray_utils import initialize_cluster # noqa: E402
23 -from vllm.entrypoints.llm import LLM # noqa: E402
24 -from vllm.outputs import CompletionOutput, RequestOutput # noqa: E402
25 -from vllm.sampling_params import SamplingParams # noqa: E402
3 +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4 +from vllm.engine.async_llm_engine import AsyncLLMEngine
5 +from vllm.engine.llm_engine import LLMEngine
6 +from vllm.engine.ray_utils import initialize_cluster
7 +from vllm.entrypoints.llm import LLM
8 +from vllm.outputs import CompletionOutput, RequestOutput
9 +from vllm.sampling_params import SamplingParams
10
11 __version__ = "0.3.3"
12
vllm/model_executor/layers/attention/attention.py
+31 -6
@@ -1,12 +1,16 @@
1 """Attention layer."""
2 +from functools import lru_cache
3 from typing import List, Optional
4
5 import torch
6 import torch.nn as nn
7
8 +from vllm.logger import init_logger
9 from vllm.model_executor.input_metadata import InputMetadata
10 from vllm.utils import is_hip
11
12 +logger = init_logger(__name__)
13 +
14
15 class Attention(nn.Module):
16 """Attention layer.
@@ -30,17 +34,12 @@ class Attention(nn.Module):
34 sliding_window: Optional[int] = None,
35 ) -> None:
36 super().__init__()
33 - if (not is_hip() and torch.cuda.get_device_capability()[0] >= 8 and
34 - torch.get_default_dtype() in (torch.float16, torch.bfloat16)):
35 - # Ampere or later NVIDIA GPUs.
36 - # NOTE(woosuk): FlashAttention does not support FP32.
37 + if _use_flash_attn():
38 from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend
39 self.backend = FlashAttentionBackend(num_heads, head_size, scale,
40 num_kv_heads, alibi_slopes,
41 sliding_window)
42 else:
42 - # Turing and Volta NVIDIA GPUs or AMD GPUs.
43 - # Or FP32 on any GPU.
43 from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend
44 self.backend = XFormersBackend(num_heads, head_size, scale,
45 num_kv_heads, alibi_slopes,
@@ -57,3 +56,29 @@ class Attention(nn.Module):
56 ) -> torch.Tensor:
57 return self.backend.forward(query, key, value, key_cache, value_cache,
58 input_metadata)
59 +
60 +
61 +@lru_cache(maxsize=1)
62 +def _use_flash_attn() -> bool:
63 + try:
64 + import flash_attn # noqa: F401
65 + except ImportError:
66 + logger.info("flash_attn is not found. Using xformers backend.")
67 + return False
68 +
69 + if is_hip():
70 + # AMD GPUs.
71 + return False
72 + if torch.cuda.get_device_capability()[0] < 8:
73 + # Volta and Turing NVIDIA GPUs.
74 + logger.info("flash_attn is not supported on Turing or older GPUs. "
75 + "Using xformers backend.")
76 + return False
77 + if torch.get_default_dtype() not in (torch.float16, torch.bfloat16):
78 + logger.info(
79 + "flash_attn only supports torch.float16 or torch.bfloat16. "
80 + "Using xformers backend.")
81 + return False
82 +
83 + logger.info("Using flash_attn backend.")
84 + return True
vllm/model_executor/layers/attention/backends/flash_attn.py
-1
@@ -1,7 +1,6 @@
1 """Attention layer with Flash and PagedAttention."""
2 from typing import List, Optional
3
4 -# NOTE(woosuk): This imports flash_attn under vllm/thirdparty_files/.
4 from flash_attn import flash_attn_func
5 import torch
6