A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Add verbose error if scipy is missing for blocksparse attention (#5695)
JGSweets committed
Jul 5, 2024 at 12:41 UTC
e58294ddf23107d93987c00611d63a20e3cfe771
1 file changed
+13
-6
vllm/attention/ops/blocksparse_attention/utils.py
+13
-6
@@ -6,7 +6,14 @@ from functools import lru_cache
6
7
import torch
8
import triton
9
-from scipy import sparse
9
+
10
+try:
11
+ from scipy import sparse
12
+except ImportError as err:
13
+ raise ImportError("Please install scipy via "
14
+ "`pip install scipy` to use "
15
+ "BlockSparseAttention in "
16
+ "models such as Phi-3.") from err
17
18
19
def dense_to_crow_col(x: torch.Tensor):
@@ -77,11 +84,11 @@ def _get_sparse_attn_mask_homo_head(
84
):
85
"""
86
:return: a tuple of 3:
80
- - tuple of crow_indices, col_indices representation
87
+ - tuple of crow_indices, col_indices representation
88
of CSR format.
89
- block dense mask
83
- - all token dense mask (be aware that it can be
84
- OOM if it is too big) if `return_dense==True`,
90
+ - all token dense mask (be aware that it can be
91
+ OOM if it is too big) if `return_dense==True`,
92
otherwise, None
93
"""
94
with torch.no_grad():
@@ -148,10 +155,10 @@ def get_sparse_attn_mask(
155
:param dense_mask_type: "binary" (0 for skip token, 1 for others)
156
or "bias" (-inf for skip token, 0 or others)
157
:return: a tuple of 3:
151
- - tuple of crow_indices, col_indices representation
158
+ - tuple of crow_indices, col_indices representation
159
of CSR format.
160
- block dense mask
154
- - all token dense mask (be aware that it can be OOM if it
161
+ - all token dense mask (be aware that it can be OOM if it
162
is too big) if `return_dense==True`, otherwise, None
163
"""
164
assert dense_mask_type in ("binary", "bias")