@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e58294ddf2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Add verbose error if scipy is missing for blocksparse attention (#5695)

JGSweets committed Jul 5, 2024 at 12:41 UTC e58294ddf23107d93987c00611d63a20e3cfe771
1 file changed +13 -6
vllm/attention/ops/blocksparse_attention/utils.py
+13 -6
@@ -6,7 +6,14 @@ from functools import lru_cache
6
7 import torch
8 import triton
9 -from scipy import sparse
9 +
10 +try:
11 + from scipy import sparse
12 +except ImportError as err:
13 + raise ImportError("Please install scipy via "
14 + "`pip install scipy` to use "
15 + "BlockSparseAttention in "
16 + "models such as Phi-3.") from err
17
18
19 def dense_to_crow_col(x: torch.Tensor):
@@ -77,11 +84,11 @@ def _get_sparse_attn_mask_homo_head(
84 ):
85 """
86 :return: a tuple of 3:
80 - - tuple of crow_indices, col_indices representation
87 + - tuple of crow_indices, col_indices representation
88 of CSR format.
89 - block dense mask
83 - - all token dense mask (be aware that it can be
84 - OOM if it is too big) if `return_dense==True`,
90 + - all token dense mask (be aware that it can be
91 + OOM if it is too big) if `return_dense==True`,
92 otherwise, None
93 """
94 with torch.no_grad():
@@ -148,10 +155,10 @@ def get_sparse_attn_mask(
155 :param dense_mask_type: "binary" (0 for skip token, 1 for others)
156 or "bias" (-inf for skip token, 0 or others)
157 :return: a tuple of 3:
151 - - tuple of crow_indices, col_indices representation
158 + - tuple of crow_indices, col_indices representation
159 of CSR format.
160 - block dense mask
154 - - all token dense mask (be aware that it can be OOM if it
161 + - all token dense mask (be aware that it can be OOM if it
162 is too big) if `return_dense==True`, otherwise, None
163 """
164 assert dense_mask_type in ("binary", "bias")