A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Model][Phi3-Small] Remove scipy from blocksparse_attention (#6343)
Michael Goin committed
Jul 11, 2024 at 22:47 UTC
d59eb98489103877e9476ef5263305aa3e3f9e23
1 file changed
+27
-8
vllm/attention/ops/blocksparse_attention/utils.py
+27
-8
@@ -4,16 +4,35 @@
4
5
from functools import lru_cache
6
7
+import numpy as np
8
import torch
9
import triton
10
10
-try:
11
- from scipy import sparse
12
-except ImportError as err:
13
- raise ImportError("Please install scipy via "
14
- "`pip install scipy` to use "
15
- "BlockSparseAttention in "
16
- "models such as Phi-3.") from err
11
+
12
+class csr_matrix:
13
+ """Simple implementation of CSR matrix conversion without scipy.
14
+ This replaced scipy.sparse.csr_matrix() previously used."""
15
+
16
+ def __init__(self, input_array):
17
+ if not isinstance(input_array, np.ndarray):
18
+ raise ValueError("Input must be a NumPy array")
19
+
20
+ self.shape = input_array.shape
21
+ rows, cols = self.shape
22
+ data = []
23
+ indices = []
24
+ indptr = [0]
25
+
26
+ for i in range(rows):
27
+ for j in range(cols):
28
+ if input_array[i, j]:
29
+ data.append(input_array[i, j])
30
+ indices.append(j)
31
+ indptr.append(len(indices))
32
+
33
+ self.data = np.array(data)
34
+ self.indices = np.array(indices)
35
+ self.indptr = np.array(indptr)
36
37
38
def dense_to_crow_col(x: torch.Tensor):
@@ -26,7 +45,7 @@ def dense_to_crow_col(x: torch.Tensor):
45
assert x.dim() in (2, 3)
46
if x.dim() == 2:
47
x = x[None]
29
- x = [sparse.csr_matrix(xi.bool().cpu().numpy()) for xi in x]
48
+ x = [csr_matrix(xi.bool().cpu().numpy()) for xi in x]
49
crows = torch.vstack([torch.from_numpy(xi.indptr) for xi in x])
50
cols = [torch.from_numpy(xi.indices) for xi in x]
51
max_cols = max(len(xi) for xi in cols)