@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d59eb98489
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Model][Phi3-Small] Remove scipy from blocksparse_attention (#6343)

Michael Goin committed Jul 11, 2024 at 22:47 UTC d59eb98489103877e9476ef5263305aa3e3f9e23
1 file changed +27 -8
vllm/attention/ops/blocksparse_attention/utils.py
+27 -8
@@ -4,16 +4,35 @@
4
5 from functools import lru_cache
6
7 +import numpy as np
8 import torch
9 import triton
10
10 -try:
11 - from scipy import sparse
12 -except ImportError as err:
13 - raise ImportError("Please install scipy via "
14 - "`pip install scipy` to use "
15 - "BlockSparseAttention in "
16 - "models such as Phi-3.") from err
11 +
12 +class csr_matrix:
13 + """Simple implementation of CSR matrix conversion without scipy.
14 + This replaced scipy.sparse.csr_matrix() previously used."""
15 +
16 + def __init__(self, input_array):
17 + if not isinstance(input_array, np.ndarray):
18 + raise ValueError("Input must be a NumPy array")
19 +
20 + self.shape = input_array.shape
21 + rows, cols = self.shape
22 + data = []
23 + indices = []
24 + indptr = [0]
25 +
26 + for i in range(rows):
27 + for j in range(cols):
28 + if input_array[i, j]:
29 + data.append(input_array[i, j])
30 + indices.append(j)
31 + indptr.append(len(indices))
32 +
33 + self.data = np.array(data)
34 + self.indices = np.array(indices)
35 + self.indptr = np.array(indptr)
36
37
38 def dense_to_crow_col(x: torch.Tensor):
@@ -26,7 +45,7 @@ def dense_to_crow_col(x: torch.Tensor):
45 assert x.dim() in (2, 3)
46 if x.dim() == 2:
47 x = x[None]
29 - x = [sparse.csr_matrix(xi.bool().cpu().numpy()) for xi in x]
48 + x = [csr_matrix(xi.bool().cpu().numpy()) for xi in x]
49 crows = torch.vstack([torch.from_numpy(xi.indptr) for xi in x])
50 cols = [torch.from_numpy(xi.indices) for xi in x]
51 max_cols = max(len(xi) for xi in cols)