A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Fix for breaking changes in xformers 0.0.21 (#834)
Woosuk Kwon committed
Aug 23, 2023 at 17:44 UTC
2a4ec90854ae3ad08a3593cb4896dfce601974c3
2 files changed
+4
-3
requirements.txt
+1
-1
@@ -5,7 +5,7 @@ sentencepiece # Required for LLaMA tokenizer.
5
numpy
6
torch >= 2.0.0
7
transformers >= 4.31.0 # Required for LLaMA-2.
8
-xformers >= 0.0.19
8
+xformers >= 0.0.21
9
fastapi
10
uvicorn
11
pydantic < 2 # Required for OpenAI server.
vllm/model_executor/layers/attention.py
+3
-2
@@ -357,11 +357,12 @@ class PagedAttentionWithALiBi(PagedAttention):
357
# be sliced from a tensor whose length is a multiple of 8.
358
padded_len = (prompt_len + 7) // 8 * 8
359
bias = torch.empty(
360
+ 1, # batch_size
361
self.num_heads,
361
- padded_len,
362
+ prompt_len,
363
padded_len,
364
device=self.alibi_slopes.device,
364
- )[:, :prompt_len, :prompt_len].copy_(bias)
365
+ )[:, :, :, :prompt_len].copy_(bias)
366
bias.mul_(self.alibi_slopes[:, None, None])
367
attn_bias = LowerTriangularMaskWithTensorBias(bias)
368
input_metadata.attn_bias.append(attn_bias)