@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2a4ec90854
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Fix for breaking changes in xformers 0.0.21 (#834)

Woosuk Kwon committed Aug 23, 2023 at 17:44 UTC 2a4ec90854ae3ad08a3593cb4896dfce601974c3
2 files changed +4 -3
requirements.txt
+1 -1
@@ -5,7 +5,7 @@ sentencepiece # Required for LLaMA tokenizer.
5 numpy
6 torch >= 2.0.0
7 transformers >= 4.31.0 # Required for LLaMA-2.
8 -xformers >= 0.0.19
8 +xformers >= 0.0.21
9 fastapi
10 uvicorn
11 pydantic < 2 # Required for OpenAI server.
vllm/model_executor/layers/attention.py
+3 -2
@@ -357,11 +357,12 @@ class PagedAttentionWithALiBi(PagedAttention):
357 # be sliced from a tensor whose length is a multiple of 8.
358 padded_len = (prompt_len + 7) // 8 * 8
359 bias = torch.empty(
360 + 1, # batch_size
361 self.num_heads,
361 - padded_len,
362 + prompt_len,
363 padded_len,
364 device=self.alibi_slopes.device,
364 - )[:, :prompt_len, :prompt_len].copy_(bias)
365 + )[:, :, :, :prompt_len].copy_(bias)
366 bias.mul_(self.alibi_slopes[:, None, None])
367 attn_bias = LowerTriangularMaskWithTensorBias(bias)
368 input_metadata.attn_bias.append(attn_bias)