A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[BugFix] Fix mla cpu - missing 3 required positional arguments (#17494)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Lucas Wilkinson committed
May 1, 2025 at 02:36 UTC
3c3d76720156380ab399c55cace6fe79a8d155f9
2 files changed
+6
-1
vllm/_ipex_ops.py
+3
-1
@@ -177,7 +177,7 @@ class ipex_ops:
177
out: torch.Tensor,
178
seqlen_q: torch.Tensor,
179
seqlen_k: torch.Tensor,
180
- alibi_slopes: torch.Tensor,
180
+ alibi_slopes: Optional[torch.Tensor],
181
max_seqlen_q: int,
182
max_seqlen_k: int,
183
pdropout: float,
@@ -193,6 +193,8 @@ class ipex_ops:
193
if ipex.__version__.endswith("cpu"):
194
if logits_soft_cap != 0.0:
195
raise ValueError("IPEX CPU does not support logits_soft_cap")
196
+ assert alibi_slopes is None
197
+ assert window_size_left < 0 and window_size_right < 0
198
ipex.llm.functional.varlen_attention(query.contiguous(),
199
key.contiguous(),
200
value.contiguous(), out,
vllm/attention/backends/cpu_mla.py
+3
@@ -273,6 +273,9 @@ class CPUMLAImpl(MLACommonImpl[CPUMLAMetadata]):
273
return_softmax=False,
274
gen_=None,
275
logits_soft_cap=0.0,
276
+ window_size_left=-1,
277
+ window_size_right=-1,
278
+ alibi_slopes=None,
279
)
280
281
# remove padding