@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6e4658c7aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Intel GPU] fix xpu not support punica kernel (which use torch.library.custom_op) (#7685)

Kunshang Ji committed Aug 21, 2024 at 03:01 UTC 6e4658c7aaa8020ebb183729d53baaf8310859ce
1 file changed +3 -1
vllm/lora/punica.py
+3 -1
@@ -10,8 +10,10 @@ from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union
10 import torch
11
12 from vllm.triton_utils import HAS_TRITON
13 +from vllm.utils import is_xpu
14
14 -if HAS_TRITON:
15 +# FIXME: xpu path doesn't support torch.library.custom_op
16 +if HAS_TRITON and not is_xpu():
17 from vllm.lora.ops.bgmv_expand import bgmv_expand
18 from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
19 from vllm.lora.ops.bgmv_shrink import bgmv_shrink