A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Intel GPU] fix xpu not support punica kernel (which use torch.library.custom_op) (#7685)
Kunshang Ji committed
Aug 21, 2024 at 03:01 UTC
6e4658c7aaa8020ebb183729d53baaf8310859ce
1 file changed
+3
-1
vllm/lora/punica.py
+3
-1
@@ -10,8 +10,10 @@ from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union
10
import torch
11
12
from vllm.triton_utils import HAS_TRITON
13
+from vllm.utils import is_xpu
14
14
-if HAS_TRITON:
15
+# FIXME: xpu path doesn't support torch.library.custom_op
16
+if HAS_TRITON and not is_xpu():
17
from vllm.lora.ops.bgmv_expand import bgmv_expand
18
from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
19
from vllm.lora.ops.bgmv_shrink import bgmv_shrink