A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Make torch registration of punica ops optional (#7970)
bnellnm committed
Aug 28, 2024 at 18:11 UTC
3cdfe1f38b2c07a10a1681cd2d60c3bea1bae2f0
7 files changed
+37
-21
vllm/lora/ops/bgmv_expand.py
+6
-3
@@ -160,6 +160,9 @@ def _bgmv_expand(
160
return
161
162
163
-bgmv_expand = torch.library.custom_op("lora::bgmv_expand",
164
- _bgmv_expand,
165
- mutates_args=["output_tensor"])
163
+try:
164
+ bgmv_expand = torch.library.custom_op("lora::bgmv_expand",
165
+ _bgmv_expand,
166
+ mutates_args=["output_tensor"])
167
+except AttributeError:
168
+ bgmv_expand = _bgmv_expand
vllm/lora/ops/bgmv_expand_slice.py
+6
-3
@@ -173,6 +173,9 @@ def _bgmv_expand_slice(
173
return
174
175
176
-bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice",
177
- _bgmv_expand_slice,
178
- mutates_args=["output_tensor"])
176
+try:
177
+ bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice",
178
+ _bgmv_expand_slice,
179
+ mutates_args=["output_tensor"])
180
+except AttributeError:
181
+ bgmv_expand_slice = _bgmv_expand_slice
vllm/lora/ops/bgmv_shrink.py
+6
-3
@@ -142,6 +142,9 @@ def _bgmv_shrink(
142
return
143
144
145
-bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink",
146
- _bgmv_shrink,
147
- mutates_args=["output_tensor"])
145
+try:
146
+ bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink",
147
+ _bgmv_shrink,
148
+ mutates_args=["output_tensor"])
149
+except AttributeError:
150
+ bgmv_shrink = _bgmv_shrink
vllm/lora/ops/sgmv_expand.py
+6
-3
@@ -192,6 +192,9 @@ def _sgmv_expand(
192
return
193
194
195
-sgmv_expand = torch.library.custom_op("lora::sgmv_expand",
196
- _sgmv_expand,
197
- mutates_args=["output_tensor"])
195
+try:
196
+ sgmv_expand = torch.library.custom_op("lora::sgmv_expand",
197
+ _sgmv_expand,
198
+ mutates_args=["output_tensor"])
199
+except AttributeError:
200
+ sgmv_expand = _sgmv_expand
vllm/lora/ops/sgmv_expand_slice.py
+6
-3
@@ -205,6 +205,9 @@ def _sgmv_expand_slice(
205
return
206
207
208
-sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice",
209
- _sgmv_expand_slice,
210
- mutates_args=["output_tensor"])
208
+try:
209
+ sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice",
210
+ _sgmv_expand_slice,
211
+ mutates_args=["output_tensor"])
212
+except AttributeError:
213
+ sgmv_expand_slice = _sgmv_expand_slice
vllm/lora/ops/sgmv_shrink.py
+6
-3
@@ -189,6 +189,9 @@ def _sgmv_shrink(
189
return
190
191
192
-sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink",
193
- _sgmv_shrink,
194
- mutates_args=["output_tensor"])
192
+try:
193
+ sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink",
194
+ _sgmv_shrink,
195
+ mutates_args=["output_tensor"])
196
+except AttributeError:
197
+ sgmv_shrink = _sgmv_shrink
vllm/lora/punica.py
+1
-3
@@ -10,10 +10,8 @@ from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union
10
import torch
11
12
from vllm.triton_utils import HAS_TRITON
13
-from vllm.utils import is_xpu
13
15
-# FIXME: xpu path doesn't support torch.library.custom_op
16
-if HAS_TRITON and not is_xpu():
14
+if HAS_TRITON:
15
from vllm.lora.ops.bgmv_expand import bgmv_expand
16
from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17
from vllm.lora.ops.bgmv_shrink import bgmv_shrink