@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3cdfe1f38b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Make torch registration of punica ops optional (#7970)

bnellnm committed Aug 28, 2024 at 18:11 UTC 3cdfe1f38b2c07a10a1681cd2d60c3bea1bae2f0
7 files changed +37 -21
vllm/lora/ops/bgmv_expand.py
+6 -3
@@ -160,6 +160,9 @@ def _bgmv_expand(
160 return
161
162
163 -bgmv_expand = torch.library.custom_op("lora::bgmv_expand",
164 - _bgmv_expand,
165 - mutates_args=["output_tensor"])
163 +try:
164 + bgmv_expand = torch.library.custom_op("lora::bgmv_expand",
165 + _bgmv_expand,
166 + mutates_args=["output_tensor"])
167 +except AttributeError:
168 + bgmv_expand = _bgmv_expand
vllm/lora/ops/bgmv_expand_slice.py
+6 -3
@@ -173,6 +173,9 @@ def _bgmv_expand_slice(
173 return
174
175
176 -bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice",
177 - _bgmv_expand_slice,
178 - mutates_args=["output_tensor"])
176 +try:
177 + bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice",
178 + _bgmv_expand_slice,
179 + mutates_args=["output_tensor"])
180 +except AttributeError:
181 + bgmv_expand_slice = _bgmv_expand_slice
vllm/lora/ops/bgmv_shrink.py
+6 -3
@@ -142,6 +142,9 @@ def _bgmv_shrink(
142 return
143
144
145 -bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink",
146 - _bgmv_shrink,
147 - mutates_args=["output_tensor"])
145 +try:
146 + bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink",
147 + _bgmv_shrink,
148 + mutates_args=["output_tensor"])
149 +except AttributeError:
150 + bgmv_shrink = _bgmv_shrink
vllm/lora/ops/sgmv_expand.py
+6 -3
@@ -192,6 +192,9 @@ def _sgmv_expand(
192 return
193
194
195 -sgmv_expand = torch.library.custom_op("lora::sgmv_expand",
196 - _sgmv_expand,
197 - mutates_args=["output_tensor"])
195 +try:
196 + sgmv_expand = torch.library.custom_op("lora::sgmv_expand",
197 + _sgmv_expand,
198 + mutates_args=["output_tensor"])
199 +except AttributeError:
200 + sgmv_expand = _sgmv_expand
vllm/lora/ops/sgmv_expand_slice.py
+6 -3
@@ -205,6 +205,9 @@ def _sgmv_expand_slice(
205 return
206
207
208 -sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice",
209 - _sgmv_expand_slice,
210 - mutates_args=["output_tensor"])
208 +try:
209 + sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice",
210 + _sgmv_expand_slice,
211 + mutates_args=["output_tensor"])
212 +except AttributeError:
213 + sgmv_expand_slice = _sgmv_expand_slice
vllm/lora/ops/sgmv_shrink.py
+6 -3
@@ -189,6 +189,9 @@ def _sgmv_shrink(
189 return
190
191
192 -sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink",
193 - _sgmv_shrink,
194 - mutates_args=["output_tensor"])
192 +try:
193 + sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink",
194 + _sgmv_shrink,
195 + mutates_args=["output_tensor"])
196 +except AttributeError:
197 + sgmv_shrink = _sgmv_shrink
vllm/lora/punica.py
+1 -3
@@ -10,10 +10,8 @@ from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union
10 import torch
11
12 from vllm.triton_utils import HAS_TRITON
13 -from vllm.utils import is_xpu
13
15 -# FIXME: xpu path doesn't support torch.library.custom_op
16 -if HAS_TRITON and not is_xpu():
14 +if HAS_TRITON:
15 from vllm.lora.ops.bgmv_expand import bgmv_expand
16 from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17 from vllm.lora.ops.bgmv_shrink import bgmv_shrink