A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Platform] Move get_punica_wrapper() function to Platform (#11516)
Signed-off-by: Shanshan Shen <467638484@qq.com>
Shanshan Shen committed
Jan 13, 2025 at 21:12 UTC
a7d59688fb75827db4316c24a057ac6097114bd3
6 files changed
+32
-17
vllm/lora/punica_wrapper/punica_selector.py
+9
-17
@@ -1,5 +1,6 @@
1
from vllm.logger import init_logger
2
from vllm.platforms import current_platform
3
+from vllm.utils import resolve_obj_by_qualname
4
5
from .punica_base import PunicaWrapperBase
6
@@ -7,20 +8,11 @@ logger = init_logger(__name__)
8
9
10
def get_punica_wrapper(*args, **kwargs) -> PunicaWrapperBase:
10
- if current_platform.is_cuda_alike():
11
- # Lazy import to avoid ImportError
12
- from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
13
- logger.info_once("Using PunicaWrapperGPU.")
14
- return PunicaWrapperGPU(*args, **kwargs)
15
- elif current_platform.is_cpu():
16
- # Lazy import to avoid ImportError
17
- from vllm.lora.punica_wrapper.punica_cpu import PunicaWrapperCPU
18
- logger.info_once("Using PunicaWrapperCPU.")
19
- return PunicaWrapperCPU(*args, **kwargs)
20
- elif current_platform.is_hpu():
21
- # Lazy import to avoid ImportError
22
- from vllm.lora.punica_wrapper.punica_hpu import PunicaWrapperHPU
23
- logger.info_once("Using PunicaWrapperHPU.")
24
- return PunicaWrapperHPU(*args, **kwargs)
25
- else:
26
- raise NotImplementedError
11
+ punica_wrapper_qualname = current_platform.get_punica_wrapper()
12
+ punica_wrapper_cls = resolve_obj_by_qualname(punica_wrapper_qualname)
13
+ punica_wrapper = punica_wrapper_cls(*args, **kwargs)
14
+ assert punica_wrapper is not None, \
15
+ "the punica_wrapper_qualname(" + punica_wrapper_qualname + ") is wrong."
16
+ logger.info_once("Using " + punica_wrapper_qualname.rsplit(".", 1)[1] +
17
+ ".")
18
+ return punica_wrapper
vllm/platforms/cpu.py
+4
@@ -109,3 +109,7 @@ class CpuPlatform(Platform):
109
def is_pin_memory_available(cls) -> bool:
110
logger.warning("Pin memory is not supported on CPU.")
111
return False
112
+
113
+ @classmethod
114
+ def get_punica_wrapper(cls) -> str:
115
+ return "vllm.lora.punica_wrapper.punica_cpu.PunicaWrapperCPU"
vllm/platforms/cuda.py
+4
@@ -218,6 +218,10 @@ class CudaPlatformBase(Platform):
218
logger.info("Using Flash Attention backend.")
219
return "vllm.attention.backends.flash_attn.FlashAttentionBackend"
220
221
+ @classmethod
222
+ def get_punica_wrapper(cls) -> str:
223
+ return "vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPU"
224
+
225
226
# NVML utils
227
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
vllm/platforms/hpu.py
+4
@@ -63,3 +63,7 @@ class HpuPlatform(Platform):
63
def is_pin_memory_available(cls):
64
logger.warning("Pin memory is not supported on HPU.")
65
return False
66
+
67
+ @classmethod
68
+ def get_punica_wrapper(cls) -> str:
69
+ return "vllm.lora.punica_wrapper.punica_hpu.PunicaWrapperHPU"
vllm/platforms/interface.py
+7
@@ -276,6 +276,13 @@ class Platform:
276
return False
277
return True
278
279
+ @classmethod
280
+ def get_punica_wrapper(cls) -> str:
281
+ """
282
+ Return the punica wrapper for current platform.
283
+ """
284
+ raise NotImplementedError
285
+
286
287
class UnspecifiedPlatform(Platform):
288
_enum = PlatformEnum.UNSPECIFIED
vllm/platforms/rocm.py
+4
@@ -153,3 +153,7 @@ class RocmPlatform(Platform):
153
"Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
154
" is not set, enabling VLLM_USE_TRITON_AWQ.")
155
envs.VLLM_USE_TRITON_AWQ = True
156
+
157
+ @classmethod
158
+ def get_punica_wrapper(cls) -> str:
159
+ return "vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPU"