@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a7d59688fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Platform] Move get_punica_wrapper() function to Platform (#11516)

Signed-off-by: Shanshan Shen <467638484@qq.com>

Shanshan Shen committed Jan 13, 2025 at 21:12 UTC a7d59688fb75827db4316c24a057ac6097114bd3
6 files changed +32 -17
vllm/lora/punica_wrapper/punica_selector.py
+9 -17
@@ -1,5 +1,6 @@
1 from vllm.logger import init_logger
2 from vllm.platforms import current_platform
3 +from vllm.utils import resolve_obj_by_qualname
4
5 from .punica_base import PunicaWrapperBase
6
@@ -7,20 +8,11 @@ logger = init_logger(__name__)
8
9
10 def get_punica_wrapper(*args, **kwargs) -> PunicaWrapperBase:
10 - if current_platform.is_cuda_alike():
11 - # Lazy import to avoid ImportError
12 - from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
13 - logger.info_once("Using PunicaWrapperGPU.")
14 - return PunicaWrapperGPU(*args, **kwargs)
15 - elif current_platform.is_cpu():
16 - # Lazy import to avoid ImportError
17 - from vllm.lora.punica_wrapper.punica_cpu import PunicaWrapperCPU
18 - logger.info_once("Using PunicaWrapperCPU.")
19 - return PunicaWrapperCPU(*args, **kwargs)
20 - elif current_platform.is_hpu():
21 - # Lazy import to avoid ImportError
22 - from vllm.lora.punica_wrapper.punica_hpu import PunicaWrapperHPU
23 - logger.info_once("Using PunicaWrapperHPU.")
24 - return PunicaWrapperHPU(*args, **kwargs)
25 - else:
26 - raise NotImplementedError
11 + punica_wrapper_qualname = current_platform.get_punica_wrapper()
12 + punica_wrapper_cls = resolve_obj_by_qualname(punica_wrapper_qualname)
13 + punica_wrapper = punica_wrapper_cls(*args, **kwargs)
14 + assert punica_wrapper is not None, \
15 + "the punica_wrapper_qualname(" + punica_wrapper_qualname + ") is wrong."
16 + logger.info_once("Using " + punica_wrapper_qualname.rsplit(".", 1)[1] +
17 + ".")
18 + return punica_wrapper
vllm/platforms/cpu.py
+4
@@ -109,3 +109,7 @@ class CpuPlatform(Platform):
109 def is_pin_memory_available(cls) -> bool:
110 logger.warning("Pin memory is not supported on CPU.")
111 return False
112 +
113 + @classmethod
114 + def get_punica_wrapper(cls) -> str:
115 + return "vllm.lora.punica_wrapper.punica_cpu.PunicaWrapperCPU"
vllm/platforms/cuda.py
+4
@@ -218,6 +218,10 @@ class CudaPlatformBase(Platform):
218 logger.info("Using Flash Attention backend.")
219 return "vllm.attention.backends.flash_attn.FlashAttentionBackend"
220
221 + @classmethod
222 + def get_punica_wrapper(cls) -> str:
223 + return "vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPU"
224 +
225
226 # NVML utils
227 # Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
vllm/platforms/hpu.py
+4
@@ -63,3 +63,7 @@ class HpuPlatform(Platform):
63 def is_pin_memory_available(cls):
64 logger.warning("Pin memory is not supported on HPU.")
65 return False
66 +
67 + @classmethod
68 + def get_punica_wrapper(cls) -> str:
69 + return "vllm.lora.punica_wrapper.punica_hpu.PunicaWrapperHPU"
vllm/platforms/interface.py
+7
@@ -276,6 +276,13 @@ class Platform:
276 return False
277 return True
278
279 + @classmethod
280 + def get_punica_wrapper(cls) -> str:
281 + """
282 + Return the punica wrapper for current platform.
283 + """
284 + raise NotImplementedError
285 +
286
287 class UnspecifiedPlatform(Platform):
288 _enum = PlatformEnum.UNSPECIFIED
vllm/platforms/rocm.py
+4
@@ -153,3 +153,7 @@ class RocmPlatform(Platform):
153 "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
154 " is not set, enabling VLLM_USE_TRITON_AWQ.")
155 envs.VLLM_USE_TRITON_AWQ = True
156 +
157 + @classmethod
158 + def get_punica_wrapper(cls) -> str:
159 + return "vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPU"