@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 390b495ff3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Don't build punica kernels by default (#2605)

Philipp Moritz committed Jan 26, 2024 at 15:19 UTC 390b495ff327e8548c3f7cd701afce87870d9102
4 files changed +11 -4
.github/workflows/scripts/build.sh
+2
@@ -13,6 +13,8 @@ $python_executable -m pip install -r requirements.txt
13
14 # Limit the number of parallel jobs to avoid OOM
15 export MAX_JOBS=1
16 +# Make sure punica is built for the release (for LoRA)
17 +export VLLM_INSTALL_PUNICA_KERNELS=1
18
19 # Build
20 $python_executable setup.py bdist_wheel --dist-dir=dist
Dockerfile
+2
@@ -45,6 +45,8 @@ ENV MAX_JOBS=${max_jobs}
45 # number of threads used by nvcc
46 ARG nvcc_threads=8
47 ENV NVCC_THREADS=$nvcc_threads
48 +# make sure punica kernels are built (for LoRA)
49 +ENV VLLM_INSTALL_PUNICA_KERNELS=1
50
51 RUN python3 setup.py build_ext --inplace
52 #################### EXTENSION Build IMAGE ####################
setup.py
+1 -1
@@ -265,7 +265,7 @@ if _is_cuda():
265 with contextlib.suppress(ValueError):
266 torch_cpp_ext.COMMON_NVCC_FLAGS.remove(flag)
267
268 - install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "1")))
268 + install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
269 device_count = torch.cuda.device_count()
270 for i in range(device_count):
271 major, minor = torch.cuda.get_device_capability(i)
vllm/lora/punica.py
+6 -3
@@ -157,10 +157,13 @@ else:
157 **kwargs # pylint: disable=unused-argument
158 ):
159 if torch.cuda.get_device_capability() < (8, 0):
160 - raise ImportError(
161 - "LoRA kernels require compute capability>=8.0") from import_exc
160 + raise ImportError("punica LoRA kernels require compute "
161 + "capability>=8.0") from import_exc
162 else:
163 - raise import_exc
163 + raise ImportError(
164 + "punica LoRA kernels could not be imported. If you built vLLM "
165 + "from source, make sure VLLM_INSTALL_PUNICA_KERNELS=1 env var "
166 + "was set.") from import_exc
167
168 bgmv = _raise_exc
169 add_lora = _raise_exc