A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Don't build punica kernels by default (#2605)
Philipp Moritz committed
Jan 26, 2024 at 15:19 UTC
390b495ff327e8548c3f7cd701afce87870d9102
4 files changed
+11
-4
.github/workflows/scripts/build.sh
+2
@@ -13,6 +13,8 @@ $python_executable -m pip install -r requirements.txt
13
14
# Limit the number of parallel jobs to avoid OOM
15
export MAX_JOBS=1
16
+# Make sure punica is built for the release (for LoRA)
17
+export VLLM_INSTALL_PUNICA_KERNELS=1
18
19
# Build
20
$python_executable setup.py bdist_wheel --dist-dir=dist
Dockerfile
+2
@@ -45,6 +45,8 @@ ENV MAX_JOBS=${max_jobs}
45
# number of threads used by nvcc
46
ARG nvcc_threads=8
47
ENV NVCC_THREADS=$nvcc_threads
48
+# make sure punica kernels are built (for LoRA)
49
+ENV VLLM_INSTALL_PUNICA_KERNELS=1
50
51
RUN python3 setup.py build_ext --inplace
52
#################### EXTENSION Build IMAGE ####################
setup.py
+1
-1
@@ -265,7 +265,7 @@ if _is_cuda():
265
with contextlib.suppress(ValueError):
266
torch_cpp_ext.COMMON_NVCC_FLAGS.remove(flag)
267
268
- install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "1")))
268
+ install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
269
device_count = torch.cuda.device_count()
270
for i in range(device_count):
271
major, minor = torch.cuda.get_device_capability(i)
vllm/lora/punica.py
+6
-3
@@ -157,10 +157,13 @@ else:
157
**kwargs # pylint: disable=unused-argument
158
):
159
if torch.cuda.get_device_capability() < (8, 0):
160
- raise ImportError(
161
- "LoRA kernels require compute capability>=8.0") from import_exc
160
+ raise ImportError("punica LoRA kernels require compute "
161
+ "capability>=8.0") from import_exc
162
else:
163
- raise import_exc
163
+ raise ImportError(
164
+ "punica LoRA kernels could not be imported. If you built vLLM "
165
+ "from source, make sure VLLM_INSTALL_PUNICA_KERNELS=1 env var "
166
+ "was set.") from import_exc
167
168
bgmv = _raise_exc
169
add_lora = _raise_exc