A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[Docker] Adding number of nvcc_threads during build as envar (#1893)
AguirreNicolas committed
Dec 7, 2023 at 16:00 UTC
24f60a54f42076e0bfa49fde113756bf4e95f9ef
3 files changed
+6
-2
Dockerfile
+3
@@ -32,6 +32,9 @@ COPY vllm/__init__.py vllm/__init__.py
32
33
# max jobs used by Ninja to build extensions
34
ENV MAX_JOBS=$max_jobs
35
+# number of threads used by nvcc
36
+ARG nvcc_threads=8
37
+ENV NVCC_THREADS=$nvcc_threads
38
RUN python3 setup.py build_ext --inplace
39
40
# image to run unit testing suite
docs/source/serving/deploying_with_docker.rst
+1
-1
@@ -29,7 +29,7 @@ You can build and run vLLM from source via the provided dockerfile. To build vLL
29
30
.. code-block:: console
31
32
- $ DOCKER_BUILDKIT=1 docker build . --target vllm-openai --tag vllm/vllm-openai --build-arg max_jobs=8
32
+ $ DOCKER_BUILDKIT=1 docker build . --target vllm-openai --tag vllm/vllm-openai # optionally specifies: --build-arg max_jobs=8 --build-arg nvcc_threads=2
33
34
To run vLLM:
35
setup.py
+2
-1
@@ -138,7 +138,8 @@ for capability in compute_capabilities:
138
139
# Use NVCC threads to parallelize the build.
140
if nvcc_cuda_version >= Version("11.2"):
141
- num_threads = min(os.cpu_count(), 8)
141
+ nvcc_threads = int(os.getenv("NVCC_THREADS"), 8)
142
+ num_threads = min(os.cpu_count(), nvcc_threads)
143
NVCC_FLAGS += ["--threads", str(num_threads)]
144
145
ext_modules = []