@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 24f60a54f4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Docker] Adding number of nvcc_threads during build as envar (#1893)

AguirreNicolas committed Dec 7, 2023 at 16:00 UTC 24f60a54f42076e0bfa49fde113756bf4e95f9ef
3 files changed +6 -2
Dockerfile
+3
@@ -32,6 +32,9 @@ COPY vllm/__init__.py vllm/__init__.py
32
33 # max jobs used by Ninja to build extensions
34 ENV MAX_JOBS=$max_jobs
35 +# number of threads used by nvcc
36 +ARG nvcc_threads=8
37 +ENV NVCC_THREADS=$nvcc_threads
38 RUN python3 setup.py build_ext --inplace
39
40 # image to run unit testing suite
docs/source/serving/deploying_with_docker.rst
+1 -1
@@ -29,7 +29,7 @@ You can build and run vLLM from source via the provided dockerfile. To build vLL
29
30 .. code-block:: console
31
32 - $ DOCKER_BUILDKIT=1 docker build . --target vllm-openai --tag vllm/vllm-openai --build-arg max_jobs=8
32 + $ DOCKER_BUILDKIT=1 docker build . --target vllm-openai --tag vllm/vllm-openai # optionally specifies: --build-arg max_jobs=8 --build-arg nvcc_threads=2
33
34 To run vLLM:
35
setup.py
+2 -1
@@ -138,7 +138,8 @@ for capability in compute_capabilities:
138
139 # Use NVCC threads to parallelize the build.
140 if nvcc_cuda_version >= Version("11.2"):
141 - num_threads = min(os.cpu_count(), 8)
141 + nvcc_threads = int(os.getenv("NVCC_THREADS"), 8)
142 + num_threads = min(os.cpu_count(), nvcc_threads)
143 NVCC_FLAGS += ["--threads", str(num_threads)]
144
145 ext_modules = []