A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[ci][gh200] dockerfile clean up (#11351)
Signed-off-by: drikster80 <ed.sealing@gmail.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: drikster80 <ed.sealing@gmail.com> Co-authored-by: cenzhiyao <2523403608@qq.com>
youkaichao committed
Dec 19, 2024 at 18:13 UTC
7801f56ed76d9bec0344728bfa3359b42c926074
7 files changed
+51
-37
.buildkite/run-gh200-test.sh
+3
@@ -4,6 +4,9 @@
4
# It serves a sanity check for compilation and basic model usage.
5
set -ex
6
7
+# Skip the new torch installation during build since we are using the specified version for arm64 in the Dockerfile
8
+python3 use_existing_torch.py
9
+
10
# Try building the docker image
11
DOCKER_BUILDKIT=1 docker build . \
12
--target vllm-openai \
Dockerfile
+20
-19
@@ -45,17 +45,21 @@ RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
45
WORKDIR /workspace
46
47
# install build and runtime dependencies
48
-COPY requirements-common.txt requirements-common.txt
49
-COPY requirements-cuda.txt requirements-cuda.txt
50
-COPY requirements-cuda-arm64.txt requirements-cuda-arm64.txt
51
-RUN --mount=type=cache,target=/root/.cache/pip \
52
- python3 -m pip install -r requirements-cuda.txt
48
49
+# arm64 (GH200) build follows the practice of "use existing pytorch" build,
50
+# we need to install torch and torchvision from the nightly builds first,
51
+# pytorch will not appear as a vLLM dependency in all of the following steps
52
+# after this step
53
RUN --mount=type=cache,target=/root/.cache/pip \
54
if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
56
- python3 -m pip install -r requirements-cuda-arm64.txt; \
55
+ python3 -m pip install --index-url https://download.pytorch.org/whl/nightly/cu124 "torch==2.6.0.dev20241210+cu124" "torchvision==0.22.0.dev20241215"; \
56
fi
57
58
+COPY requirements-common.txt requirements-common.txt
59
+COPY requirements-cuda.txt requirements-cuda.txt
60
+RUN --mount=type=cache,target=/root/.cache/pip \
61
+ python3 -m pip install -r requirements-cuda.txt
62
+
63
# cuda arch list used by torch
64
# can be useful for both `dev` and `test`
65
# explicitly set the list to avoid issues with torch 2.2
@@ -77,11 +81,6 @@ COPY requirements-build.txt requirements-build.txt
81
RUN --mount=type=cache,target=/root/.cache/pip \
82
python3 -m pip install -r requirements-build.txt
83
80
-RUN --mount=type=cache,target=/root/.cache/pip \
81
- if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
82
- python3 -m pip install -r requirements-cuda-arm64.txt; \
83
- fi
84
-
84
COPY . .
85
ARG GIT_REPO_CHECK=0
86
RUN --mount=type=bind,source=.git,target=.git \
@@ -157,8 +156,6 @@ WORKDIR /vllm-workspace
156
ENV DEBIAN_FRONTEND=noninteractive
157
ARG TARGETPLATFORM
158
160
-COPY requirements-cuda-arm64.txt requirements-cuda-arm64.txt
161
-
159
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
160
echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
161
@@ -183,17 +180,20 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
180
# or future versions of triton.
181
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
182
183
+# arm64 (GH200) build follows the practice of "use existing pytorch" build,
184
+# we need to install torch and torchvision from the nightly builds first,
185
+# pytorch will not appear as a vLLM dependency in all of the following steps
186
+# after this step
187
+RUN --mount=type=cache,target=/root/.cache/pip \
188
+ if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
189
+ python3 -m pip install --index-url https://download.pytorch.org/whl/nightly/cu124 "torch==2.6.0.dev20241210+cu124" "torchvision==0.22.0.dev20241215"; \
190
+ fi
191
+
192
# Install vllm wheel first, so that torch etc will be installed.
193
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
194
--mount=type=cache,target=/root/.cache/pip \
195
python3 -m pip install dist/*.whl --verbose
196
191
-RUN --mount=type=cache,target=/root/.cache/pip \
192
- if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
193
- pip uninstall -y torch && \
194
- python3 -m pip install -r requirements-cuda-arm64.txt; \
195
- fi
196
-
197
RUN --mount=type=cache,target=/root/.cache/pip \
198
. /etc/environment && \
199
if [ "$TARGETPLATFORM" != "linux/arm64" ]; then \
@@ -244,6 +244,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
244
else \
245
pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' 'timm==0.9.10'; \
246
fi
247
+
248
ENV VLLM_USAGE_SOURCE production-docker-image
249
250
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
docs/source/serving/deploying_with_docker.rst
+21
-9
@@ -3,6 +3,9 @@
3
Deploying with Docker
4
============================
5
6
+Use vLLM's Official Docker Image
7
+--------------------------------
8
+
9
vLLM offers an official Docker image for deployment.
10
The image can be used to run OpenAI compatible server and is available on Docker Hub as `vllm/vllm-openai <https://hub.docker.com/r/vllm/vllm-openai/tags>`_.
11
@@ -24,12 +27,15 @@ The image can be used to run OpenAI compatible server and is available on Docker
27
memory to share data between processes under the hood, particularly for tensor parallel inference.
28
29
30
+Building vLLM's Docker Image from Source
31
+----------------------------------------
32
+
33
You can build and run vLLM from source via the provided `Dockerfile <https://github.com/vllm-project/vllm/blob/main/Dockerfile>`_. To build vLLM:
34
35
.. code-block:: console
36
31
- $ DOCKER_BUILDKIT=1 docker build . --target vllm-openai --tag vllm/vllm-openai # optionally specifies: --build-arg max_jobs=8 --build-arg nvcc_threads=2
32
-
37
+ $ # optionally specifies: --build-arg max_jobs=8 --build-arg nvcc_threads=2
38
+ $ DOCKER_BUILDKIT=1 docker build . --target vllm-openai --tag vllm/vllm-openai
39
40
.. note::
41
@@ -41,18 +47,19 @@ Building for Arm64/aarch64
47
--------------------------
48
49
A docker container can be built for aarch64 systems such as the Nvidia Grace-Hopper. At time of this writing, this requires the use
44
-of PyTorch Nightly and should be considered **experimental**. Using the flag `--platform "linux/arm64"` will attempt to build for arm64.
50
+of PyTorch Nightly and should be considered **experimental**. Using the flag ``--platform "linux/arm64"`` will attempt to build for arm64.
51
52
.. note::
53
48
- Multiple modules must be compiled, so this process can take a while. Recommend using `--build-arg max_jobs=` & `--build-arg nvcc_threads=`
49
- flags to speed up build process. However, ensure your 'max_jobs' is substantially larger than 'nvcc_threads' to get the most benefits.
54
+ Multiple modules must be compiled, so this process can take a while. Recommend using ``--build-arg max_jobs=`` & ``--build-arg nvcc_threads=``
55
+ flags to speed up build process. However, ensure your ``max_jobs`` is substantially larger than ``nvcc_threads`` to get the most benefits.
56
Keep an eye on memory usage with parallel jobs as it can be substantial (see example below).
57
58
.. code-block:: console
59
54
- # Example of building on Nvidia GH200 server. (Memory usage: ~12GB, Build time: ~1475s / ~25 min, Image size: 7.26GB)
55
- $ DOCKER_BUILDKIT=1 sudo docker build . \
60
+ # Example of building on Nvidia GH200 server. (Memory usage: ~15GB, Build time: ~1475s / ~25 min, Image size: 6.93GB)
61
+ $ python3 use_existing_torch.py
62
+ $ DOCKER_BUILDKIT=1 docker build . \
63
--target vllm-openai \
64
--platform "linux/arm64" \
65
-t vllm/vllm-gh200-openai:latest \
@@ -61,7 +68,10 @@ of PyTorch Nightly and should be considered **experimental**. Using the flag `--
68
--build-arg torch_cuda_arch_list="9.0+PTX" \
69
--build-arg vllm_fa_cmake_gpu_arches="90-real"
70
64
-To run vLLM:
71
+Use the custom-built vLLM Docker image
72
+--------------------------------------
73
+
74
+To run vLLM with the custom-built Docker image:
75
76
.. code-block:: console
77
@@ -71,6 +81,8 @@ To run vLLM:
81
--env "HUGGING_FACE_HUB_TOKEN=<secret>" \
82
vllm/vllm-openai <args...>
83
84
+The argument ``vllm/vllm-openai`` specifies the image to run, and should be replaced with the name of the custom-built image (the ``-t`` tag from the build command).
85
+
86
.. note::
87
76
- **For `v0.4.1` and `v0.4.2` only** - the vLLM docker images under these versions are supposed to be run under the root user since a library under the root user's home directory, i.e. ``/root/.config/vllm/nccl/cu12/libnccl.so.2.18.1`` is required to be loaded during runtime. If you are running the container under a different user, you may need to first change the permissions of the library (and all the parent directories) to allow the user to access it, then run vLLM with environment variable ``VLLM_NCCL_SO_PATH=/root/.config/vllm/nccl/cu12/libnccl.so.2.18.1`` .
88
+ **For version 0.4.1 and 0.4.2 only** - the vLLM docker images under these versions are supposed to be run under the root user since a library under the root user's home directory, i.e. ``/root/.config/vllm/nccl/cu12/libnccl.so.2.18.1`` is required to be loaded during runtime. If you are running the container under a different user, you may need to first change the permissions of the library (and all the parent directories) to allow the user to access it, then run vLLM with environment variable ``VLLM_NCCL_SO_PATH=/root/.config/vllm/nccl/cu12/libnccl.so.2.18.1`` .
requirements-build.txt
+1
-1
@@ -4,6 +4,6 @@ ninja
4
packaging
5
setuptools>=61
6
setuptools-scm>=8
7
-torch==2.5.1; platform_machine != 'aarch64'
7
+torch==2.5.1
8
wheel
9
jinja2
requirements-common.txt
+4
-3
@@ -19,7 +19,7 @@ pillow # Required for image processing
19
prometheus-fastapi-instrumentator >= 7.0.0
20
tiktoken >= 0.6.0 # Required for DBRX tokenizer
21
lm-format-enforcer >= 0.10.9, < 0.11
22
-outlines == 0.1.11
22
+outlines == 0.1.11 # Requires pytorch
23
lark == 1.2.2
24
xgrammar >= 0.1.6; platform_machine == "x86_64"
25
typing_extensions >= 4.10
@@ -34,5 +34,6 @@ pyyaml
34
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
35
setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
36
einops # Required for Qwen2-VL.
37
-compressed-tensors == 0.8.1 # required for compressed-tensors
38
-depyf==0.18.0 # required for profiling and debugging torch.compile
37
+compressed-tensors == 0.8.1 # required for compressed-tensors, requires pytorch
38
+depyf==0.18.0 # required for profiling and debugging with compilation config
39
+cloudpickle # allows pickling lambda functions in model_executor/models/registry.py
requirements-cuda-arm64.txt
deleted
-3
@@ -1,3 +0,0 @@
1
---index-url https://download.pytorch.org/whl/nightly/cu124
2
-torchvision==0.22.0.dev20241215; platform_machine == 'aarch64'
3
-torch==2.6.0.dev20241210+cu124; platform_machine == 'aarch64'
requirements-cuda.txt
+2
-2
@@ -4,7 +4,7 @@
4
# Dependencies for NVIDIA GPUs
5
ray >= 2.9
6
nvidia-ml-py >= 12.560.30 # for pynvml package
7
-torch == 2.5.1; platform_machine != 'aarch64'
7
+torch == 2.5.1
8
# These must be updated alongside torch
9
-torchvision == 0.20.1; platform_machine != 'aarch64' # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
9
+torchvision == 0.20.1 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
xformers == 0.0.28.post3; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.5.1