A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[CI/Build] refactor dockerfile & fix pip cache
[CI/Build] fix pip cache with vllm_nccl & refactor dockerfile to build wheels (#3859)
youkaichao committed
Apr 4, 2024 at 21:53 UTC
d03d64fd2e22f1a48e7b78c66d7644e6b6230fb7
3 files changed
+49
-41
.buildkite/test-pipeline.yaml
+1
-1
@@ -90,7 +90,7 @@ steps:
90
- bash run-benchmarks.sh
91
92
- label: Documentation Build
93
- working_dir: "/vllm-workspace/docs"
93
+ working_dir: "/vllm-workspace/test_docs/docs"
94
no_gpu: True
95
commands:
96
- pip install -r requirements-docs.txt
Dockerfile
+48
-37
@@ -2,6 +2,7 @@
2
# to run the OpenAI compatible server.
3
4
#################### BASE BUILD IMAGE ####################
5
+# prepare basic build environment
6
FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
7
8
RUN apt-get update -y \
@@ -34,7 +35,7 @@ ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
35
#################### BASE BUILD IMAGE ####################
36
37
37
-#################### EXTENSION BUILD IMAGE ####################
38
+#################### WHEEL BUILD IMAGE ####################
39
FROM dev AS build
40
41
# install build dependencies
@@ -45,14 +46,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \
46
# install compiler cache to speed up compilation leveraging local or remote caching
47
RUN apt-get update -y && apt-get install -y ccache
48
48
-# copy input files
49
+# files and directories related to build wheels
50
COPY csrc csrc
51
COPY setup.py setup.py
52
COPY cmake cmake
53
COPY CMakeLists.txt CMakeLists.txt
54
COPY requirements.txt requirements.txt
55
COPY pyproject.toml pyproject.toml
55
-COPY vllm/__init__.py vllm/__init__.py
56
+COPY vllm vllm
57
58
# max jobs used by Ninja to build extensions
59
ARG max_jobs=2
@@ -65,7 +66,15 @@ ENV VLLM_INSTALL_PUNICA_KERNELS=1
66
67
ENV CCACHE_DIR=/root/.cache/ccache
68
RUN --mount=type=cache,target=/root/.cache/ccache \
68
- python3 setup.py build_ext --inplace
69
+ --mount=type=cache,target=/root/.cache/pip \
70
+ python3 setup.py bdist_wheel --dist-dir=dist
71
+
72
+# the `vllm_nccl` package must be installed from source distribution
73
+# pip is too smart to store a wheel in the cache, and other CI jobs
74
+# will directly use the wheel from the cache, which is not what we want.
75
+# we need to remove it manually
76
+RUN --mount=type=cache,target=/root/.cache/pip \
77
+ pip cache remove vllm_nccl*
78
#################### EXTENSION Build IMAGE ####################
79
80
#################### FLASH_ATTENTION Build IMAGE ####################
@@ -85,57 +94,59 @@ RUN pip --verbose wheel flash-attn==${FLASH_ATTN_VERSION} \
94
95
#################### FLASH_ATTENTION Build IMAGE ####################
96
88
-#################### TEST IMAGE ####################
89
-# image to run unit testing suite
90
-FROM dev AS test
91
-
92
-# copy pytorch extensions separately to avoid having to rebuild
93
-# when python code changes
97
+#################### vLLM installation IMAGE ####################
98
+# image with vLLM installed
99
+FROM nvidia/cuda:12.1.0-base-ubuntu22.04 AS vllm-base
100
WORKDIR /vllm-workspace
95
-# ADD is used to preserve directory structure
96
-ADD . /vllm-workspace/
97
-COPY --from=build /workspace/vllm/*.so /vllm-workspace/vllm/
98
-# Install flash attention (from pre-built wheel)
101
+
102
+RUN apt-get update -y \
103
+ && apt-get install -y python3-pip git vim
104
+
105
+# Workaround for https://github.com/openai/triton/issues/2507 and
106
+# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
107
+# this won't be needed for future versions of this docker image
108
+# or future versions of triton.
109
+RUN ldconfig /usr/local/cuda-12.1/compat/
110
+
111
+# install vllm wheel first, so that torch etc will be installed
112
+RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
113
+ --mount=type=cache,target=/root/.cache/pip \
114
+ pip install dist/*.whl --verbose
115
+
116
RUN --mount=type=bind,from=flash-attn-builder,src=/usr/src/flash-attention-v2,target=/usr/src/flash-attention-v2 \
117
+ --mount=type=cache,target=/root/.cache/pip \
118
pip install /usr/src/flash-attention-v2/*.whl --no-cache-dir
101
-# ignore build dependencies installation because we are using pre-complied extensions
102
-RUN rm pyproject.toml
103
-RUN --mount=type=cache,target=/root/.cache/pip VLLM_USE_PRECOMPILED=1 pip install . --verbose
104
-#################### TEST IMAGE ####################
119
+#################### vLLM installation IMAGE ####################
120
121
107
-#################### RUNTIME BASE IMAGE ####################
108
-# We used base cuda image because pytorch installs its own cuda libraries.
109
-# However pynccl depends on cuda libraries so we had to switch to the runtime image
110
-# In the future it would be nice to get a container with pytorch and cuda without duplicating cuda
111
-FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS vllm-base
122
+#################### TEST IMAGE ####################
123
+# image to run unit testing suite
124
+# note that this uses vllm installed by `pip`
125
+FROM vllm-base AS test
126
113
-# libnccl required for ray
114
-RUN apt-get update -y \
115
- && apt-get install -y python3-pip
127
+ADD . /vllm-workspace/
128
117
-WORKDIR /workspace
118
-COPY requirements.txt requirements.txt
129
+# install development dependencies (for testing)
130
RUN --mount=type=cache,target=/root/.cache/pip \
120
- pip install -r requirements.txt
121
-
122
-# Install flash attention (from pre-built wheel)
123
-RUN --mount=type=bind,from=flash-attn-builder,src=/usr/src/flash-attention-v2,target=/usr/src/flash-attention-v2 \
124
- pip install /usr/src/flash-attention-v2/*.whl --no-cache-dir
131
+ pip install -r requirements-dev.txt
132
126
-#################### RUNTIME BASE IMAGE ####################
133
+# doc requires source code
134
+# we hide them inside `test_docs/` , so that this source code
135
+# will not be imported by other tests
136
+RUN mkdir test_docs
137
+RUN mv docs test_docs/
138
+RUN mv vllm test_docs/
139
140
+#################### TEST IMAGE ####################
141
142
#################### OPENAI API SERVER ####################
143
# openai api server alternative
144
FROM vllm-base AS vllm-openai
145
+
146
# install additional dependencies for openai api server
147
RUN --mount=type=cache,target=/root/.cache/pip \
148
pip install accelerate hf_transfer modelscope
149
136
-COPY --from=build /workspace/vllm/*.so /workspace/vllm/
137
-COPY vllm vllm
138
-
150
ENV VLLM_USAGE_SOURCE production-docker-image
151
152
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
docs/source/conf.py
-3
@@ -11,13 +11,10 @@
11
# documentation root, use os.path.abspath to make it absolute, like shown here.
12
13
import logging
14
-import os
14
import sys
15
16
from sphinx.ext import autodoc
17
19
-sys.path.insert(0, os.path.abspath(os.path.join('..', '..')))
20
-
18
logger = logging.getLogger(__name__)
19
20
# -- Project information -----------------------------------------------------