@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d03d64fd2e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[CI/Build] refactor dockerfile & fix pip cache

[CI/Build] fix pip cache with vllm_nccl & refactor dockerfile to build wheels (#3859)

youkaichao committed Apr 4, 2024 at 21:53 UTC d03d64fd2e22f1a48e7b78c66d7644e6b6230fb7
3 files changed +49 -41
.buildkite/test-pipeline.yaml
+1 -1
@@ -90,7 +90,7 @@ steps:
90 - bash run-benchmarks.sh
91
92 - label: Documentation Build
93 - working_dir: "/vllm-workspace/docs"
93 + working_dir: "/vllm-workspace/test_docs/docs"
94 no_gpu: True
95 commands:
96 - pip install -r requirements-docs.txt
Dockerfile
+48 -37
@@ -2,6 +2,7 @@
2 # to run the OpenAI compatible server.
3
4 #################### BASE BUILD IMAGE ####################
5 +# prepare basic build environment
6 FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
7
8 RUN apt-get update -y \
@@ -34,7 +35,7 @@ ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
35 #################### BASE BUILD IMAGE ####################
36
37
37 -#################### EXTENSION BUILD IMAGE ####################
38 +#################### WHEEL BUILD IMAGE ####################
39 FROM dev AS build
40
41 # install build dependencies
@@ -45,14 +46,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \
46 # install compiler cache to speed up compilation leveraging local or remote caching
47 RUN apt-get update -y && apt-get install -y ccache
48
48 -# copy input files
49 +# files and directories related to build wheels
50 COPY csrc csrc
51 COPY setup.py setup.py
52 COPY cmake cmake
53 COPY CMakeLists.txt CMakeLists.txt
54 COPY requirements.txt requirements.txt
55 COPY pyproject.toml pyproject.toml
55 -COPY vllm/__init__.py vllm/__init__.py
56 +COPY vllm vllm
57
58 # max jobs used by Ninja to build extensions
59 ARG max_jobs=2
@@ -65,7 +66,15 @@ ENV VLLM_INSTALL_PUNICA_KERNELS=1
66
67 ENV CCACHE_DIR=/root/.cache/ccache
68 RUN --mount=type=cache,target=/root/.cache/ccache \
68 - python3 setup.py build_ext --inplace
69 + --mount=type=cache,target=/root/.cache/pip \
70 + python3 setup.py bdist_wheel --dist-dir=dist
71 +
72 +# the `vllm_nccl` package must be installed from source distribution
73 +# pip is too smart to store a wheel in the cache, and other CI jobs
74 +# will directly use the wheel from the cache, which is not what we want.
75 +# we need to remove it manually
76 +RUN --mount=type=cache,target=/root/.cache/pip \
77 + pip cache remove vllm_nccl*
78 #################### EXTENSION Build IMAGE ####################
79
80 #################### FLASH_ATTENTION Build IMAGE ####################
@@ -85,57 +94,59 @@ RUN pip --verbose wheel flash-attn==${FLASH_ATTN_VERSION} \
94
95 #################### FLASH_ATTENTION Build IMAGE ####################
96
88 -#################### TEST IMAGE ####################
89 -# image to run unit testing suite
90 -FROM dev AS test
91 -
92 -# copy pytorch extensions separately to avoid having to rebuild
93 -# when python code changes
97 +#################### vLLM installation IMAGE ####################
98 +# image with vLLM installed
99 +FROM nvidia/cuda:12.1.0-base-ubuntu22.04 AS vllm-base
100 WORKDIR /vllm-workspace
95 -# ADD is used to preserve directory structure
96 -ADD . /vllm-workspace/
97 -COPY --from=build /workspace/vllm/*.so /vllm-workspace/vllm/
98 -# Install flash attention (from pre-built wheel)
101 +
102 +RUN apt-get update -y \
103 + && apt-get install -y python3-pip git vim
104 +
105 +# Workaround for https://github.com/openai/triton/issues/2507 and
106 +# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
107 +# this won't be needed for future versions of this docker image
108 +# or future versions of triton.
109 +RUN ldconfig /usr/local/cuda-12.1/compat/
110 +
111 +# install vllm wheel first, so that torch etc will be installed
112 +RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
113 + --mount=type=cache,target=/root/.cache/pip \
114 + pip install dist/*.whl --verbose
115 +
116 RUN --mount=type=bind,from=flash-attn-builder,src=/usr/src/flash-attention-v2,target=/usr/src/flash-attention-v2 \
117 + --mount=type=cache,target=/root/.cache/pip \
118 pip install /usr/src/flash-attention-v2/*.whl --no-cache-dir
101 -# ignore build dependencies installation because we are using pre-complied extensions
102 -RUN rm pyproject.toml
103 -RUN --mount=type=cache,target=/root/.cache/pip VLLM_USE_PRECOMPILED=1 pip install . --verbose
104 -#################### TEST IMAGE ####################
119 +#################### vLLM installation IMAGE ####################
120
121
107 -#################### RUNTIME BASE IMAGE ####################
108 -# We used base cuda image because pytorch installs its own cuda libraries.
109 -# However pynccl depends on cuda libraries so we had to switch to the runtime image
110 -# In the future it would be nice to get a container with pytorch and cuda without duplicating cuda
111 -FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS vllm-base
122 +#################### TEST IMAGE ####################
123 +# image to run unit testing suite
124 +# note that this uses vllm installed by `pip`
125 +FROM vllm-base AS test
126
113 -# libnccl required for ray
114 -RUN apt-get update -y \
115 - && apt-get install -y python3-pip
127 +ADD . /vllm-workspace/
128
117 -WORKDIR /workspace
118 -COPY requirements.txt requirements.txt
129 +# install development dependencies (for testing)
130 RUN --mount=type=cache,target=/root/.cache/pip \
120 - pip install -r requirements.txt
121 -
122 -# Install flash attention (from pre-built wheel)
123 -RUN --mount=type=bind,from=flash-attn-builder,src=/usr/src/flash-attention-v2,target=/usr/src/flash-attention-v2 \
124 - pip install /usr/src/flash-attention-v2/*.whl --no-cache-dir
131 + pip install -r requirements-dev.txt
132
126 -#################### RUNTIME BASE IMAGE ####################
133 +# doc requires source code
134 +# we hide them inside `test_docs/` , so that this source code
135 +# will not be imported by other tests
136 +RUN mkdir test_docs
137 +RUN mv docs test_docs/
138 +RUN mv vllm test_docs/
139
140 +#################### TEST IMAGE ####################
141
142 #################### OPENAI API SERVER ####################
143 # openai api server alternative
144 FROM vllm-base AS vllm-openai
145 +
146 # install additional dependencies for openai api server
147 RUN --mount=type=cache,target=/root/.cache/pip \
148 pip install accelerate hf_transfer modelscope
149
136 -COPY --from=build /workspace/vllm/*.so /workspace/vllm/
137 -COPY vllm vllm
138 -
150 ENV VLLM_USAGE_SOURCE production-docker-image
151
152 ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
docs/source/conf.py
-3
@@ -11,13 +11,10 @@
11 # documentation root, use os.path.abspath to make it absolute, like shown here.
12
13 import logging
14 -import os
14 import sys
15
16 from sphinx.ext import autodoc
17
19 -sys.path.insert(0, os.path.abspath(os.path.join('..', '..')))
20 -
18 logger = logging.getLogger(__name__)
19
20 # -- Project information -----------------------------------------------------