| 1 | # The vLLM Dockerfile is used to construct vLLM image that can be directly used |
| 2 | # to run the OpenAI compatible server. |
| 3 | |
| 4 | # Please update any changes made here to |
| 5 | # docs/contributing/dockerfile/dockerfile.md and |
| 6 | # docs/assets/contributing/dockerfile-stages-dependency.png |
| 7 | |
| 8 | # ============================================================================= |
| 9 | # VERSION MANAGEMENT |
| 10 | # ============================================================================= |
| 11 | # ARG defaults in this Dockerfile are the source of truth for pinned versions. |
| 12 | # docker/versions.json is auto-generated for use with docker buildx bake. |
| 13 | # |
| 14 | # When updating versions: |
| 15 | # 1. Edit the ARG defaults below |
| 16 | # 2. Run: python tools/generate_versions_json.py |
| 17 | # |
| 18 | # To query versions programmatically: |
| 19 | # jq -r '.variable.CUDA_VERSION.default' docker/versions.json |
| 20 | # |
| 21 | # To build with bake: |
| 22 | # docker buildx bake -f docker/docker-bake.hcl -f docker/versions.json |
| 23 | # ============================================================================= |
| 24 | |
| 25 | ARG CUDA_VERSION=13.0.3 |
| 26 | ARG PYTHON_VERSION=3.12 |
| 27 | ARG UBUNTU_VERSION=24.04 |
| 28 | ARG INSTALL_RUBIN_PRERELEASE=false |
| 29 | ARG TRITON_INSTALL_FROM_SOURCE_REPO= |
| 30 | ARG TRITON_INSTALL_FROM_SOURCE_REVISION= |
| 31 | # DeepEPv2 requires NCCL >= 2.30.4 (GIN backend). |
| 32 | # This version is used for CUDA 12+ builds to keep the DeepEP build and |
| 33 | # runtime NCCL versions compatible. |
| 34 | ARG NCCL_VERSION=2.30.7 |
| 35 | |
| 36 | # By parameterizing the base images, we allow third-party to use their own |
| 37 | # base images. One use case is hermetic builds with base images stored in |
| 38 | # private registries that use a different repository naming conventions. |
| 39 | # |
| 40 | # Example: |
| 41 | # docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/pytorch/manylinux2_28-builder:cuda13.0 |
| 42 | |
| 43 | # Build wheels against the same glibc floor as PyTorch's published wheels. |
| 44 | ARG BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359 |
| 45 | # Using cuda base image with minimal dependencies necessary for JIT compilation (FlashInfer, DeepGEMM, EP kernels) |
| 46 | ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-base-ubuntu${UBUNTU_VERSION} |
| 47 | |
| 48 | # By parameterizing the Deadsnakes repository URL, we allow third-party to use |
| 49 | # their own mirror. When doing so, we don't benefit from the transparent |
| 50 | # installation of the GPG key of the PPA, as done by add-apt-repository, so we |
| 51 | # also need a URL for the GPG key. |
| 52 | ARG DEADSNAKES_MIRROR_URL |
| 53 | ARG DEADSNAKES_GPGKEY_URL |
| 54 | |
| 55 | # The PyPA get-pip.py script is a self contained script+zip file, that provides |
| 56 | # both the installer script and the pip base85-encoded zip archive. This allows |
| 57 | # bootstrapping pip in environment where a distribution package does not exist. |
| 58 | # |
| 59 | # By parameterizing the URL for get-pip.py installation script, we allow |
| 60 | # third-party to use their own copy of the script stored in a private mirror. |
| 61 | # We set the default value to the PyPA owned get-pip.py script. |
| 62 | # |
| 63 | # Reference: https://pip.pypa.io/en/stable/installation/#get-pip-py |
| 64 | ARG GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py" |
| 65 | |
| 66 | # PIP supports fetching the packages from custom indexes, allowing third-party |
| 67 | # to host the packages in private mirrors. The PIP_INDEX_URL and |
| 68 | # PIP_EXTRA_INDEX_URL are standard PIP environment variables to override the |
| 69 | # default indexes. By letting them empty by default, PIP will use its default |
| 70 | # indexes if the build process doesn't override the indexes. |
| 71 | # |
| 72 | # Uv uses different variables. We set them by default to the same values as |
| 73 | # PIP, but they can be overridden. |
| 74 | ARG PIP_INDEX_URL |
| 75 | ARG PIP_EXTRA_INDEX_URL |
| 76 | ARG UV_INDEX_URL=${PIP_INDEX_URL} |
| 77 | ARG UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} |
| 78 | |
| 79 | # PyTorch provides its own indexes for standard and nightly builds |
| 80 | ARG PYTORCH_CUDA_INDEX_BASE_URL=https://download.pytorch.org/whl |
| 81 | |
| 82 | # PIP supports multiple authentication schemes, including keyring |
| 83 | # By parameterizing the PIP_KEYRING_PROVIDER variable and setting it to |
| 84 | # disabled by default, we allow third-party to use keyring authentication for |
| 85 | # their private Python indexes, while not changing the default behavior which |
| 86 | # is no authentication. |
| 87 | # |
| 88 | # Reference: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support |
| 89 | ARG PIP_KEYRING_PROVIDER=disabled |
| 90 | ARG UV_KEYRING_PROVIDER=${PIP_KEYRING_PROVIDER} |
| 91 | |
| 92 | # Flag enables built-in KV-connector dependency libs into docker images |
| 93 | ARG INSTALL_KV_CONNECTORS=false |
| 94 | |
| 95 | #################### BASE BUILD IMAGE #################### |
| 96 | # prepare basic build environment |
| 97 | FROM ${BUILD_BASE_IMAGE} AS base |
| 98 | |
| 99 | ARG TARGETPLATFORM |
| 100 | ARG INSTALL_RUBIN_PRERELEASE |
| 101 | ARG CUDA_VERSION |
| 102 | ARG PYTHON_VERSION |
| 103 | ARG USE_SCCACHE |
| 104 | ARG SCCACHE_DOWNLOAD_URL |
| 105 | ARG SCCACHE_ENDPOINT |
| 106 | ARG SCCACHE_BUCKET_NAME=vllm-build-sccache |
| 107 | ARG SCCACHE_REGION_NAME=us-west-2 |
| 108 | ARG SCCACHE_S3_NO_CREDENTIALS=0 |
| 109 | |
| 110 | # Environment for uv |
| 111 | # Declared BEFORE the installer + `uv venv` invocations below so the uv |
| 112 | # binary, managed Python, download cache, and /opt/venv all land under |
| 113 | # /opt/uv instead of /root/.local/. Without this, the venv created at |
| 114 | # build time hardlinks back to /root/.local/share/uv/python and |
| 115 | # descendants of this stage (`build`, `dev`, `csrc-build`, |
| 116 | # `extensions-build`) inherit a root-owned, non-root-unreadable layout. |
| 117 | # See #15174, #15359, #31959. Child stages inherit these via Dockerfile |
| 118 | # `ENV` unless they override them explicitly. |
| 119 | ENV UV_HTTP_TIMEOUT=500 |
| 120 | ENV UV_INDEX_STRATEGY="unsafe-best-match" |
| 121 | ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python |
| 122 | ENV UV_CACHE_DIR=/opt/uv/cache |
| 123 | ENV UV_INSTALL_DIR=/opt/uv/bin |
| 124 | ENV PATH="/opt/venv/bin:/opt/uv/bin:$PATH" |
| 125 | ENV VIRTUAL_ENV="/opt/venv" |
| 126 | |
| 127 | # Install system dependencies including build tools. The PyTorch manylinux |
| 128 | # image enables EPEL, which provides ccache. |
| 129 | RUN dnf install -y --setopt=install_weak_deps=False \ |
| 130 | ccache \ |
| 131 | git \ |
| 132 | curl \ |
| 133 | sudo \ |
| 134 | elfutils-devel \ |
| 135 | rdma-core-devel \ |
| 136 | && dnf clean all \ |
| 137 | && rm -rf /var/cache/dnf |
| 138 | |
| 139 | # Install sccache once in base so Rust and CMake/CUDA build stages share the |
| 140 | # same binary and remote cache configuration. |
| 141 | RUN if [ "$USE_SCCACHE" = "1" ]; then \ |
| 142 | echo "Installing sccache..." \ |
| 143 | && case "${TARGETPLATFORM}" in \ |
| 144 | linux/arm64) SCCACHE_ARCH="aarch64" ;; \ |
| 145 | linux/amd64) SCCACHE_ARCH="x86_64" ;; \ |
| 146 | *) echo "Unsupported TARGETPLATFORM for sccache: ${TARGETPLATFORM}" >&2; exit 1 ;; \ |
| 147 | esac \ |
| 148 | && export SCCACHE_DOWNLOAD_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \ |
| 149 | && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \ |
| 150 | && tar -xzf sccache.tar.gz \ |
| 151 | && sudo mv sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \ |
| 152 | && rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \ |
| 153 | fi |
| 154 | |
| 155 | ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}} |
| 156 | ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} |
| 157 | ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}} |
| 158 | ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0} |
| 159 | |
| 160 | # Install uv and bootstrap /opt/venv from the Python interpreter included in |
| 161 | # the manylinux image. |
| 162 | RUN mkdir -p "${UV_PYTHON_INSTALL_DIR}" "${UV_CACHE_DIR}" "${UV_INSTALL_DIR}" \ |
| 163 | && chmod -R a+rX /opt/uv \ |
| 164 | && curl -LsSf https://astral.sh/uv/install.sh | sh \ |
| 165 | # `--seed` installs pip/setuptools/wheel into the venv so `python3 -m |
| 166 | # pip` works regardless of how uv happens to link the venv back to the |
| 167 | # managed Python install (which, at a non-default UV_PYTHON_INSTALL_DIR, |
| 168 | # doesn't always expose ensurepip via the default venv layout). |
| 169 | && PYV_NODOT=$(echo ${PYTHON_VERSION} | tr -d '.') \ |
| 170 | && MANYLINUX_PY=/opt/python/cp${PYV_NODOT}-cp${PYV_NODOT}/bin/python${PYTHON_VERSION} \ |
| 171 | && uv venv --seed /opt/venv --python "$MANYLINUX_PY" \ |
| 172 | && rm -f /usr/bin/python3 /usr/bin/python3-config /usr/bin/pip \ |
| 173 | && ln -sf /opt/venv/bin/python3 /usr/bin/python3 \ |
| 174 | && ln -sf /opt/venv/bin/python3-config /usr/bin/python3-config \ |
| 175 | && ln -sf /opt/venv/bin/pip /usr/bin/pip \ |
| 176 | && python3 --version && python3 -m pip --version |
| 177 | |
| 178 | # UV_LINK_MODE=copy applies to subsequent `uv pip install` RUNs (avoids |
| 179 | # hardlink failures with BuildKit cache mounts); it must not be set during |
| 180 | # `uv venv` above, which relies on hardlinking /opt/venv back to the |
| 181 | # managed Python source so ensurepip / `python3 -m pip` still resolve. |
| 182 | ENV UV_LINK_MODE=copy |
| 183 | |
| 184 | # Verify GCC version |
| 185 | RUN gcc --version |
| 186 | |
| 187 | # Enable CUDA forward compatibility by setting '-e VLLM_ENABLE_CUDA_COMPATIBILITY=1' |
| 188 | # Only needed for datacenter/professional GPUs with older drivers. |
| 189 | # See: https://docs.nvidia.com/deploy/cuda-compatibility/ |
| 190 | ENV VLLM_ENABLE_CUDA_COMPATIBILITY=0 |
| 191 | |
| 192 | # ============================================================ |
| 193 | # SLOW-CHANGING DEPENDENCIES BELOW |
| 194 | # These are the expensive layers that we want to cache |
| 195 | # ============================================================ |
| 196 | |
| 197 | # Install PyTorch and core CUDA dependencies |
| 198 | # This is ~2GB and rarely changes |
| 199 | ARG PYTORCH_CUDA_INDEX_BASE_URL |
| 200 | |
| 201 | WORKDIR /workspace |
| 202 | |
| 203 | # We can specify the standard or nightly build of PyTorch |
| 204 | ARG PYTORCH_NIGHTLY |
| 205 | |
| 206 | # Install build and runtime dependencies, including PyTorch |
| 207 | # Check whether to install torch nightly instead of release for this build |
| 208 | COPY requirements/common.txt requirements/common.txt |
| 209 | COPY requirements/cuda.txt requirements/cuda.txt |
| 210 | COPY tools/use_existing_torch.py tools/use_existing_torch.py |
| 211 | COPY pyproject.toml pyproject.toml |
| 212 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 213 | --mount=type=bind,source=requirements,target=/tmp/vllm-requirements,ro \ |
| 214 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ] && \ |
| 215 | [ ! -f /tmp/vllm-requirements/rubin-prerelease.txt ]; then \ |
| 216 | echo "INSTALL_RUBIN_PRERELEASE=true requires requirements/rubin-prerelease.txt" >&2; \ |
| 217 | exit 1; \ |
| 218 | fi \ |
| 219 | && if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ |
| 220 | sed -i 's/^nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' requirements/cuda.txt; \ |
| 221 | sed -i 's/^humming-kernels\[cu13\]/humming-kernels[cu12]/' requirements/cuda.txt; \ |
| 222 | fi \ |
| 223 | && if [ "$(uname -m)" = "aarch64" ]; then \ |
| 224 | # The manylinux 2_28 (glibc 2.28) aarch64 builder cannot use |
| 225 | # tilelang's manylinux_2_34 wheel, so tilelang builds from sdist. |
| 226 | # Cython 3.3 rejects tilelang's hardcoded Py_LIMITED_API=3.8. |
| 227 | printf 'cython<3.3\n' > /tmp/build-constraints.txt \ |
| 228 | && export UV_BUILD_CONSTRAINT=/tmp/build-constraints.txt; \ |
| 229 | fi \ |
| 230 | && if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 231 | echo "Installing Rubin prerelease requirements..." \ |
| 232 | && /opt/venv/bin/python3 tools/use_existing_torch.py --prefix \ |
| 233 | && uv pip install --python /opt/venv/bin/python3 \ |
| 234 | --prerelease=allow --index-strategy unsafe-first-match \ |
| 235 | -r /tmp/vllm-requirements/rubin-prerelease.txt \ |
| 236 | && uv pip install --python /opt/venv/bin/python3 \ |
| 237 | -r requirements/cuda.txt \ |
| 238 | && uv pip install --python /opt/venv/bin/python3 \ |
| 239 | --prerelease=allow --index-strategy unsafe-first-match \ |
| 240 | -r /tmp/vllm-requirements/rubin-prerelease.txt; \ |
| 241 | elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ |
| 242 | echo "Installing torch nightly..." \ |
| 243 | && uv pip install --python /opt/venv/bin/python3 torch torchaudio torchvision --pre \ |
| 244 | --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \ |
| 245 | && echo "Installing other requirements..." \ |
| 246 | && /opt/venv/bin/python3 tools/use_existing_torch.py --prefix \ |
| 247 | && uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \ |
| 248 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 249 | else \ |
| 250 | uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \ |
| 251 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 252 | fi |
| 253 | |
| 254 | # CUDA 13.4 Rubin images need a compatibility link for the prerelease |
| 255 | # PyTorch wheel's libcurand.so.11 dependency. CUDA 13.5 must use its native |
| 256 | # libcurand.so.11 instead. |
| 257 | RUN if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 258 | CUDA_MAJOR_MINOR=$(echo "${CUDA_VERSION}" | cut -d. -f1,2) \ |
| 259 | && if [ "${CUDA_MAJOR_MINOR}" = "13.4" ]; then \ |
| 260 | CURAND_LIBRARY=$(find -H /usr/local/cuda -name libcurand.so.10 \ |
| 261 | -print -quit) \ |
| 262 | && test -n "${CURAND_LIBRARY}" \ |
| 263 | && ln -sf "${CURAND_LIBRARY}" \ |
| 264 | /usr/local/cuda/lib64/libcurand.so.11; \ |
| 265 | elif [ "${CUDA_MAJOR_MINOR}" = "13.5" ]; then \ |
| 266 | CURAND_LIBRARY=$(find -H /usr/local/cuda -name libcurand.so.11 \ |
| 267 | -print -quit) \ |
| 268 | && test -n "${CURAND_LIBRARY}" \ |
| 269 | && ! readlink -f "${CURAND_LIBRARY}" | grep -q 'libcurand\.so\.10'; \ |
| 270 | else \ |
| 271 | echo "Unsupported Rubin CUDA version: ${CUDA_MAJOR_MINOR}" >&2; \ |
| 272 | exit 1; \ |
| 273 | fi \ |
| 274 | && printf '%s\n' \ |
| 275 | /opt/venv/lib \ |
| 276 | "/opt/venv/lib/python${PYTHON_VERSION}/site-packages/nvpl/lib" \ |
| 277 | > /etc/ld.so.conf.d/vllm-rubin-python.conf \ |
| 278 | && ldconfig; \ |
| 279 | fi |
| 280 | |
| 281 | # Track PyTorch lib versions used during build and match in downstream instances. |
| 282 | # We do this for both nightly and release so we can strip dependencies/*.txt as needed. |
| 283 | # Otherwise library dependencies can upgrade/downgrade torch incorrectly. |
| 284 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 285 | uv pip freeze | grep -i "^torch=\|^torchvision=\|^torchaudio=" > torch_lib_versions.txt \ |
| 286 | && TORCH_LIB_VERSIONS=$(cat torch_lib_versions.txt | xargs) \ |
| 287 | && echo "Installed torch libs: ${TORCH_LIB_VERSIONS}" |
| 288 | |
| 289 | # CUDA arch list used by torch. Keep this explicit so GPU-less container builds |
| 290 | # do not depend on PyTorch's changing common-architecture fallback. |
| 291 | # From versions.json: .torch.cuda_arch_list |
| 292 | # Do not add +PTX here: vLLM filters torch's top-level PTX flag when it |
| 293 | # converts global gencode flags into per-kernel arch lists. If a specific |
| 294 | # kernel needs PTX, add +PTX to that kernel's CMake arch list instead. |
| 295 | ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0' |
| 296 | ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} |
| 297 | #################### BUILD BASE IMAGE #################### |
| 298 | |
| 299 | #################### RUST BUILD IMAGE #################### |
| 300 | # Build the Rust frontend (`vllm-rs`) in a dedicated stage so the main wheel |
| 301 | # build stage doesn't need the rust toolchain or the rust source. |
| 302 | # This stage reuses the Python environment from base and runs in parallel with |
| 303 | # csrc-build/extensions-build. |
| 304 | FROM base AS rust-build-cache |
| 305 | ARG USE_SCCACHE |
| 306 | ARG SCCACHE_ENDPOINT |
| 307 | |
| 308 | # Install native tools needed only for Rust builds. |
| 309 | RUN dnf install -y --setopt=install_weak_deps=False make perl-core \ |
| 310 | && dnf clean all \ |
| 311 | && rm -rf /var/cache/dnf |
| 312 | |
| 313 | WORKDIR /workspace |
| 314 | |
| 315 | COPY requirements/build/rust.txt requirements/build/rust.txt |
| 316 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 317 | uv pip install --python /opt/venv/bin/python3 -r requirements/build/rust.txt |
| 318 | |
| 319 | # Copy only the Rust build inputs; tools/build_rust.sh publishes artifacts needed |
| 320 | # by the wheel build stage. |
| 321 | COPY rust rust |
| 322 | COPY rust-toolchain.toml rust-toolchain.toml |
| 323 | COPY tools/build_rust.py tools/build_rust.py |
| 324 | COPY tools/build_rust.sh tools/build_rust.sh |
| 325 | |
| 326 | # Cap cargo parallelism to avoid exhausting the CI host's open-file limit |
| 327 | # (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise). |
| 328 | ENV CARGO_BUILD_JOBS=4 |
| 329 | |
| 330 | # BuildKit can run this stage in parallel with csrc-build. Keep Rust on a |
| 331 | # separate local sccache daemon while sharing the same remote cache backend. |
| 332 | ENV SCCACHE_SERVER_PORT=4227 |
| 333 | |
| 334 | # Build with a stable version so this target snapshot is keyed only by Rust |
| 335 | # inputs. The child stage below relinks the crates that embed the exact version. |
| 336 | RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ |
| 337 | --mount=type=cache,target=/root/.cargo/git,sharing=locked \ |
| 338 | --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \ |
| 339 | if [ "$USE_SCCACHE" = "1" ]; then \ |
| 340 | if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \ |
| 341 | export RUSTC_WRAPPER=sccache; \ |
| 342 | sccache --show-stats; \ |
| 343 | fi \ |
| 344 | && SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash tools/build_rust.sh \ |
| 345 | && if [ "$USE_SCCACHE" = "1" ]; then \ |
| 346 | sccache --show-stats; \ |
| 347 | fi |
| 348 | |
| 349 | # Relink with the exact Git-derived package version. |
| 350 | FROM rust-build-cache AS rust-build |
| 351 | |
| 352 | RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ |
| 353 | --mount=type=cache,target=/root/.cargo/git,sharing=locked \ |
| 354 | --mount=type=bind,source=.git,target=.git \ |
| 355 | SETUPTOOLS_SCM_PRETEND_METADATA="{dirty=false}" bash tools/build_rust.sh |
| 356 | #################### RUST BUILD IMAGE #################### |
| 357 | |
| 358 | #################### CSRC BUILD IMAGE #################### |
| 359 | FROM base AS csrc-build |
| 360 | |
| 361 | ARG INSTALL_RUBIN_PRERELEASE |
| 362 | ARG PIP_INDEX_URL UV_INDEX_URL |
| 363 | ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL |
| 364 | ARG PYTORCH_CUDA_INDEX_BASE_URL |
| 365 | |
| 366 | # We can specify the standard or nightly build of PyTorch |
| 367 | ARG PYTORCH_NIGHTLY |
| 368 | |
| 369 | # Install build dependencies |
| 370 | COPY requirements/build/cuda.txt requirements/build/cuda.txt |
| 371 | COPY tools/use_existing_torch.py tools/use_existing_torch.py |
| 372 | COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt |
| 373 | |
| 374 | # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out |
| 375 | # Reference: https://github.com/astral-sh/uv/pull/1694 |
| 376 | ENV UV_HTTP_TIMEOUT=500 |
| 377 | ENV UV_INDEX_STRATEGY="unsafe-best-match" |
| 378 | # Use copy mode to avoid hardlink failures with Docker cache mounts |
| 379 | ENV UV_LINK_MODE=copy |
| 380 | |
| 381 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 382 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 383 | echo "Installing build requirements with Rubin prerelease PyTorch..." \ |
| 384 | && python3 tools/use_existing_torch.py --prefix \ |
| 385 | && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt; \ |
| 386 | elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ |
| 387 | echo "Installing build requirements without torch..." \ |
| 388 | && python3 tools/use_existing_torch.py --prefix \ |
| 389 | && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt \ |
| 390 | && echo "Installing torch nightly..." \ |
| 391 | && uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | grep -i "^torch=" | xargs) --pre \ |
| 392 | --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 393 | else \ |
| 394 | echo "Installing build requirements..." \ |
| 395 | && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt \ |
| 396 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 397 | fi |
| 398 | |
| 399 | WORKDIR /workspace |
| 400 | |
| 401 | COPY pyproject.toml setup.py CMakeLists.txt ./ |
| 402 | COPY tools/build_rust.py tools/build_rust.py |
| 403 | COPY cmake cmake/ |
| 404 | COPY csrc csrc/ |
| 405 | COPY vllm/envs.py vllm/envs.py |
| 406 | COPY vllm/__init__.py vllm/__init__.py |
| 407 | |
| 408 | # max jobs used by Ninja to build extensions |
| 409 | ARG max_jobs=2 |
| 410 | ENV MAX_JOBS=${max_jobs} |
| 411 | # number of threads used by nvcc |
| 412 | ARG nvcc_threads=8 |
| 413 | ENV NVCC_THREADS=$nvcc_threads |
| 414 | |
| 415 | ARG USE_SCCACHE |
| 416 | ARG SCCACHE_ENDPOINT |
| 417 | |
| 418 | # Flag to control whether to use pre-built vLLM wheels |
| 419 | ARG VLLM_USE_PRECOMPILED="" |
| 420 | ARG VLLM_MERGE_BASE_COMMIT="" |
| 421 | ARG VLLM_MAIN_CUDA_VERSION="" |
| 422 | |
| 423 | # Use dummy version for csrc-build wheel (only .so files are extracted, version doesn't matter) |
| 424 | ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+csrc.build" |
| 425 | |
| 426 | # Use existing torch for nightly and Rubin prerelease builds |
| 427 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 428 | if [ "${PYTORCH_NIGHTLY}" = "1" ] || \ |
| 429 | [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 430 | python3 tools/use_existing_torch.py --prefix; \ |
| 431 | fi |
| 432 | |
| 433 | # Provision one bare Python per `requires-python` entry; cmake reads |
| 434 | # DEEPGEMM_PYTHON_INTERPRETERS to build DeepGEMM `_C` for each. See |
| 435 | # cmake/external_projects/deepgemm.cmake for the full picture. |
| 436 | COPY tools/setup_deepgemm_pythons.sh tools/build_deepgemm_C.py tools/ |
| 437 | ENV DEEPGEMM_VENV_PREFIX=/opt/dgenv |
| 438 | RUN --mount=type=cache,target=/root/.cache/uv \ |
| 439 | tools/setup_deepgemm_pythons.sh > /tmp/dg_pythons.txt |
| 440 | |
| 441 | # Build the vLLM wheel |
| 442 | # if USE_SCCACHE is set, use sccache to speed up compilation |
| 443 | # AWS credentials mounted at ~/.aws/credentials for sccache S3 auth (optional) |
| 444 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 445 | --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \ |
| 446 | if [ "$USE_SCCACHE" = "1" ]; then \ |
| 447 | if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi \ |
| 448 | && export CMAKE_BUILD_TYPE=Release \ |
| 449 | && export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \ |
| 450 | && export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" \ |
| 451 | && export VLLM_MAIN_CUDA_VERSION="${VLLM_MAIN_CUDA_VERSION}" \ |
| 452 | && export VLLM_DOCKER_BUILD_CONTEXT=1 \ |
| 453 | && export DEEPGEMM_PYTHON_INTERPRETERS=$(cat /tmp/dg_pythons.txt) \ |
| 454 | && sccache --show-stats \ |
| 455 | && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \ |
| 456 | && sccache --show-stats; \ |
| 457 | fi |
| 458 | |
| 459 | ARG vllm_target_device="cuda" |
| 460 | ENV VLLM_TARGET_DEVICE=${vllm_target_device} |
| 461 | ENV CCACHE_DIR=/root/.cache/ccache |
| 462 | RUN --mount=type=cache,target=/root/.cache/ccache \ |
| 463 | --mount=type=cache,target=/opt/uv/cache \ |
| 464 | if [ "$USE_SCCACHE" != "1" ]; then \ |
| 465 | # Clean any existing CMake artifacts |
| 466 | rm -rf .deps && \ |
| 467 | mkdir -p .deps && \ |
| 468 | export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" && \ |
| 469 | export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" && \ |
| 470 | export VLLM_DOCKER_BUILD_CONTEXT=1 && \ |
| 471 | export DEEPGEMM_PYTHON_INTERPRETERS=$(cat /tmp/dg_pythons.txt) && \ |
| 472 | python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \ |
| 473 | fi |
| 474 | |
| 475 | #################### CSRC BUILD IMAGE #################### |
| 476 | |
| 477 | #################### EXTENSIONS BUILD IMAGE #################### |
| 478 | # Build DeepEP - runs in PARALLEL with csrc-build |
| 479 | # This stage is independent and doesn't affect csrc cache |
| 480 | # Kernel symbol map for CI test selection: which source file produced each GPU |
| 481 | # kernel symbol, read from the objects this stage just compiled. Only when |
| 482 | # VLLM_KERNEL_SYMBOL_MAP=1 (nightly), and placed after the compile layers so |
| 483 | # their cache is untouched. Never fails the build; an empty map is a signal. |
| 484 | ARG VLLM_KERNEL_SYMBOL_MAP="" |
| 485 | ARG buildkite_commit="" |
| 486 | COPY tools/ci/kernel_symbol_map.py tools/ci/kernel_symbol_map.py |
| 487 | RUN if [ "${VLLM_KERNEL_SYMBOL_MAP}" = "1" ]; then \ |
| 488 | mkdir -p dist && python3 tools/ci/kernel_symbol_map.py \ |
| 489 | --build-root build --source-root . --commit "${buildkite_commit}" \ |
| 490 | --out dist/kernel_symbol_map.json.gz; \ |
| 491 | else echo "Skipping kernel symbol map (VLLM_KERNEL_SYMBOL_MAP != 1)"; fi |
| 492 | |
| 493 | # Exported on its own by the CI bake target of the same name (ci-infra |
| 494 | # docker/ci.hcl) with output=type=local, so the map becomes a build artifact. |
| 495 | FROM scratch AS kernel-symbol-map |
| 496 | COPY --from=csrc-build /workspace/dist/kernel_symbol_map.json.gz /kernel_symbol_map.json.gz |
| 497 | |
| 498 | FROM base AS extensions-build |
| 499 | ARG TARGETARCH |
| 500 | ARG CUDA_VERSION |
| 501 | ARG INSTALL_RUBIN_PRERELEASE |
| 502 | ARG TRITON_INSTALL_FROM_SOURCE_REPO |
| 503 | ARG TRITON_INSTALL_FROM_SOURCE_REVISION |
| 504 | ARG max_jobs |
| 505 | |
| 506 | # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out |
| 507 | ENV UV_HTTP_TIMEOUT=500 |
| 508 | ENV UV_INDEX_STRATEGY="unsafe-best-match" |
| 509 | ENV UV_LINK_MODE=copy |
| 510 | |
| 511 | WORKDIR /workspace |
| 512 | |
| 513 | # Build DeepEP wheels |
| 514 | COPY tools/ep_kernels/install_python_libraries.sh /tmp/install_python_libraries.sh |
| 515 | # Defaults moved here from tools/ep_kernels/install_python_libraries.sh for centralized version management |
| 516 | ARG DEEPEP_COMMIT_HASH=d4f41e4e93 |
| 517 | ARG NVSHMEM_VER |
| 518 | ARG NCCL_VERSION |
| 519 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 520 | mkdir -p /tmp/ep_kernels_workspace/dist && \ |
| 521 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 522 | # The Rubin CUDA builder provides NCCL in its architecture-specific \ |
| 523 | # toolkit directory rather than as a Python package. Give DeepEP a \ |
| 524 | # stable prefix that also exists in the Ubuntu runtime image. \ |
| 525 | NCCL_HEADER=$(find /usr/local/cuda/targets -name nccl.h -print -quit) \ |
| 526 | && test -n "${NCCL_HEADER}" \ |
| 527 | && NCCL_LIBRARY=$(find -H /usr/local/cuda/targets \ |
| 528 | -name libnccl.so.2 -print -quit) \ |
| 529 | && test -n "${NCCL_LIBRARY}" \ |
| 530 | && ln -sf "${NCCL_HEADER}" /usr/include/nccl.h \ |
| 531 | && ln -sf "${NCCL_LIBRARY}" /usr/lib/libnccl.so.2 \ |
| 532 | && ln -sf libnccl.so.2 /usr/lib/libnccl.so \ |
| 533 | && export EP_NCCL_ROOT_DIR=/usr; \ |
| 534 | fi && \ |
| 535 | CUDA_MAJOR=$(echo $CUDA_VERSION | cut -d. -f1) && \ |
| 536 | if [ -n "$NCCL_VERSION" ]; then \ |
| 537 | echo "nvidia-nccl-cu${CUDA_MAJOR}==${NCCL_VERSION}" \ |
| 538 | > /tmp/nccl-override.txt && \ |
| 539 | export UV_OVERRIDE=/tmp/nccl-override.txt; \ |
| 540 | fi && \ |
| 541 | export TORCH_CUDA_ARCH_LIST='9.0a 10.0a' && \ |
| 542 | /tmp/install_python_libraries.sh \ |
| 543 | --workspace /tmp/ep_kernels_workspace \ |
| 544 | --mode wheel \ |
| 545 | ${DEEPEP_COMMIT_HASH:+--deepep-ref "$DEEPEP_COMMIT_HASH"} \ |
| 546 | ${NVSHMEM_VER:+--nvshmem-ver "$NVSHMEM_VER"} && \ |
| 547 | find /tmp/ep_kernels_workspace/nvshmem -name '*.a' -delete |
| 548 | |
| 549 | # Build Triton from source when a repository is configured. Triton 3.8 pins |
| 550 | # ptxas-blackwell 13.3, which cannot assemble SM107. Rubin CUDA 13.4 and 13.5 |
| 551 | # builds use their SM107-capable system ptxas during the wheel build and at |
| 552 | # runtime. |
| 553 | COPY tools/build_triton_from_source.sh /tmp/build_triton_from_source.sh |
| 554 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 555 | --mount=type=cache,id=triton-source-${TARGETARCH},target=/root/.triton,sharing=locked \ |
| 556 | --mount=type=cache,id=triton-source-ccache-${TARGETARCH},target=/root/.cache/ccache,sharing=locked \ |
| 557 | set -eux; \ |
| 558 | TRITON_WORKSPACE=/tmp/triton_from_source_workspace; \ |
| 559 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ] \ |
| 560 | && [ -z "${TRITON_INSTALL_FROM_SOURCE_REPO}" ]; then \ |
| 561 | echo "Rubin prerelease builds require TRITON_INSTALL_FROM_SOURCE_REPO" >&2; \ |
| 562 | exit 1; \ |
| 563 | elif [ -z "${TRITON_INSTALL_FROM_SOURCE_REPO}" ] \ |
| 564 | && [ -n "${TRITON_INSTALL_FROM_SOURCE_REVISION}" ]; then \ |
| 565 | echo "TRITON_INSTALL_FROM_SOURCE_REVISION requires TRITON_INSTALL_FROM_SOURCE_REPO" >&2; \ |
| 566 | exit 1; \ |
| 567 | elif [ -n "${TRITON_INSTALL_FROM_SOURCE_REPO}" ]; then \ |
| 568 | dnf install -y --setopt=install_weak_deps=False zlib-devel; \ |
| 569 | dnf clean all; \ |
| 570 | rm -rf /var/cache/dnf; \ |
| 571 | CUDA_MAJOR_MINOR=$(echo "${CUDA_VERSION}" | cut -d. -f1,2); \ |
| 572 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 573 | case "${CUDA_MAJOR_MINOR}" in \ |
| 574 | 13.4|13.5) \ |
| 575 | export TRITON_PTXAS_BLACKWELL_PATH=/usr/local/cuda/bin/ptxas \ |
| 576 | ;; \ |
| 577 | *) \ |
| 578 | echo "Unsupported Rubin CUDA version: ${CUDA_MAJOR_MINOR}" >&2; \ |
| 579 | exit 1 \ |
| 580 | ;; \ |
| 581 | esac; \ |
| 582 | fi; \ |
| 583 | TRITON_WORKSPACE="${TRITON_WORKSPACE}" \ |
| 584 | MAX_JOBS="${max_jobs:-}" \ |
| 585 | bash /tmp/build_triton_from_source.sh; \ |
| 586 | else \ |
| 587 | mkdir -p "${TRITON_WORKSPACE}/dist"; \ |
| 588 | printf '%s\n' disabled > "${TRITON_WORKSPACE}/dist/build-state"; \ |
| 589 | (cd "${TRITON_WORKSPACE}/dist" \ |
| 590 | && sha256sum build-state > wheel.sha256); \ |
| 591 | fi |
| 592 | #################### EXTENSIONS BUILD IMAGE #################### |
| 593 | |
| 594 | #################### WHEEL BUILD IMAGE #################### |
| 595 | FROM base AS build |
| 596 | ARG TARGETPLATFORM |
| 597 | ARG INSTALL_RUBIN_PRERELEASE |
| 598 | |
| 599 | ARG PIP_INDEX_URL UV_INDEX_URL |
| 600 | ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL |
| 601 | ARG PYTORCH_CUDA_INDEX_BASE_URL |
| 602 | |
| 603 | # We can specify the standard or nightly build of PyTorch |
| 604 | ARG PYTORCH_NIGHTLY |
| 605 | |
| 606 | # Install build dependencies |
| 607 | COPY requirements/build/cuda.txt requirements/build/cuda.txt |
| 608 | COPY tools/use_existing_torch.py tools/use_existing_torch.py |
| 609 | COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt |
| 610 | |
| 611 | # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out |
| 612 | # Reference: https://github.com/astral-sh/uv/pull/1694 |
| 613 | ENV UV_HTTP_TIMEOUT=500 |
| 614 | ENV UV_INDEX_STRATEGY="unsafe-best-match" |
| 615 | # Use copy mode to avoid hardlink failures with Docker cache mounts |
| 616 | ENV UV_LINK_MODE=copy |
| 617 | |
| 618 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 619 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 620 | echo "Installing build requirements with Rubin prerelease PyTorch..." \ |
| 621 | && python3 tools/use_existing_torch.py --prefix \ |
| 622 | && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt; \ |
| 623 | elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ |
| 624 | echo "Installing build requirements without torch..." \ |
| 625 | && python3 tools/use_existing_torch.py --prefix \ |
| 626 | && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt \ |
| 627 | && echo "Installing torch nightly..." \ |
| 628 | && uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | grep -i "^torch=" | xargs) --pre \ |
| 629 | --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 630 | else \ |
| 631 | echo "Installing build requirements..." \ |
| 632 | && uv pip install --python /opt/venv/bin/python3 -r requirements/build/cuda.txt \ |
| 633 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 634 | fi |
| 635 | |
| 636 | WORKDIR /workspace |
| 637 | |
| 638 | # Copy pre-built csrc wheel directly |
| 639 | COPY --from=csrc-build /workspace/dist /precompiled-wheels |
| 640 | COPY . . |
| 641 | |
| 642 | # Drop the pre-built Rust artifacts into the source tree. setup.py detects |
| 643 | # them and ships them as-is, skipping the local Rust build. |
| 644 | COPY --from=rust-build /workspace/vllm/vllm-rs vllm/vllm-rs |
| 645 | COPY --from=rust-build /workspace/vllm/_rust_*.so vllm/ |
| 646 | |
| 647 | ARG GIT_REPO_CHECK=0 |
| 648 | RUN --mount=type=bind,source=.git,target=.git \ |
| 649 | if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi |
| 650 | |
| 651 | ARG vllm_target_device="cuda" |
| 652 | ENV VLLM_TARGET_DEVICE=${vllm_target_device} |
| 653 | |
| 654 | # Skip adding +precompiled suffix to version (preserves git-derived version) |
| 655 | ENV VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX=1 |
| 656 | |
| 657 | # Use existing torch for nightly and Rubin prerelease builds |
| 658 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 659 | if [ "${PYTORCH_NIGHTLY}" = "1" ] || \ |
| 660 | [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 661 | python3 tools/use_existing_torch.py --prefix; \ |
| 662 | fi \ |
| 663 | && if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 664 | sed -Ei '/^[[:space:]]*(flashinfer-python|nvidia-cutlass-dsl\[cu13\]|quack-kernels)[[:space:]]*==/Id' \ |
| 665 | requirements/cuda.txt; \ |
| 666 | fi |
| 667 | |
| 668 | # Build the vLLM wheel |
| 669 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 670 | --mount=type=bind,source=.git,target=.git \ |
| 671 | if [ "${vllm_target_device}" = "cuda" ]; then \ |
| 672 | export VLLM_USE_PRECOMPILED=1; \ |
| 673 | export VLLM_PRECOMPILED_WHEEL_LOCATION=$(ls /precompiled-wheels/*.whl); \ |
| 674 | fi && \ |
| 675 | python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 |
| 676 | |
| 677 | # Record the wheel checksum so downstream stages can bust their layer cache |
| 678 | # when the wheel changes, without copying the wheel itself into the image. |
| 679 | RUN sha256sum dist/*.whl > dist/wheel.sha256 |
| 680 | |
| 681 | # Copy extension wheels from extensions-build stage for later use |
| 682 | COPY --from=extensions-build /tmp/ep_kernels_workspace/dist /tmp/ep_kernels_workspace/dist |
| 683 | |
| 684 | # Record the EP kernels wheel checksum for the same cache-busting purpose. |
| 685 | RUN sha256sum /tmp/ep_kernels_workspace/dist/*.whl \ |
| 686 | > /tmp/ep_kernels_workspace/dist/wheels.sha256 |
| 687 | |
| 688 | # Check the size of the CUDA 13 wheel uploaded to PyPI |
| 689 | COPY .buildkite/check-wheel-size.py check-wheel-size.py |
| 690 | # sync the default value with .buildkite/check-wheel-size.py |
| 691 | ARG VLLM_MAX_SIZE_MB=500 |
| 692 | ENV VLLM_MAX_SIZE_MB=$VLLM_MAX_SIZE_MB |
| 693 | ARG RUN_WHEEL_CHECK=true |
| 694 | RUN if [ "$RUN_WHEEL_CHECK" = "true" ] && \ |
| 695 | [ "${CUDA_VERSION%%.*}" = "13" ]; then \ |
| 696 | python3 check-wheel-size.py dist; \ |
| 697 | else \ |
| 698 | echo "Skipping wheel size check."; \ |
| 699 | fi |
| 700 | |
| 701 | #################### WHEEL BUILD IMAGE #################### |
| 702 | |
| 703 | #################### DEV IMAGE #################### |
| 704 | FROM base AS dev |
| 705 | |
| 706 | ARG PIP_INDEX_URL UV_INDEX_URL |
| 707 | ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL |
| 708 | ARG PYTORCH_CUDA_INDEX_BASE_URL |
| 709 | |
| 710 | # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out |
| 711 | # Reference: https://github.com/astral-sh/uv/pull/1694 |
| 712 | ENV UV_HTTP_TIMEOUT=500 |
| 713 | ENV UV_INDEX_STRATEGY="unsafe-best-match" |
| 714 | # Use copy mode to avoid hardlink failures with Docker cache mounts |
| 715 | ENV UV_LINK_MODE=copy |
| 716 | |
| 717 | # Install libnuma-dev, required by fastsafetensors (fixes #20384) |
| 718 | RUN dnf install -y numactl-devel \ |
| 719 | && dnf clean all \ |
| 720 | && rm -rf /var/cache/dnf |
| 721 | |
| 722 | |
| 723 | # We can specify the standard or nightly build of PyTorch |
| 724 | ARG PYTORCH_NIGHTLY |
| 725 | |
| 726 | # Install development dependencies |
| 727 | COPY requirements/common.txt requirements/common.txt |
| 728 | COPY requirements/lint.txt requirements/lint.txt |
| 729 | COPY requirements/test/cuda.in requirements/test/cuda.in |
| 730 | COPY requirements/test/cuda.txt requirements/test/cuda.txt |
| 731 | COPY requirements/dev.txt requirements/dev.txt |
| 732 | # uv reads `[tool.uv] exclude-dependencies` from here when recompiling below. |
| 733 | COPY pyproject.toml pyproject.toml |
| 734 | COPY tools/use_existing_torch.py tools/use_existing_torch.py |
| 735 | COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt |
| 736 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 737 | if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ |
| 738 | echo "Installing dev requirements plus torch nightly..." \ |
| 739 | && python3 tools/use_existing_torch.py --prefix \ |
| 740 | && cat torch_lib_versions.txt >> requirements/test/cuda.in \ |
| 741 | && uv pip compile requirements/test/cuda.in -o requirements/test/cuda.txt --index-strategy unsafe-best-match \ |
| 742 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \ |
| 743 | && uv pip install --python /opt/venv/bin/python3 $(cat torch_lib_versions.txt | xargs) --pre \ |
| 744 | -r requirements/dev.txt \ |
| 745 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 746 | else \ |
| 747 | echo "Installing dev requirements..." \ |
| 748 | && uv pip install --python /opt/venv/bin/python3 -r requirements/dev.txt \ |
| 749 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 750 | fi |
| 751 | |
| 752 | #################### DEV IMAGE #################### |
| 753 | #################### vLLM installation IMAGE #################### |
| 754 | # image with vLLM installed |
| 755 | FROM ${FINAL_BASE_IMAGE} AS vllm-runtime-base |
| 756 | |
| 757 | ARG INSTALL_RUBIN_PRERELEASE |
| 758 | ARG TARGETPLATFORM |
| 759 | ARG max_jobs=2 |
| 760 | ARG CUDA_VERSION |
| 761 | ARG PYTHON_VERSION |
| 762 | ARG NCCL_VERSION |
| 763 | ARG DEADSNAKES_MIRROR_URL |
| 764 | ARG DEADSNAKES_GPGKEY_URL |
| 765 | ARG GET_PIP_URL |
| 766 | |
| 767 | # Keep the output image's CUDA metadata aligned with the selected toolkit. |
| 768 | # This also normalizes prerelease base images with four-component metadata for |
| 769 | # OCI runtimes that evaluate NVIDIA's CUDA driver requirement. |
| 770 | ENV CUDA_VERSION=${CUDA_VERSION} |
| 771 | |
| 772 | ENV DEBIAN_FRONTEND=noninteractive |
| 773 | WORKDIR /vllm-workspace |
| 774 | |
| 775 | |
| 776 | # Python version string for paths (e.g., "312" for 3.12) |
| 777 | RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \ |
| 778 | echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment |
| 779 | |
| 780 | # Install Python and system dependencies |
| 781 | RUN apt-get update -y \ |
| 782 | && apt-get install -y --no-install-recommends \ |
| 783 | software-properties-common \ |
| 784 | curl \ |
| 785 | sudo \ |
| 786 | ffmpeg \ |
| 787 | libsm6 \ |
| 788 | libxext6 \ |
| 789 | libgl1 \ |
| 790 | && if [ ! -z ${DEADSNAKES_MIRROR_URL} ] ; then \ |
| 791 | if [ ! -z "${DEADSNAKES_GPGKEY_URL}" ] ; then \ |
| 792 | mkdir -p -m 0755 /etc/apt/keyrings ; \ |
| 793 | curl -L ${DEADSNAKES_GPGKEY_URL} | gpg --dearmor > /etc/apt/keyrings/deadsnakes.gpg ; \ |
| 794 | sudo chmod 644 /etc/apt/keyrings/deadsnakes.gpg ; \ |
| 795 | echo "deb [signed-by=/etc/apt/keyrings/deadsnakes.gpg] ${DEADSNAKES_MIRROR_URL} $(lsb_release -cs) main" > /etc/apt/sources.list.d/deadsnakes.list ; \ |
| 796 | fi ; \ |
| 797 | else \ |
| 798 | for i in 1 2 3; do \ |
| 799 | add-apt-repository -y ppa:deadsnakes/ppa && break || \ |
| 800 | { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \ |
| 801 | done ; \ |
| 802 | fi \ |
| 803 | && apt-get update -y \ |
| 804 | && apt-get install -y --no-install-recommends \ |
| 805 | python${PYTHON_VERSION} \ |
| 806 | python${PYTHON_VERSION}-dev \ |
| 807 | python${PYTHON_VERSION}-venv \ |
| 808 | libibverbs-dev \ |
| 809 | && rm -rf /var/lib/apt/lists/* \ |
| 810 | && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ |
| 811 | && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \ |
| 812 | && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \ |
| 813 | && rm -f /usr/lib/python${PYTHON_VERSION}/EXTERNALLY-MANAGED \ |
| 814 | && curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \ |
| 815 | && python3 --version && python3 -m pip --version |
| 816 | |
| 817 | # Install CUDA development tools for runtime JIT compilation |
| 818 | # (FlashInfer, DeepGEMM, EP kernels all require compilation at runtime) |
| 819 | RUN CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-') && \ |
| 820 | apt-get update -y && \ |
| 821 | apt-get install -y --no-install-recommends --allow-change-held-packages \ |
| 822 | cuda-nvcc-${CUDA_VERSION_DASH} \ |
| 823 | cuda-cudart-${CUDA_VERSION_DASH} \ |
| 824 | cuda-nvrtc-${CUDA_VERSION_DASH} \ |
| 825 | # FlashInfer's runtime JIT-compiled kernels (e.g. trtllm_fmha_kernel_launcher.cu) |
| 826 | # #include <nvrtc.h>, which only the -dev package ships. |
| 827 | cuda-nvrtc-dev-${CUDA_VERSION_DASH} \ |
| 828 | cuda-cuobjdump-${CUDA_VERSION_DASH} \ |
| 829 | libcurand-dev-${CUDA_VERSION_DASH} \ |
| 830 | libcublas-dev-${CUDA_VERSION_DASH} \ |
| 831 | # Required by fastsafetensors (fixes #20384) |
| 832 | libnuma-dev \ |
| 833 | # numactl CLI for NUMA binding at runtime |
| 834 | numactl && \ |
| 835 | rm -rf /var/lib/apt/lists/* |
| 836 | |
| 837 | # Ensures nvcc/ptxas (installed above, needed at runtime for FlashInfer/ |
| 838 | # DeepGEMM/EP kernel JIT compilation) are reachable on PATH. Harmless no-op |
| 839 | # if already set by FINAL_BASE_IMAGE. |
| 840 | ENV CUDA_HOME=/usr/local/cuda |
| 841 | ENV PATH="${CUDA_HOME}/bin:${PATH}" |
| 842 | ENV LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" |
| 843 | |
| 844 | # Apply the CUDA 13.4-only libcurand compatibility link described in the build |
| 845 | # stage. CUDA 13.5 uses its native libcurand.so.11. DeepEP expects NCCL directly |
| 846 | # under its configured /usr prefix, while Ubuntu installs it in a multiarch |
| 847 | # subdirectory - but that symlink is only possible when NCCL is already |
| 848 | # present (e.g. nvcr.io/nvidia/cuda-dl-base bundles it; a bare-CUDA image |
| 849 | # doesn't, and gets it later via the standard nvidia-nccl-cu* pip package, |
| 850 | # UV_OVERRIDE below). Check reality (whether libnccl.so.2 is actually found) |
| 851 | # and tolerate not finding it instead of failing the build. |
| 852 | RUN if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 853 | CUDA_MAJOR_MINOR=$(echo "${CUDA_VERSION}" | cut -d. -f1,2) \ |
| 854 | && if [ "${CUDA_MAJOR_MINOR}" = "13.4" ]; then \ |
| 855 | CURAND_LIBRARY=$(find -H /usr/local/cuda -name libcurand.so.10 \ |
| 856 | -print -quit) \ |
| 857 | && test -n "${CURAND_LIBRARY}" \ |
| 858 | && ln -sf "${CURAND_LIBRARY}" \ |
| 859 | /usr/local/cuda/lib64/libcurand.so.11; \ |
| 860 | elif [ "${CUDA_MAJOR_MINOR}" = "13.5" ]; then \ |
| 861 | CURAND_LIBRARY=$(find -H /usr/local/cuda -name libcurand.so.11 \ |
| 862 | -print -quit) \ |
| 863 | && test -n "${CURAND_LIBRARY}" \ |
| 864 | && ! readlink -f "${CURAND_LIBRARY}" | grep -q 'libcurand\.so\.10'; \ |
| 865 | else \ |
| 866 | echo "Unsupported Rubin CUDA version: ${CUDA_MAJOR_MINOR}" >&2; \ |
| 867 | exit 1; \ |
| 868 | fi \ |
| 869 | && if NCCL_LIBRARY=$(find /usr/lib -name libnccl.so.2 -print -quit) \ |
| 870 | && [ -n "${NCCL_LIBRARY}" ]; then \ |
| 871 | if [ "${NCCL_LIBRARY}" != "/usr/lib/libnccl.so.2" ]; then \ |
| 872 | ln -sf "${NCCL_LIBRARY}" /usr/lib/libnccl.so.2; \ |
| 873 | fi \ |
| 874 | && ln -sf libnccl.so.2 /usr/lib/libnccl.so; \ |
| 875 | fi \ |
| 876 | && ldconfig; \ |
| 877 | fi |
| 878 | |
| 879 | # Install the initialized-snapshot runtime on the architecture supported by |
| 880 | # `vllm snapshot`. CRIU and its CUDA plugin are built from the same pinned |
| 881 | # source. The `cuda-checkpoint`-compatible helper is built from NVIDIA Dynamo's |
| 882 | # Apache-2.0 source against the public CUDA Driver checkpoint APIs instead of |
| 883 | # redistributing NVIDIA's proprietary cuda-checkpoint binary. The helper uses |
| 884 | # checkpoint APIs (CUcheckpointGpuPair) that first appear in the CUDA 13 |
| 885 | # headers, so CUDA 12.x images skip the snapshot runtime. |
| 886 | COPY tools/install_snapshot_runtime.sh /usr/local/bin/install_snapshot_runtime.sh |
| 887 | RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ] \ |
| 888 | && [ "${CUDA_VERSION%%.*}" -ge 13 ]; then \ |
| 889 | MAX_JOBS="${max_jobs}" /usr/local/bin/install_snapshot_runtime.sh; \ |
| 890 | else \ |
| 891 | echo "Initialized snapshots are not installed for ${TARGETPLATFORM} with CUDA ${CUDA_VERSION}"; \ |
| 892 | fi |
| 893 | |
| 894 | ENV CRIU_CUDA_PLUGIN_DIR=/usr/local/lib/criu |
| 895 | |
| 896 | # Install uv for faster pip installs |
| 897 | RUN python3 -m pip install uv |
| 898 | |
| 899 | # Environment for uv |
| 900 | # Redirect uv's managed Python and download cache out of /root/ so downstream |
| 901 | # images (`FROM vllm/vllm-openai` + `USER <uid>`) and direct non-root runs |
| 902 | # (`docker run --user <uid>:<gid>`) can read and execute them. See #15174, |
| 903 | # #15359, #31959. |
| 904 | ENV UV_HTTP_TIMEOUT=500 |
| 905 | ENV UV_INDEX_STRATEGY="unsafe-best-match" |
| 906 | ENV UV_LINK_MODE=copy |
| 907 | ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python |
| 908 | ENV UV_CACHE_DIR=/opt/uv/cache |
| 909 | ENV UV_COMPILE_BYTECODE=1 |
| 910 | RUN mkdir -p "${UV_PYTHON_INSTALL_DIR}" "${UV_CACHE_DIR}" \ |
| 911 | && chgrp -R 0 /opt/uv \ |
| 912 | && chmod -R g+rwX,a+rX /opt/uv |
| 913 | |
| 914 | # DeepEPv2 GIN requires NCCL >= 2.30.4 at both compile and runtime. torch pins |
| 915 | # an older version as a transitive dep; this override forces uv to use our |
| 916 | # pinned version whenever nvidia-nccl-cu* is resolved. |
| 917 | RUN CUDA_MAJOR=$(echo $CUDA_VERSION | cut -d. -f1) && \ |
| 918 | if [ -n "$NCCL_VERSION" ]; then \ |
| 919 | echo "nvidia-nccl-cu${CUDA_MAJOR}==${NCCL_VERSION}" \ |
| 920 | > /etc/uv-overrides.txt; \ |
| 921 | else \ |
| 922 | touch /etc/uv-overrides.txt; \ |
| 923 | fi |
| 924 | ENV UV_OVERRIDE=/etc/uv-overrides.txt |
| 925 | |
| 926 | # ---------------------------------------------------------------------- |
| 927 | # Non-root support (opt-in) |
| 928 | # ---------------------------------------------------------------------- |
| 929 | # Create a conventional `vllm` user (UID 2000, GID 0) so the image can be |
| 930 | # run under `--user 2000:0` or the opt-in `vllm-openai-nonroot` target. |
| 931 | # |
| 932 | # Design notes: |
| 933 | # * GID 0 + group-writable cache dirs follow the OpenShift arbitrary-UID |
| 934 | # pattern, so any UID that is a member of group 0 at runtime can write |
| 935 | # to /home/vllm and /opt/uv without additional chown work. |
| 936 | # * The default `vllm-openai` image keeps `USER root`, so every existing |
| 937 | # `docker run vllm/vllm-openai ...` / K8s manifest / `FROM vllm/vllm-openai` |
| 938 | # + `RUN uv pip install --system ...` flow is unchanged. |
| 939 | # * The entrypoint wrapper below is only used by `vllm-openai-nonroot`; it |
| 940 | # handles the OpenShift arbitrary-UID case (UID not in /etc/passwd). |
| 941 | # See #31959 and docs/deployment/docker.md. |
| 942 | RUN useradd --uid 2000 --gid 0 --create-home --home-dir /home/vllm \ |
| 943 | --shell /bin/bash vllm \ |
| 944 | && mkdir -p /home/vllm/.cache /home/vllm/.config \ |
| 945 | && chown -R 2000:0 /home/vllm \ |
| 946 | && chmod -R g+rwX /home/vllm \ |
| 947 | # Allow the entrypoint wrapper to append a /etc/passwd entry for an |
| 948 | # arbitrary runtime UID that shares GID 0. Without this, `whoami`, bash's |
| 949 | # `\u` prompt, `id -un`, and anything else that calls `getpwuid()` |
| 950 | # directly return "I have no name!" for OpenShift-style arbitrary UIDs. |
| 951 | # This matches the convention used by Red Hat UBI base images. |
| 952 | && chgrp 0 /etc/passwd /etc/group \ |
| 953 | && chmod g=u /etc/passwd /etc/group |
| 954 | COPY docker/entrypoints/vllm-nonroot-entrypoint.sh \ |
| 955 | /usr/local/bin/vllm-nonroot-entrypoint.sh |
| 956 | RUN chmod 0755 /usr/local/bin/vllm-nonroot-entrypoint.sh |
| 957 | |
| 958 | # Enable CUDA forward compatibility by setting '-e VLLM_ENABLE_CUDA_COMPATIBILITY=1' |
| 959 | # Only needed for datacenter/professional GPUs with older drivers. |
| 960 | # See: https://docs.nvidia.com/deploy/cuda-compatibility/ |
| 961 | ENV VLLM_ENABLE_CUDA_COMPATIBILITY=0 |
| 962 | |
| 963 | # ============================================================ |
| 964 | # SLOW-CHANGING DEPENDENCIES BELOW |
| 965 | # These are the expensive layers that we want to cache |
| 966 | # ============================================================ |
| 967 | |
| 968 | # Install PyTorch and core CUDA dependencies |
| 969 | # This is ~2GB and rarely changes |
| 970 | ARG PYTORCH_CUDA_INDEX_BASE_URL |
| 971 | COPY requirements/common.txt /tmp/common.txt |
| 972 | COPY requirements/cuda.txt /tmp/requirements-cuda.txt |
| 973 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 974 | --mount=type=bind,source=requirements,target=/tmp/vllm-requirements,ro \ |
| 975 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ] && \ |
| 976 | [ ! -f /tmp/vllm-requirements/rubin-prerelease.txt ]; then \ |
| 977 | echo "INSTALL_RUBIN_PRERELEASE=true requires requirements/rubin-prerelease.txt" >&2; \ |
| 978 | exit 1; \ |
| 979 | fi && \ |
| 980 | if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ |
| 981 | sed -i 's/^nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' /tmp/requirements-cuda.txt; \ |
| 982 | sed -i 's/^humming-kernels\[cu13\]/humming-kernels[cu12]/' /tmp/requirements-cuda.txt; \ |
| 983 | fi && \ |
| 984 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 985 | uv pip install --system \ |
| 986 | --prerelease=allow --index-strategy unsafe-first-match \ |
| 987 | -r /tmp/vllm-requirements/rubin-prerelease.txt \ |
| 988 | && sed -Ei '/^[[:space:]]*(torch|torchvision|torchaudio)[[:space:]]*==/Id' \ |
| 989 | /tmp/requirements-cuda.txt \ |
| 990 | && uv pip install --system -r /tmp/requirements-cuda.txt \ |
| 991 | && uv pip install --system \ |
| 992 | --prerelease=allow --index-strategy unsafe-first-match \ |
| 993 | -r /tmp/vllm-requirements/rubin-prerelease.txt \ |
| 994 | && printf '%s\n' \ |
| 995 | "/usr/local/lib/python${PYTHON_VERSION}/dist-packages/nvpl/lib" \ |
| 996 | > /etc/ld.so.conf.d/vllm-rubin-python.conf \ |
| 997 | && ldconfig; \ |
| 998 | else \ |
| 999 | uv pip install --system \ |
| 1000 | -r /tmp/requirements-cuda.txt \ |
| 1001 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1002 | fi && \ |
| 1003 | rm /tmp/requirements-cuda.txt /tmp/common.txt |
| 1004 | |
| 1005 | # Install FlashInfer JIT cache (requires CUDA-version-specific index URL) |
| 1006 | # https://docs.flashinfer.ai/installation.html |
| 1007 | # From versions.json: .flashinfer.version |
| 1008 | ARG FLASHINFER_VERSION=0.7.0.post1 |
| 1009 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 1010 | if [ "${INSTALL_RUBIN_PRERELEASE}" != "true" ]; then \ |
| 1011 | uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \ |
| 1012 | --index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1013 | fi |
| 1014 | |
| 1015 | # ============================================================ |
| 1016 | # OPENAI API SERVER DEPENDENCIES |
| 1017 | # Pre-install these to avoid reinstalling on every vLLM wheel rebuild |
| 1018 | # ============================================================ |
| 1019 | |
| 1020 | # Install gdrcopy (saves ~6s per build) |
| 1021 | # TODO (huydhn): There is no prebuilt gdrcopy package on 12.9 at the moment |
| 1022 | ARG GDRCOPY_CUDA_VERSION=12.8 |
| 1023 | ARG GDRCOPY_OS_VERSION=Ubuntu24_04 |
| 1024 | ARG TARGETPLATFORM |
| 1025 | COPY tools/install_gdrcopy.sh /tmp/install_gdrcopy.sh |
| 1026 | # Some Rubin FINAL_BASE_IMAGE choices (internal Rubin CUDA images, |
| 1027 | # nvcr.io/nvidia/cuda-dl-base) already provide GDRCopy; others don't, so it |
| 1028 | # falls through to the normal install below. Check reality (ldconfig) rather |
| 1029 | # than inferring from which FINAL_BASE_IMAGE flag is set. |
| 1030 | RUN set -eux; \ |
| 1031 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ] && ldconfig -p | grep -q 'libgdrapi\.so'; then \ |
| 1032 | true; \ |
| 1033 | else \ |
| 1034 | case "${TARGETPLATFORM}" in \ |
| 1035 | linux/arm64) UUARCH="aarch64" ;; \ |
| 1036 | linux/amd64) UUARCH="x64" ;; \ |
| 1037 | *) echo "Unsupported TARGETPLATFORM: ${TARGETPLATFORM}" >&2; exit 1 ;; \ |
| 1038 | esac; \ |
| 1039 | /tmp/install_gdrcopy.sh \ |
| 1040 | "${GDRCOPY_OS_VERSION}" "${GDRCOPY_CUDA_VERSION}" "${UUARCH}"; \ |
| 1041 | fi; \ |
| 1042 | rm /tmp/install_gdrcopy.sh |
| 1043 | |
| 1044 | # Install vllm-openai dependencies (saves ~2.6s per build) |
| 1045 | # These are stable packages that don't depend on vLLM itself |
| 1046 | # From versions.json: .openai_server_extras.timm, .openai_server_extras.runai_model_streamer |
| 1047 | ARG TIMM_VERSION=">=1.0.17" |
| 1048 | ARG RUNAI_MODEL_STREAMER_VERSION=">=0.15.7" |
| 1049 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 1050 | uv pip install --system accelerate 'modelscope<1.38' \ |
| 1051 | "timm${TIMM_VERSION}" "runai-model-streamer[s3,gcs,azure]${RUNAI_MODEL_STREAMER_VERSION}" |
| 1052 | |
| 1053 | # ============================================================ |
| 1054 | # VLLM INSTALLATION (depends on build stage) |
| 1055 | # ============================================================ |
| 1056 | |
| 1057 | FROM vllm-runtime-base AS vllm-base |
| 1058 | |
| 1059 | ARG PIP_INDEX_URL UV_INDEX_URL |
| 1060 | ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL |
| 1061 | ARG PYTORCH_CUDA_INDEX_BASE_URL |
| 1062 | ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER |
| 1063 | |
| 1064 | # We can specify the standard or nightly build of PyTorch |
| 1065 | ARG PYTORCH_NIGHTLY |
| 1066 | |
| 1067 | # Install vLLM wheel first, so that torch etc will be installed. |
| 1068 | # Check whether to install torch nightly instead of release for this build. |
| 1069 | COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt |
| 1070 | # Copy only the wheel checksum (a few bytes) so a wheel change invalidates this |
| 1071 | # install layer. The wheel itself is bind-mounted below and never enters the |
| 1072 | # image. Without this the bind mount is not part of the layer cache key, so a |
| 1073 | # warm BuildKit agent can skip the install and ship a stale wheel. |
| 1074 | COPY --from=build /workspace/dist/wheel.sha256 /tmp/vllm-wheel.sha256 |
| 1075 | RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \ |
| 1076 | --mount=type=cache,target=/opt/uv/cache \ |
| 1077 | if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ |
| 1078 | echo "Installing torch nightly..." \ |
| 1079 | && uv pip install --system $(cat torch_lib_versions.txt | xargs) --pre \ |
| 1080 | --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \ |
| 1081 | && echo "Installing vLLM..." \ |
| 1082 | && uv pip install --system dist/*.whl --verbose \ |
| 1083 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1084 | else \ |
| 1085 | echo "Installing vLLM..." \ |
| 1086 | && uv pip install --system dist/*.whl --verbose \ |
| 1087 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1088 | fi |
| 1089 | |
| 1090 | # Expose the Rust frontend bundled in the wheel on the container's PATH. |
| 1091 | RUN ln -s "$(python3 -c 'import sysconfig; print(sysconfig.get_path("platlib"))')/vllm/vllm-rs" /usr/local/bin/vllm-rs \ |
| 1092 | && vllm-rs --version |
| 1093 | |
| 1094 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 1095 | . /etc/environment && \ |
| 1096 | uv pip list |
| 1097 | |
| 1098 | # Pytorch now installs NVSHMEM, setting LD_LIBRARY_PATH |
| 1099 | ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH |
| 1100 | |
| 1101 | # Install EP kernels wheels (DeepEP) that have been built in the `build` stage. |
| 1102 | # As with the vLLM wheel above, copy only the checksum to bust the layer cache |
| 1103 | # and bind-mount the wheel for the actual install to keep it out of the image. |
| 1104 | COPY --from=build /tmp/ep_kernels_workspace/dist/wheels.sha256 /tmp/ep-kernels-wheels.sha256 |
| 1105 | RUN --mount=type=bind,from=build,src=/tmp/ep_kernels_workspace/dist,target=/vllm-workspace/ep_kernels/dist \ |
| 1106 | --mount=type=cache,target=/opt/uv/cache \ |
| 1107 | uv pip install --system ep_kernels/dist/*.whl --verbose \ |
| 1108 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') |
| 1109 | |
| 1110 | # CUDA image changed from /usr/local/nvidia to /usr/local/cuda in 12.8 but will |
| 1111 | # return to /usr/local/nvidia in 13.0 to allow container providers to mount drivers |
| 1112 | # consistently from the host (see https://github.com/vllm-project/vllm/issues/18859). |
| 1113 | # Until then, add /usr/local/nvidia/lib64 before the image cuda path to allow override. |
| 1114 | ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH} |
| 1115 | |
| 1116 | # Copy examples and benchmarks at the end to minimize cache invalidation |
| 1117 | COPY examples examples |
| 1118 | COPY benchmarks benchmarks |
| 1119 | COPY ./vllm/collect_env.py . |
| 1120 | #################### vLLM installation IMAGE #################### |
| 1121 | #################### TEST IMAGE #################### |
| 1122 | # Stable dependencies for the unit testing image |
| 1123 | FROM vllm-runtime-base AS test-deps |
| 1124 | |
| 1125 | ARG PYTHON_VERSION |
| 1126 | ARG TARGETPLATFORM |
| 1127 | |
| 1128 | ARG PIP_INDEX_URL UV_INDEX_URL |
| 1129 | ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL |
| 1130 | ARG PYTORCH_CUDA_INDEX_BASE_URL |
| 1131 | ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER |
| 1132 | |
| 1133 | # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out |
| 1134 | # Reference: https://github.com/astral-sh/uv/pull/1694 |
| 1135 | ENV UV_HTTP_TIMEOUT=500 |
| 1136 | ENV UV_INDEX_STRATEGY="unsafe-best-match" |
| 1137 | # Use copy mode to avoid hardlink failures with Docker cache mounts |
| 1138 | ENV UV_LINK_MODE=copy |
| 1139 | |
| 1140 | RUN apt-get update -y \ |
| 1141 | && apt-get install -y git |
| 1142 | |
| 1143 | # We can specify the standard or nightly build of PyTorch |
| 1144 | ARG PYTORCH_NIGHTLY |
| 1145 | |
| 1146 | # Install development dependencies (for testing) |
| 1147 | COPY requirements/common.txt requirements/common.txt |
| 1148 | COPY requirements/lint.txt requirements/lint.txt |
| 1149 | COPY requirements/test/cuda.in requirements/test/cuda.in |
| 1150 | COPY requirements/test/cuda.txt requirements/test/cuda.txt |
| 1151 | COPY requirements/dev.txt requirements/dev.txt |
| 1152 | # uv reads `[tool.uv] exclude-dependencies` from here when recompiling below. |
| 1153 | COPY pyproject.toml pyproject.toml |
| 1154 | COPY tools/use_existing_torch.py tools/use_existing_torch.py |
| 1155 | COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt |
| 1156 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 1157 | CUDA_MAJOR="${CUDA_VERSION%%.*}"; \ |
| 1158 | if [ "$CUDA_MAJOR" -ge 12 ]; then \ |
| 1159 | if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ |
| 1160 | echo "Installing dev requirements plus torch nightly..." \ |
| 1161 | && python3 tools/use_existing_torch.py --prefix \ |
| 1162 | && cat torch_lib_versions.txt >> requirements/test/cuda.in \ |
| 1163 | && uv pip compile requirements/test/cuda.in -o requirements/test/cuda.txt --index-strategy unsafe-best-match \ |
| 1164 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \ |
| 1165 | && uv pip install --system $(cat torch_lib_versions.txt | xargs) --pre \ |
| 1166 | -r requirements/dev.txt \ |
| 1167 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1168 | else \ |
| 1169 | echo "Installing dev requirements..." \ |
| 1170 | && if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ |
| 1171 | echo "Recompiling test requirements for arm64..." \ |
| 1172 | && uv pip compile requirements/test/cuda.in -o requirements/test/cuda.txt --index-strategy unsafe-best-match \ |
| 1173 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1174 | fi \ |
| 1175 | && uv pip install --system -r requirements/dev.txt \ |
| 1176 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1177 | fi \ |
| 1178 | fi |
| 1179 | |
| 1180 | # image to run unit testing suite |
| 1181 | # note that this uses vllm installed by `pip` |
| 1182 | FROM test-deps AS test |
| 1183 | |
| 1184 | ARG PYTHON_VERSION |
| 1185 | |
| 1186 | ARG PIP_INDEX_URL UV_INDEX_URL |
| 1187 | ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL |
| 1188 | ARG PYTORCH_CUDA_INDEX_BASE_URL |
| 1189 | ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER |
| 1190 | |
| 1191 | # We can specify the standard or nightly build of PyTorch |
| 1192 | ARG PYTORCH_NIGHTLY |
| 1193 | |
| 1194 | # Install vLLM and EP kernel wheels after the stable test dependencies so |
| 1195 | # source-only changes do not invalidate the dependency layers above. |
| 1196 | COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt |
| 1197 | COPY --from=build /workspace/dist/wheel.sha256 /tmp/vllm-wheel.sha256 |
| 1198 | RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \ |
| 1199 | --mount=type=cache,target=/opt/uv/cache \ |
| 1200 | if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ |
| 1201 | echo "Installing torch nightly..." \ |
| 1202 | && uv pip install --system $(cat torch_lib_versions.txt | xargs) --pre \ |
| 1203 | --index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \ |
| 1204 | && echo "Installing vLLM..." \ |
| 1205 | && uv pip install --system dist/*.whl --verbose \ |
| 1206 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/nightly/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1207 | else \ |
| 1208 | echo "Installing vLLM..." \ |
| 1209 | && uv pip install --system dist/*.whl --verbose \ |
| 1210 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ |
| 1211 | fi |
| 1212 | |
| 1213 | # Pytorch now installs NVSHMEM, setting LD_LIBRARY_PATH |
| 1214 | ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH |
| 1215 | |
| 1216 | COPY --from=build /tmp/ep_kernels_workspace/dist/wheels.sha256 /tmp/ep-kernels-wheels.sha256 |
| 1217 | RUN --mount=type=bind,from=build,src=/tmp/ep_kernels_workspace/dist,target=/vllm-workspace/ep_kernels/dist \ |
| 1218 | --mount=type=cache,target=/opt/uv/cache \ |
| 1219 | uv pip install --system ep_kernels/dist/*.whl --verbose \ |
| 1220 | --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') |
| 1221 | |
| 1222 | # See the matching comment in the vllm-base stage. |
| 1223 | ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH} |
| 1224 | |
| 1225 | ADD . /vllm-workspace/ |
| 1226 | |
| 1227 | # install development dependencies (for testing) |
| 1228 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 1229 | uv pip install --system -e tests/vllm_test_utils |
| 1230 | |
| 1231 | # enable fast downloads from hf (for testing) |
| 1232 | ENV HF_XET_HIGH_PERFORMANCE 1 |
| 1233 | |
| 1234 | # increase timeout for hf downloads (for testing) |
| 1235 | ENV HF_HUB_DOWNLOAD_TIMEOUT 60 |
| 1236 | |
| 1237 | # Catch GPU<->CPU syncs in execute_model/sample_tokens |
| 1238 | ENV VLLM_GPU_SYNC_CHECK=error |
| 1239 | |
| 1240 | # Copy in the v1 package for testing (it isn't distributed yet) |
| 1241 | COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1 |
| 1242 | |
| 1243 | # Source code is used in the `python_only_compile.sh` test |
| 1244 | # We hide it inside `src/` so that this source code |
| 1245 | # will not be imported by other tests |
| 1246 | RUN mkdir src |
| 1247 | RUN mv vllm src/vllm |
| 1248 | #################### TEST IMAGE #################### |
| 1249 | |
| 1250 | #################### OPENAI API SERVER #################### |
| 1251 | # base openai image with additional requirements, for any subsequent openai-style images |
| 1252 | FROM vllm-base AS vllm-openai-base |
| 1253 | ARG TARGETPLATFORM |
| 1254 | ARG INSTALL_KV_CONNECTORS=false |
| 1255 | ARG INSTALL_RUBIN_PRERELEASE |
| 1256 | ARG TRITON_INSTALL_FROM_SOURCE_REPO |
| 1257 | ARG CUDA_VERSION |
| 1258 | ARG VLLM_BUILD_COMMIT |
| 1259 | ARG VLLM_BUILD_PIPELINE |
| 1260 | ARG VLLM_BUILD_URL |
| 1261 | ARG VLLM_IMAGE_TAG |
| 1262 | |
| 1263 | ARG PIP_INDEX_URL UV_INDEX_URL |
| 1264 | ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL |
| 1265 | |
| 1266 | # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out |
| 1267 | # Reference: https://github.com/astral-sh/uv/pull/1694 |
| 1268 | ENV UV_HTTP_TIMEOUT=500 |
| 1269 | |
| 1270 | # install kv_connectors if requested |
| 1271 | # Do not add +PTX here; see the main TORCH_CUDA_ARCH_LIST comment above. |
| 1272 | ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0' |
| 1273 | ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} |
| 1274 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 1275 | --mount=type=bind,source=requirements,target=/tmp/vllm-requirements,ro \ |
| 1276 | set -eu; \ |
| 1277 | CUDA_MAJOR="${CUDA_VERSION%%.*}"; \ |
| 1278 | CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-'); \ |
| 1279 | CUDA_HOME=/usr/local/cuda; \ |
| 1280 | # lmcache requires explicit specifying CUDA_HOME |
| 1281 | BUILD_PKGS="libcusparse-dev-${CUDA_VERSION_DASH} \ |
| 1282 | libcublas-dev-${CUDA_VERSION_DASH} \ |
| 1283 | libcusolver-dev-${CUDA_VERSION_DASH}"; \ |
| 1284 | if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \ |
| 1285 | KV_REQUIREMENTS=/tmp/vllm-requirements/kv_connectors.txt; \ |
| 1286 | if [ "${CUDA_MAJOR}" = "12" ]; then \ |
| 1287 | KV_REQUIREMENTS=/tmp/vllm-requirements/kv_connectors_cu12.txt; \ |
| 1288 | fi; \ |
| 1289 | # Preserve the stack used to compile vLLM when resolving optional dependencies. \ |
| 1290 | uv pip freeze --system > /tmp/kv-installed.txt; \ |
| 1291 | grep -E '^(torch|torchaudio|torchvision|triton|cuda-bindings|cuda-python|cuda-toolkit)==' \ |
| 1292 | /tmp/kv-installed.txt > /tmp/kv-constraints.txt; \ |
| 1293 | uv pip install --system -c /tmp/kv-constraints.txt -r "${KV_REQUIREMENTS}" --no-build || ( \ |
| 1294 | # if the above fails, install from source |
| 1295 | apt-get update -y && \ |
| 1296 | apt-get install -y --no-install-recommends --allow-change-held-packages ${BUILD_PKGS} && \ |
| 1297 | uv pip install --system -c /tmp/kv-constraints.txt -r "${KV_REQUIREMENTS}" --no-build-isolation && \ |
| 1298 | apt-get purge -y ${BUILD_PKGS} && \ |
| 1299 | # clean up -dev packages, keep runtime libraries |
| 1300 | rm -rf /var/lib/apt/lists/* \ |
| 1301 | ); \ |
| 1302 | # The default mooncake wheel links libcudart.so.12, so on CUDA 13 swap in the |
| 1303 | # cuda13 variant. Uninstall first; both ship the `mooncake` package. |
| 1304 | # Mirrors .buildkite/scripts/install-kv-connectors.sh. |
| 1305 | MOONCAKE_VERSION=$(python3 -c "import importlib.metadata as m; print(m.version('mooncake-transfer-engine'))" 2>/dev/null || true) && \ |
| 1306 | if [ "${CUDA_MAJOR}" = "13" ] && [ -n "${MOONCAKE_VERSION}" ]; then \ |
| 1307 | uv pip uninstall --system mooncake-transfer-engine 2>/dev/null || true; \ |
| 1308 | uv pip install --system -c /tmp/kv-constraints.txt "mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"; \ |
| 1309 | fi; \ |
| 1310 | rm -f /tmp/kv-installed.txt /tmp/kv-constraints.txt; \ |
| 1311 | fi |
| 1312 | |
| 1313 | # Mooncake registers GPU memory for RDMA via nvidia-peermem by default. Hosts |
| 1314 | # without that module loaded (e.g. GB200) must pass WITH_NVIDIA_PEERMEM=0 at run |
| 1315 | # time; see docs/features/mooncake_connector_usage.md. Deliberately not an ENV |
| 1316 | # here so the image does not override mooncake's own default. |
| 1317 | |
| 1318 | # Install Rubin prerelease dependencies when explicitly requested. Keep this |
| 1319 | # after the public vLLM and KV connector dependency pins installed above. |
| 1320 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 1321 | --mount=type=bind,source=requirements,target=/tmp/vllm-requirements,ro \ |
| 1322 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 1323 | echo "Installing Rubin prerelease requirements..." \ |
| 1324 | && uv pip install --system \ |
| 1325 | --prerelease=allow --index-strategy unsafe-first-match \ |
| 1326 | -r /tmp/vllm-requirements/rubin-prerelease.txt; \ |
| 1327 | fi |
| 1328 | |
| 1329 | # Install source-built Triton last so dependency resolution cannot restore the |
| 1330 | # previously installed version. |
| 1331 | COPY --from=extensions-build \ |
| 1332 | /tmp/triton_from_source_workspace/dist/wheel.sha256 \ |
| 1333 | /tmp/triton-source-wheel.sha256 |
| 1334 | COPY tools/install_triton_from_source.sh /tmp/install_triton_from_source.sh |
| 1335 | RUN --mount=type=cache,target=/opt/uv/cache \ |
| 1336 | --mount=type=bind,from=extensions-build,source=/tmp/triton_from_source_workspace/dist,target=/tmp/triton-source-dist,ro \ |
| 1337 | if [ -n "${TRITON_INSTALL_FROM_SOURCE_REPO}" ]; then \ |
| 1338 | CUDA_MAJOR_MINOR=$(echo "${CUDA_VERSION}" | cut -d. -f1,2); \ |
| 1339 | if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ |
| 1340 | case "${CUDA_MAJOR_MINOR}" in \ |
| 1341 | 13.4|13.5) \ |
| 1342 | TRITON_SYSTEM_PTXAS_PATH=/usr/local/cuda/bin/ptxas \ |
| 1343 | TRITON_EXPECTED_PTXAS_VERSION="${CUDA_MAJOR_MINOR}" \ |
| 1344 | bash /tmp/install_triton_from_source.sh \ |
| 1345 | /tmp/triton-source-dist \ |
| 1346 | /tmp/triton-source-wheel.sha256 \ |
| 1347 | ;; \ |
| 1348 | *) \ |
| 1349 | echo "Unsupported Rubin CUDA version: ${CUDA_MAJOR_MINOR}" >&2; \ |
| 1350 | exit 1 \ |
| 1351 | ;; \ |
| 1352 | esac; \ |
| 1353 | else \ |
| 1354 | bash /tmp/install_triton_from_source.sh \ |
| 1355 | /tmp/triton-source-dist \ |
| 1356 | /tmp/triton-source-wheel.sha256; \ |
| 1357 | fi; \ |
| 1358 | fi \ |
| 1359 | && rm -f \ |
| 1360 | /tmp/install_triton_from_source.sh \ |
| 1361 | /tmp/triton-source-wheel.sha256 |
| 1362 | |
| 1363 | ENV VLLM_USAGE_SOURCE production-docker-image |
| 1364 | ENV VLLM_BUILD_COMMIT=${VLLM_BUILD_COMMIT:-unknown} \ |
| 1365 | VLLM_BUILD_PIPELINE=${VLLM_BUILD_PIPELINE:-local} \ |
| 1366 | VLLM_BUILD_URL=${VLLM_BUILD_URL:-} \ |
| 1367 | VLLM_IMAGE_TAG=${VLLM_IMAGE_TAG:-local/vllm-openai:dev} |
| 1368 | LABEL org.opencontainers.image.source="https://github.com/vllm-project/vllm" \ |
| 1369 | org.opencontainers.image.revision="${VLLM_BUILD_COMMIT}" \ |
| 1370 | org.opencontainers.image.version="${VLLM_IMAGE_TAG}" \ |
| 1371 | org.opencontainers.image.url="${VLLM_BUILD_URL}" \ |
| 1372 | ai.vllm.build.commit="${VLLM_BUILD_COMMIT}" \ |
| 1373 | ai.vllm.build.pipeline="${VLLM_BUILD_PIPELINE}" \ |
| 1374 | ai.vllm.build.url="${VLLM_BUILD_URL}" \ |
| 1375 | ai.vllm.image.tag="${VLLM_IMAGE_TAG}" |
| 1376 | |
| 1377 | # define sagemaker first, so it is not default from `docker build` |
| 1378 | FROM vllm-openai-base AS vllm-sagemaker |
| 1379 | |
| 1380 | COPY examples/deployment/sagemaker-entrypoint.sh . |
| 1381 | RUN chmod +x sagemaker-entrypoint.sh |
| 1382 | ENTRYPOINT ["./sagemaker-entrypoint.sh"] |
| 1383 | |
| 1384 | FROM vllm-openai-base AS vllm-openai |
| 1385 | |
| 1386 | # To run the image as non-root, either build the `vllm-openai-nonroot` target |
| 1387 | # below, or in a derived Dockerfile uncomment the following line and ensure |
| 1388 | # any additional layers chgrp-0 / chmod-g+rwX paths they write to. The `vllm` |
| 1389 | # user (UID 2000, GID 0) is already created in the `vllm-base` stage. |
| 1390 | # See docs/deployment/docker.md. |
| 1391 | # USER vllm |
| 1392 | ENTRYPOINT ["vllm", "serve"] |
| 1393 | #################### OPENAI API SERVER #################### |
| 1394 | |
| 1395 | #################### OPENAI API SERVER (NON-ROOT, OPT-IN) #################### |
| 1396 | # Non-root-ready variant of `vllm-openai`. Built via: |
| 1397 | # docker build --target vllm-openai-nonroot -t vllm:openai-nonroot \ |
| 1398 | # -f docker/Dockerfile . |
| 1399 | # |
| 1400 | # Runtime behavior: |
| 1401 | # * Default USER is `vllm` (UID 2000, GID 0) created in `vllm-base`. |
| 1402 | # * HOME is /home/vllm, pre-created group-0-writable so arbitrary UIDs in |
| 1403 | # group 0 (OpenShift / `--user <uid>:0`) can also use the image. |
| 1404 | # * Entrypoint wrapper handles the "UID not in /etc/passwd" case for truly |
| 1405 | # arbitrary UIDs by falling back HOME/USER to sane writable defaults. |
| 1406 | # * All cache/config envs (HF_HOME, VLLM_CACHE_ROOT, TRITON_CACHE_DIR, ...) |
| 1407 | # remain unset so their library defaults resolve to $HOME/.cache/... , |
| 1408 | # which is writable. |
| 1409 | FROM vllm-openai AS vllm-openai-nonroot |
| 1410 | |
| 1411 | USER vllm |
| 1412 | WORKDIR /home/vllm |
| 1413 | ENTRYPOINT ["/usr/local/bin/vllm-nonroot-entrypoint.sh"] |
| 1414 | #################### OPENAI API SERVER (NON-ROOT, OPT-IN) #################### |