A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 10:00 UTC
[docker] Restructure Dockerfile for more efficient and cache-friendly builds (#30626)
Signed-off-by: Amr Mahdi <amrmahdi@meta.com>
Amr Mahdi committed
Dec 16, 2025 at 04:52 UTC
ff21a0fc859390385d4da3363c23f43eacefd5c6
2 files changed
+157
-115
docker/Dockerfile
+157
-115
@@ -32,7 +32,7 @@ ARG DEADSNAKES_GPGKEY_URL
32
33
# The PyPA get-pip.py script is a self contained script+zip file, that provides
34
# both the installer script and the pip base85-encoded zip archive. This allows
35
-# bootstrapping pip in environment where a dsitribution package does not exist.
35
+# bootstrapping pip in environment where a distribution package does not exist.
36
#
37
# By parameterizing the URL for get-pip.py installation script, we allow
38
# third-party to use their own copy of the script stored in a private mirror.
@@ -73,15 +73,13 @@ ARG INSTALL_KV_CONNECTORS=false
73
#################### BASE BUILD IMAGE ####################
74
# prepare basic build environment
75
FROM ${BUILD_BASE_IMAGE} AS base
76
+
77
ARG CUDA_VERSION
78
ARG PYTHON_VERSION
78
-ARG TARGETPLATFORM
79
-ARG INSTALL_KV_CONNECTORS=false
80
-ENV DEBIAN_FRONTEND=noninteractive
79
82
-ARG GET_PIP_URL
80
+ENV DEBIAN_FRONTEND=noninteractive
81
84
-# Install system dependencies and uv, then create Python virtual environment
82
+# Install system dependencies including build tools
83
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
84
&& echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
85
&& apt-get update -y \
@@ -107,32 +105,30 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
105
&& ln -s /opt/venv/bin/pip /usr/bin/pip \
106
&& python3 --version && python3 -m pip --version
107
110
-ARG PIP_INDEX_URL UV_INDEX_URL
111
-ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
112
-ARG PYTORCH_CUDA_INDEX_BASE_URL
113
-ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER
114
-
108
# Activate virtual environment and add uv to PATH
109
ENV PATH="/opt/venv/bin:/root/.local/bin:$PATH"
110
ENV VIRTUAL_ENV="/opt/venv"
111
119
-# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
120
-# Reference: https://github.com/astral-sh/uv/pull/1694
112
+# Environment for uv
113
ENV UV_HTTP_TIMEOUT=500
114
ENV UV_INDEX_STRATEGY="unsafe-best-match"
123
-# Use copy mode to avoid hardlink failures with Docker cache mounts
115
ENV UV_LINK_MODE=copy
116
126
-RUN <<EOF
127
-gcc --version
128
-EOF
117
+# Verify GCC version
118
+RUN gcc --version
119
130
-# Workaround for https://github.com/openai/triton/issues/2507 and
131
-# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
132
-# this won't be needed for future versions of this docker image
133
-# or future versions of triton.
120
+# Workaround for triton/pytorch issues
121
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
122
123
+# ============================================================
124
+# SLOW-CHANGING DEPENDENCIES BELOW
125
+# These are the expensive layers that we want to cache
126
+# ============================================================
127
+
128
+# Install PyTorch and core CUDA dependencies
129
+# This is ~2GB and rarely changes
130
+ARG PYTORCH_CUDA_INDEX_BASE_URL
131
+
132
WORKDIR /workspace
133
134
# install build and runtime dependencies
@@ -142,13 +138,12 @@ RUN --mount=type=cache,target=/root/.cache/uv \
138
uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \
139
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
140
145
-# cuda arch list used by torch
146
-# can be useful for both `dev` and `test`
147
-# explicitly set the list to avoid issues with torch 2.2
148
-# see https://github.com/pytorch/pytorch/pull/123243
141
+# CUDA arch list used by torch
142
+# Explicitly set the list to avoid issues with torch 2.2
143
+# See https://github.com/pytorch/pytorch/pull/123243
144
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
145
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
151
-#################### BASE BUILD IMAGE ####################
146
+#################### BUILD BASE IMAGE ####################
147
148
#################### CSRC BUILD IMAGE ####################
149
FROM base AS csrc-build
@@ -241,6 +236,48 @@ RUN --mount=type=cache,target=/root/.cache/ccache \
236
fi
237
#################### CSRC BUILD IMAGE ####################
238
239
+#################### EXTENSIONS BUILD IMAGE ####################
240
+# Build DeepGEMM, pplx-kernels, DeepEP - runs in PARALLEL with csrc-build
241
+# This stage is independent and doesn't affect csrc cache
242
+FROM base AS extensions-build
243
+ARG CUDA_VERSION
244
+
245
+# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
246
+ENV UV_HTTP_TIMEOUT=500
247
+ENV UV_INDEX_STRATEGY="unsafe-best-match"
248
+ENV UV_LINK_MODE=copy
249
+
250
+WORKDIR /workspace
251
+
252
+# Build DeepGEMM wheel
253
+ARG DEEPGEMM_GIT_REF
254
+COPY tools/install_deepgemm.sh /tmp/install_deepgemm.sh
255
+RUN --mount=type=cache,target=/root/.cache/uv \
256
+ mkdir -p /tmp/deepgemm/dist && \
257
+ VLLM_DOCKER_BUILD_CONTEXT=1 TORCH_CUDA_ARCH_LIST="9.0a 10.0a" /tmp/install_deepgemm.sh \
258
+ --cuda-version "${CUDA_VERSION}" \
259
+ ${DEEPGEMM_GIT_REF:+--ref "$DEEPGEMM_GIT_REF"} \
260
+ --wheel-dir /tmp/deepgemm/dist || \
261
+ echo "DeepGEMM build skipped (CUDA version requirement not met)"
262
+
263
+# Ensure the wheel dir exists so COPY won't fail when DeepGEMM is skipped
264
+RUN mkdir -p /tmp/deepgemm/dist && touch /tmp/deepgemm/dist/.deepgemm_skipped
265
+
266
+# Build pplx-kernels and DeepEP wheels
267
+COPY tools/ep_kernels/install_python_libraries.sh /tmp/install_python_libraries.sh
268
+ARG PPLX_COMMIT_HASH
269
+ARG DEEPEP_COMMIT_HASH
270
+RUN --mount=type=cache,target=/root/.cache/uv \
271
+ mkdir -p /tmp/ep_kernels_workspace/dist && \
272
+ export TORCH_CUDA_ARCH_LIST='9.0a 10.0a' && \
273
+ /tmp/install_python_libraries.sh \
274
+ --workspace /tmp/ep_kernels_workspace \
275
+ --mode wheel \
276
+ ${PPLX_COMMIT_HASH:+--pplx-ref "$PPLX_COMMIT_HASH"} \
277
+ ${DEEPEP_COMMIT_HASH:+--deepep-ref "$DEEPEP_COMMIT_HASH"} && \
278
+ find /tmp/ep_kernels_workspace/nvshmem -name '*.a' -delete
279
+#################### EXTENSIONS BUILD IMAGE ####################
280
+
281
#################### WHEEL BUILD IMAGE ####################
282
FROM base AS build
283
ARG TARGETPLATFORM
@@ -265,6 +302,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
302
303
WORKDIR /workspace
304
305
+# Copy pre-built csrc wheel directly
306
COPY --from=csrc-build /workspace/dist /precompiled-wheels
307
308
COPY . .
@@ -286,27 +324,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
324
fi && \
325
python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
326
289
-# Install DeepGEMM from source
290
-ARG DEEPGEMM_GIT_REF
291
-COPY tools/install_deepgemm.sh /tmp/install_deepgemm.sh
292
-RUN --mount=type=cache,target=/root/.cache/uv \
293
- VLLM_DOCKER_BUILD_CONTEXT=1 TORCH_CUDA_ARCH_LIST="9.0a 10.0a" /tmp/install_deepgemm.sh --cuda-version "${CUDA_VERSION}" ${DEEPGEMM_GIT_REF:+--ref "$DEEPGEMM_GIT_REF"} --wheel-dir /tmp/deepgemm/dist
294
-
295
-# Ensure the wheel dir exists so later-stage COPY won't fail when DeepGEMM is skipped
296
-RUN mkdir -p /tmp/deepgemm/dist && touch /tmp/deepgemm/dist/.deepgemm_skipped
297
-
298
-COPY tools/ep_kernels/install_python_libraries.sh /tmp/install_python_libraries.sh
299
-# Install EP kernels(pplx-kernels and DeepEP)
300
-ARG PPLX_COMMIT_HASH
301
-ARG DEEPEP_COMMIT_HASH
302
-RUN --mount=type=cache,target=/root/.cache/uv \
303
- export TORCH_CUDA_ARCH_LIST='9.0a 10.0a' && \
304
- /tmp/install_python_libraries.sh \
305
- --workspace /tmp/ep_kernels_workspace \
306
- --mode wheel \
307
- ${PPLX_COMMIT_HASH:+--pplx-ref "$PPLX_COMMIT_HASH"} \
308
- ${DEEPEP_COMMIT_HASH:+--deepep-ref "$DEEPEP_COMMIT_HASH"} && \
309
- find /tmp/ep_kernels_workspace/nvshmem -name '*.a' -delete
327
+# Copy extension wheels from extensions-build stage for later use
328
+COPY --from=extensions-build /tmp/deepgemm/dist /tmp/deepgemm/dist
329
+COPY --from=extensions-build /tmp/ep_kernels_workspace/dist /tmp/ep_kernels_workspace/dist
330
331
# Check the size of the wheel if RUN_WHEEL_CHECK is true
332
COPY .buildkite/check-wheel-size.py check-wheel-size.py
@@ -344,32 +364,25 @@ RUN --mount=type=cache,target=/root/.cache/uv \
364
uv pip install --python /opt/venv/bin/python3 -r requirements/dev.txt \
365
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
366
#################### DEV IMAGE ####################
347
-
367
#################### vLLM installation IMAGE ####################
368
# image with vLLM installed
369
FROM ${FINAL_BASE_IMAGE} AS vllm-base
370
+
371
ARG CUDA_VERSION
372
ARG PYTHON_VERSION
353
-ARG INSTALL_KV_CONNECTORS=false
354
-WORKDIR /vllm-workspace
355
-ENV DEBIAN_FRONTEND=noninteractive
356
-ARG TARGETPLATFORM
357
-
358
-# TODO (huydhn): There is no prebuilt gdrcopy package on 12.9 at the moment
359
-ARG GDRCOPY_CUDA_VERSION=12.8
360
-# Keep in line with FINAL_BASE_IMAGE
361
-ARG GDRCOPY_OS_VERSION=Ubuntu22_04
362
-
363
-SHELL ["/bin/bash", "-c"]
364
-
373
ARG DEADSNAKES_MIRROR_URL
374
ARG DEADSNAKES_GPGKEY_URL
375
ARG GET_PIP_URL
376
377
+ENV DEBIAN_FRONTEND=noninteractive
378
+WORKDIR /vllm-workspace
379
+
380
+
381
+# Python version string for paths (e.g., "312" for 3.12)
382
RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \
383
echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment
384
372
-# Install Python and other dependencies
385
+# Install Python and system dependencies
386
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
387
&& echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
388
&& apt-get update -y \
@@ -408,62 +421,103 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
421
&& curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
422
&& python3 --version && python3 -m pip --version
423
411
-# Install CUDA development tools and build essentials for runtime JIT compilation
424
+# Install CUDA development tools for runtime JIT compilation
425
# (FlashInfer, DeepGEMM, EP kernels all require compilation at runtime)
426
RUN CUDA_VERSION_DASH=$(echo $CUDA_VERSION | cut -d. -f1,2 | tr '.' '-') && \
427
apt-get update -y && \
428
apt-get install -y --no-install-recommends \
416
- cuda-nvcc-${CUDA_VERSION_DASH} \
417
- cuda-cudart-${CUDA_VERSION_DASH} \
418
- cuda-nvrtc-${CUDA_VERSION_DASH} \
419
- cuda-cuobjdump-${CUDA_VERSION_DASH} \
420
- # https://github.com/vllm-project/vllm/issues/29590
421
- libcurand-dev-${CUDA_VERSION_DASH} \
422
- libcublas-${CUDA_VERSION_DASH} \
423
- # Fixes nccl_allocator requiring nccl.h at runtime
424
- # https://github.com/vllm-project/vllm/blob/1336a1ea244fa8bfd7e72751cabbdb5b68a0c11a/vllm/distributed/device_communicators/pynccl_allocator.py#L22
425
- libnccl-dev && \
429
+ cuda-nvcc-${CUDA_VERSION_DASH} \
430
+ cuda-cudart-${CUDA_VERSION_DASH} \
431
+ cuda-nvrtc-${CUDA_VERSION_DASH} \
432
+ cuda-cuobjdump-${CUDA_VERSION_DASH} \
433
+ libcurand-dev-${CUDA_VERSION_DASH} \
434
+ libcublas-${CUDA_VERSION_DASH} \
435
+ # Fixes nccl_allocator requiring nccl.h at runtime
436
+ # https://github.com/vllm-project/vllm/blob/1336a1ea244fa8bfd7e72751cabbdb5b68a0c11a/vllm/distributed/device_communicators/pynccl_allocator.py#L22
437
+ libnccl-dev && \
438
rm -rf /var/lib/apt/lists/*
439
428
-ARG PIP_INDEX_URL UV_INDEX_URL
429
-ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
430
-ARG PYTORCH_CUDA_INDEX_BASE_URL
431
-ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER
432
-
440
# Install uv for faster pip installs
434
-RUN --mount=type=cache,target=/root/.cache/uv \
435
- python3 -m pip install uv
441
+RUN python3 -m pip install uv
442
437
-# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
438
-# Reference: https://github.com/astral-sh/uv/pull/1694
443
+# Environment for uv
444
ENV UV_HTTP_TIMEOUT=500
445
ENV UV_INDEX_STRATEGY="unsafe-best-match"
441
-# Use copy mode to avoid hardlink failures with Docker cache mounts
446
ENV UV_LINK_MODE=copy
447
444
-# Workaround for https://github.com/openai/triton/issues/2507 and
445
-# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
446
-# this won't be needed for future versions of this docker image
447
-# or future versions of triton.
448
+# Workaround for triton/pytorch issues
449
RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
450
450
-# Install vllm wheel first, so that torch etc will be installed.
451
-RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
452
- --mount=type=cache,target=/root/.cache/uv \
453
- uv pip install --system dist/*.whl --verbose \
454
- --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
451
+# ============================================================
452
+# SLOW-CHANGING DEPENDENCIES BELOW
453
+# These are the expensive layers that we want to cache
454
+# ============================================================
455
+
456
+# Install PyTorch and core CUDA dependencies
457
+# This is ~2GB and rarely changes
458
+ARG PYTORCH_CUDA_INDEX_BASE_URL
459
+COPY requirements/common.txt /tmp/common.txt
460
+COPY requirements/cuda.txt /tmp/requirements-cuda.txt
461
+RUN --mount=type=cache,target=/root/.cache/uv \
462
+ uv pip install --system -r /tmp/requirements-cuda.txt \
463
+ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') && \
464
+ rm /tmp/requirements-cuda.txt /tmp/common.txt
465
466
# Install FlashInfer pre-compiled kernel cache and binaries
467
+# This is ~1.1GB and only changes when FlashInfer version bumps
468
# https://docs.flashinfer.ai/installation.html
469
+ARG FLASHINFER_VERSION=0.5.3
470
RUN --mount=type=cache,target=/root/.cache/uv \
459
- uv pip install --system flashinfer-cubin==0.5.3 \
460
- && uv pip install --system flashinfer-jit-cache==0.5.3 \
471
+ uv pip install --system flashinfer-cubin==${FLASHINFER_VERSION} \
472
+ && uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
473
--extra-index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') \
474
&& flashinfer show-config
475
464
-COPY examples examples
465
-COPY benchmarks benchmarks
466
-COPY ./vllm/collect_env.py .
476
+# ============================================================
477
+# OPENAI API SERVER DEPENDENCIES
478
+# Pre-install these to avoid reinstalling on every vLLM wheel rebuild
479
+# ============================================================
480
+
481
+# Install gdrcopy (saves ~6s per build)
482
+# TODO (huydhn): There is no prebuilt gdrcopy package on 12.9 at the moment
483
+ARG GDRCOPY_CUDA_VERSION=12.8
484
+ARG GDRCOPY_OS_VERSION=Ubuntu22_04
485
+ARG TARGETPLATFORM
486
+COPY tools/install_gdrcopy.sh /tmp/install_gdrcopy.sh
487
+RUN set -eux; \
488
+ case "${TARGETPLATFORM}" in \
489
+ linux/arm64) UUARCH="aarch64" ;; \
490
+ linux/amd64) UUARCH="x64" ;; \
491
+ *) echo "Unsupported TARGETPLATFORM: ${TARGETPLATFORM}" >&2; exit 1 ;; \
492
+ esac; \
493
+ /tmp/install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "${GDRCOPY_CUDA_VERSION}" "${UUARCH}" && \
494
+ rm /tmp/install_gdrcopy.sh
495
+
496
+# Install vllm-openai dependencies (saves ~2.6s per build)
497
+# These are stable packages that don't depend on vLLM itself
498
+RUN --mount=type=cache,target=/root/.cache/uv \
499
+ if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
500
+ BITSANDBYTES_VERSION="0.42.0"; \
501
+ else \
502
+ BITSANDBYTES_VERSION="0.46.1"; \
503
+ fi; \
504
+ uv pip install --system accelerate hf_transfer modelscope \
505
+ "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.15.3'
506
+
507
+# ============================================================
508
+# VLLM INSTALLATION (depends on build stage)
509
+# ============================================================
510
+
511
+ARG PIP_INDEX_URL UV_INDEX_URL
512
+ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
513
+ARG PYTORCH_CUDA_INDEX_BASE_URL
514
+ARG PIP_KEYRING_PROVIDER UV_KEYRING_PROVIDER
515
+
516
+# Install vllm wheel first, so that torch etc will be installed.
517
+RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
518
+ --mount=type=cache,target=/root/.cache/uv \
519
+ uv pip install --system dist/*.whl --verbose \
520
+ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
521
522
RUN --mount=type=cache,target=/root/.cache/uv \
523
. /etc/environment && \
@@ -478,7 +532,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
532
echo "No DeepGEMM wheels to install; skipping."; \
533
fi'
534
481
-# Pytorch now installs NVSHMEM, setting LD_LIBRARY_PATH (https://github.com/pytorch/pytorch/blob/d38164a545b4a4e4e0cf73ce67173f70574890b6/.ci/manywheel/build_cuda.sh#L141C14-L141C36)
535
+# Pytorch now installs NVSHMEM, setting LD_LIBRARY_PATH
536
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
537
538
# Install EP kernels wheels (pplx-kernels and DeepEP) that have been built in the `build` stage
@@ -487,23 +541,17 @@ RUN --mount=type=bind,from=build,src=/tmp/ep_kernels_workspace/dist,target=/vllm
541
uv pip install --system ep_kernels/dist/*.whl --verbose \
542
--extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
543
490
-RUN --mount=type=bind,source=tools/install_gdrcopy.sh,target=/tmp/install_gdrcopy.sh,ro \
491
- set -eux; \
492
- case "${TARGETPLATFORM}" in \
493
- linux/arm64) UUARCH="aarch64" ;; \
494
- linux/amd64) UUARCH="x64" ;; \
495
- *) echo "Unsupported TARGETPLATFORM: ${TARGETPLATFORM}" >&2; exit 1 ;; \
496
- esac; \
497
- /tmp/install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "${GDRCOPY_CUDA_VERSION}" "${UUARCH}"
498
-
544
# CUDA image changed from /usr/local/nvidia to /usr/local/cuda in 12.8 but will
545
# return to /usr/local/nvidia in 13.0 to allow container providers to mount drivers
546
# consistently from the host (see https://github.com/vllm-project/vllm/issues/18859).
547
# Until then, add /usr/local/nvidia/lib64 before the image cuda path to allow override.
548
ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
549
550
+# Copy examples and benchmarks at the end to minimize cache invalidation
551
+COPY examples examples
552
+COPY benchmarks benchmarks
553
+COPY ./vllm/collect_env.py .
554
#################### vLLM installation IMAGE ####################
506
-
555
#################### TEST IMAGE ####################
556
# image to run unit testing suite
557
# note that this uses vllm installed by `pip`
@@ -569,18 +617,12 @@ ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
617
# Reference: https://github.com/astral-sh/uv/pull/1694
618
ENV UV_HTTP_TIMEOUT=500
619
572
-# install additional dependencies for openai api server
620
+# install kv_connectors if requested
621
RUN --mount=type=cache,target=/root/.cache/uv \
622
--mount=type=bind,source=requirements/kv_connectors.txt,target=/tmp/kv_connectors.txt,ro \
623
if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
624
uv pip install --system -r /tmp/kv_connectors.txt; \
577
- fi; \
578
- if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
579
- BITSANDBYTES_VERSION="0.42.0"; \
580
- else \
581
- BITSANDBYTES_VERSION="0.46.1"; \
582
- fi; \
583
- uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3,gcs]>=0.15.3'
625
+ fi
626
627
ENV VLLM_USAGE_SOURCE production-docker-image
628
docs/assets/contributing/dockerfile-stages-dependency.png
Binary files a/docs/assets/contributing/dockerfile-stages-dependency.png and b/docs/assets/contributing/dockerfile-stages-dependency.png differ