[CI] Enable sccache for Rust build under CUDA/ROCm (#45246)
Bugen Zhao committed
Jul 3, 2026 at 14:45 UTC
b790c84cde050c39f37a0a1f967e877d4c2fb4c9
3 files changed
+67
-33
docker/Dockerfile
+45
-23
@@ -99,9 +99,16 @@ ARG INSTALL_KV_CONNECTORS=false
99
# prepare basic build environment
100
FROM ${BUILD_BASE_IMAGE} AS base
101
102
+ARG TARGETPLATFORM
103
ARG CUDA_VERSION
104
ARG PYTHON_VERSION
105
ARG BUILD_OS
106
+ARG USE_SCCACHE
107
+ARG SCCACHE_DOWNLOAD_URL
108
+ARG SCCACHE_ENDPOINT
109
+ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
110
+ARG SCCACHE_REGION_NAME=us-west-2
111
+ARG SCCACHE_S3_NO_CREDENTIALS=0
112
113
ENV DEBIAN_FRONTEND=noninteractive
114
@@ -162,6 +169,27 @@ RUN if [ "${BUILD_OS}" = "manylinux" ]; then \
169
&& rm -rf /var/lib/apt/lists/*; \
170
fi
171
172
+# Install sccache once in base so Rust and CMake/CUDA build stages share the
173
+# same binary and remote cache configuration.
174
+RUN if [ "$USE_SCCACHE" = "1" ]; then \
175
+ echo "Installing sccache..." \
176
+ && case "${TARGETPLATFORM}" in \
177
+ linux/arm64) SCCACHE_ARCH="aarch64" ;; \
178
+ linux/amd64) SCCACHE_ARCH="x86_64" ;; \
179
+ *) echo "Unsupported TARGETPLATFORM for sccache: ${TARGETPLATFORM}" >&2; exit 1 ;; \
180
+ esac \
181
+ && export SCCACHE_DOWNLOAD_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \
182
+ && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \
183
+ && tar -xzf sccache.tar.gz \
184
+ && sudo mv sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \
185
+ && rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \
186
+ fi
187
+
188
+ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
189
+ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
190
+ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
191
+ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
192
+
193
# Install uv and bootstrap /opt/venv. Both paths converge on /opt/venv so all
194
# downstream stages stay distro-agnostic.
195
RUN mkdir -p "${UV_PYTHON_INSTALL_DIR}" "${UV_CACHE_DIR}" "${UV_INSTALL_DIR}" \
@@ -275,6 +303,8 @@ ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
303
# csrc-build/extensions-build.
304
FROM base AS rust-build
305
ARG BUILD_OS
306
+ARG USE_SCCACHE
307
+ARG SCCACHE_ENDPOINT
308
309
# Install native tools needed only for Rust/protoc builds.
310
RUN if [ "${BUILD_OS}" = "manylinux" ]; then \
@@ -308,18 +338,29 @@ COPY build_rust.sh build_rust.sh
338
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
339
ENV CARGO_BUILD_JOBS=4
340
341
+# BuildKit can run this stage in parallel with csrc-build. Keep Rust on a
342
+# separate local sccache daemon while sharing the same remote cache backend.
343
+ENV SCCACHE_SERVER_PORT=4227
344
+
345
# Build the release artifacts. Cache cargo registry/git, but not target/,
346
# because stale target metadata can outlive source updates across BuildKit
347
# cache reuse.
348
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
349
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
316
- bash build_rust.sh
317
-
350
+ --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
351
+ if [ "$USE_SCCACHE" = "1" ]; then \
352
+ if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
353
+ export RUSTC_WRAPPER=sccache; \
354
+ sccache --show-stats; \
355
+ fi \
356
+ && bash build_rust.sh \
357
+ && if [ "$USE_SCCACHE" = "1" ]; then \
358
+ sccache --show-stats; \
359
+ fi
360
#################### RUST BUILD IMAGE ####################
361
362
#################### CSRC BUILD IMAGE ####################
363
FROM base AS csrc-build
322
-ARG TARGETPLATFORM
364
365
ARG PIP_INDEX_URL UV_INDEX_URL
366
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
@@ -371,11 +412,7 @@ ARG nvcc_threads=8
412
ENV NVCC_THREADS=$nvcc_threads
413
414
ARG USE_SCCACHE
374
-ARG SCCACHE_DOWNLOAD_URL
415
ARG SCCACHE_ENDPOINT
376
-ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
377
-ARG SCCACHE_REGION_NAME=us-west-2
378
-ARG SCCACHE_S3_NO_CREDENTIALS=0
416
417
# Flag to control whether to use pre-built vLLM wheels
418
ARG VLLM_USE_PRECOMPILED=""
@@ -405,22 +442,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
442
RUN --mount=type=cache,target=/opt/uv/cache \
443
--mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
444
if [ "$USE_SCCACHE" = "1" ]; then \
408
- echo "Installing sccache..." \
409
- && case "${TARGETPLATFORM}" in \
410
- linux/arm64) SCCACHE_ARCH="aarch64" ;; \
411
- linux/amd64) SCCACHE_ARCH="x86_64" ;; \
412
- *) echo "Unsupported TARGETPLATFORM for sccache: ${TARGETPLATFORM}" >&2; exit 1 ;; \
413
- esac \
414
- && export SCCACHE_DOWNLOAD_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \
415
- && curl -L -o sccache.tar.gz ${SCCACHE_DOWNLOAD_URL} \
416
- && tar -xzf sccache.tar.gz \
417
- && sudo mv sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \
418
- && rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl \
419
- && if [ ! -z ${SCCACHE_ENDPOINT} ] ; then export SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} ; fi \
420
- && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \
421
- && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \
422
- && export SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS} \
423
- && export SCCACHE_IDLE_TIMEOUT=0 \
445
+ if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi \
446
&& export CMAKE_BUILD_TYPE=Release \
447
&& export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \
448
&& export VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_MERGE_BASE_COMMIT}" \
docker/Dockerfile.rocm
+13
-1
@@ -129,6 +129,7 @@ FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
129
# don't need the rust toolchain or protoc.
130
FROM fetch_vllm AS rust-build
131
ARG COMMON_WORKDIR
132
+ARG USE_SCCACHE
133
134
# protoc is used by tonic-build/prost-build.
135
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
@@ -144,6 +145,10 @@ ENV CARGO_BUILD_JOBS=4
145
ENV CARGO_NET_RETRY=10
146
ENV RUSTUP_MAX_RETRIES=10
147
148
+# BuildKit can run this stage in parallel with ROCm native builds. Keep Rust on
149
+# a separate local sccache daemon while sharing the same remote cache backend.
150
+ENV SCCACHE_SERVER_PORT=4227
151
+
152
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
153
cd ${COMMON_WORKDIR}/vllm \
154
&& uv pip install --system -r requirements/build/rust.txt
@@ -155,8 +160,15 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
160
RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \
161
--mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
162
cd ${COMMON_WORKDIR}/vllm \
163
+ && if [ "$USE_SCCACHE" = "1" ]; then \
164
+ export RUSTC_WRAPPER=sccache \
165
+ && sccache --show-stats; \
166
+ fi \
167
&& bash build_rust.sh \
159
- && test -x vllm/vllm-rs
168
+ && test -x vllm/vllm-rs \
169
+ && if [ "$USE_SCCACHE" = "1" ]; then \
170
+ sccache --show-stats; \
171
+ fi
172
173
# -----------------------
174
# vLLM native build stages
docker/versions.json
+9
-9
@@ -34,15 +34,6 @@
34
"INSTALL_KV_CONNECTORS": {
35
"default": "false"
36
},
37
- "TORCH_CUDA_ARCH_LIST": {
38
- "default": "7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0"
39
- },
40
- "MAX_JOBS": {
41
- "default": "2"
42
- },
43
- "NVCC_THREADS": {
44
- "default": "8"
45
- },
37
"SCCACHE_BUCKET_NAME": {
38
"default": "vllm-build-sccache"
39
},
@@ -52,6 +43,15 @@
43
"SCCACHE_S3_NO_CREDENTIALS": {
44
"default": "0"
45
},
46
+ "TORCH_CUDA_ARCH_LIST": {
47
+ "default": "7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0"
48
+ },
49
+ "MAX_JOBS": {
50
+ "default": "2"
51
+ },
52
+ "NVCC_THREADS": {
53
+ "default": "8"
54
+ },
55
"vllm_target_device": {
56
"default": "cuda"
57
},