[CI] Add registry layer cache to x86 CPU image build (#51911)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Aug 12, 2026 at 17:58 UTC
6accb779a361c723cded3f9422b48d3fe4da0901
2 files changed
+172
-24
.buildkite/image_build/image_build_cpu.sh
+72
-5
@@ -11,24 +11,91 @@ REPO=$2
11
BUILDKITE_COMMIT=$3
12
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
13
14
-# authenticate with AWS ECR
14
+# replace invalid characters in Docker image tags and truncate to 128 chars
15
+clean_docker_tag() {
16
+ local input="$1"
17
+ echo "$input" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128
18
+}
19
+
20
+# resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN
21
+# Reuses the same ECR cache repos as the CUDA image build, with an
22
+# "x86_cpu" suffix on every tag so CPU and CUDA cache blobs stay isolated.
23
+prepare_cache_tags() {
24
+ TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache"
25
+ MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache"
26
+
27
+ if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
28
+ if [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
29
+ cache="${MAIN_CACHE_ECR}:latest-x86_cpu"
30
+ else
31
+ clean_branch=$(clean_docker_tag "${BUILDKITE_BRANCH:-unknown}")
32
+ cache="${TEST_CACHE_ECR}:${clean_branch}-x86_cpu"
33
+ fi
34
+ CACHE_TO="$cache"
35
+ CACHE_FROM="$cache"
36
+ CACHE_FROM_BASE_BRANCH="$cache"
37
+ else
38
+ CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu"
39
+ CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu"
40
+ if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" == "main" ]]; then
41
+ CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-x86_cpu"
42
+ else
43
+ clean_base=$(clean_docker_tag "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}")
44
+ CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-x86_cpu"
45
+ fi
46
+ fi
47
+
48
+ CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-x86_cpu"
49
+}
50
+
51
+# authenticate with AWS ECR (public, for the image; private, for the cache repos)
52
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
53
+aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
54
55
# skip build if image already exists
56
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
57
echo "Image found"
58
else
59
echo "Image not found, proceeding with build..."
22
- # build
23
- docker build --file docker/Dockerfile.cpu \
60
+
61
+ prepare_cache_tags
62
+ echo "--- :mag: Cache tags"
63
+ echo "CACHE_TO: ${CACHE_TO}"
64
+ echo "CACHE_FROM: ${CACHE_FROM}"
65
+ echo "CACHE_FROM_BASE_BRANCH: ${CACHE_FROM_BASE_BRANCH}"
66
+ echo "CACHE_FROM_MAIN: ${CACHE_FROM_MAIN}"
67
+
68
+ # dedupe cache-from refs so identical branch/main fallbacks aren't repeated
69
+ CACHE_FROM_ARGS=()
70
+ SEEN_REFS=""
71
+ for ref in "$CACHE_FROM" "$CACHE_FROM_BASE_BRANCH" "$CACHE_FROM_MAIN"; do
72
+ if [[ "$SEEN_REFS" != *"|${ref}|"* ]]; then
73
+ CACHE_FROM_ARGS+=(--cache-from "type=registry,ref=${ref}")
74
+ SEEN_REFS="${SEEN_REFS}|${ref}|"
75
+ fi
76
+ done
77
+
78
+ echo "--- :docker: Setting up buildx"
79
+ # network=host so the isolated docker-container builder can still reach the
80
+ # EC2 instance metadata service for sccache's S3 credentials; recreate on
81
+ # every run in case a stale builder without this driver-opt already exists
82
+ # on a warm CI host.
83
+ docker buildx rm vllm-cpu-builder >/dev/null 2>&1 || true
84
+ docker buildx create --name vllm-cpu-builder --driver docker-container --driver-opt network=host --use
85
+ docker buildx inspect --bootstrap
86
+
87
+ # build and push
88
+ docker buildx build --file docker/Dockerfile.cpu \
89
--build-arg max_jobs=16 \
90
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
91
--build-arg VLLM_CPU_X86=true \
92
+ --build-arg USE_SCCACHE=1 \
93
--tag "$IMAGE" \
94
--target vllm-test \
95
+ "${CACHE_FROM_ARGS[@]}" \
96
+ --cache-to "type=registry,ref=${CACHE_TO},mode=max" \
97
+ --push \
98
--progress plain .
30
- # push
31
- docker push "$IMAGE"
99
fi
100
101
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
docker/Dockerfile.cpu
+100
-19
@@ -19,8 +19,42 @@
19
# VLLM_CPU_ARM_BF16=false (default)|true (for cross-compilation)
20
#
21
22
+######################### BASE IMAGE #########################
23
+# Common apt packages and the optional sccache binary install, shared by
24
+# base-common and rust-build (the latter is deliberately not `FROM
25
+# base-common`, to stay minimal and build in parallel with vllm-build).
26
+#
27
+# Optional remote (S3-backed) compiler cache. Local BuildKit `--mount=type=cache`
28
+# cache mounts (ccache, cargo registry) aren't exported by `--cache-to
29
+# type=registry`, so they don't survive a build landing on a different/fresh
30
+# builder. sccache's cache lives in S3 instead, so it does.
31
+FROM ubuntu:22.04 AS base
32
+
33
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
34
+ --mount=type=cache,target=/var/lib/apt,sharing=locked \
35
+ apt-get update -y \
36
+ && apt-get install -y --no-install-recommends ca-certificates curl git
37
+
38
+ARG TARGETARCH
39
+ARG USE_SCCACHE
40
+ARG SCCACHE_DOWNLOAD_URL
41
+
42
+RUN if [ "$USE_SCCACHE" = "1" ]; then \
43
+ echo "Installing sccache..." \
44
+ && case "${TARGETARCH}" in \
45
+ arm64) SCCACHE_ARCH="aarch64" ;; \
46
+ amd64) SCCACHE_ARCH="x86_64" ;; \
47
+ *) echo "Unsupported TARGETARCH for sccache: ${TARGETARCH}" >&2; exit 1 ;; \
48
+ esac \
49
+ && export SCCACHE_DOWNLOAD_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \
50
+ && curl -L -o sccache.tar.gz "${SCCACHE_DOWNLOAD_URL}" \
51
+ && tar -xzf sccache.tar.gz \
52
+ && mv sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \
53
+ && rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \
54
+ fi
55
+
56
######################### COMMON BASE IMAGE #########################
23
-FROM ubuntu:22.04 AS base-common
57
+FROM base AS base-common
58
59
WORKDIR /workspace
60
@@ -33,11 +67,22 @@ ENV MAX_JOBS=${max_jobs}
67
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
68
--mount=type=cache,target=/var/lib/apt,sharing=locked \
69
apt-get update -y \
36
- && apt-get install -y --no-install-recommends sudo ccache git curl wget ca-certificates zlib1g-dev \
70
+ && apt-get install -y --no-install-recommends sudo ccache wget zlib1g-dev \
71
gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 jq lsof make xz-utils \
72
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \
73
&& curl -LsSf https://astral.sh/uv/install.sh | sh
74
75
+ARG TARGETARCH
76
+ARG USE_SCCACHE
77
+ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
78
+ARG SCCACHE_REGION_NAME=us-west-2
79
+ARG SCCACHE_S3_NO_CREDENTIALS=0
80
+
81
+ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
82
+ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
83
+ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
84
+ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
85
+
86
# Compiler and linker environment
87
ENV CC=/usr/bin/gcc-12 CXX=/usr/bin/g++-12
88
ENV CCACHE_DIR=/root/.cache/ccache
@@ -63,7 +108,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
108
uv pip install --upgrade pip && \
109
uv pip install -r requirements/cpu.txt --torch-backend cpu
110
66
-ARG TARGETARCH
111
ENV TARGETARCH=${TARGETARCH}
112
113
######################### x86_64 BASE IMAGE #########################
@@ -76,8 +120,8 @@ FROM base-common AS base-arm64
120
121
ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"
122
79
-######################### BASE IMAGE #########################
80
-FROM base-${TARGETARCH} AS base
123
+######################### ARCH BASE IMAGE #########################
124
+FROM base-${TARGETARCH} AS base-arch
125
126
RUN echo 'ulimit -c 0' >> ~/.bashrc
127
@@ -85,12 +129,12 @@ RUN echo 'ulimit -c 0' >> ~/.bashrc
129
# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build
130
# stage doesn't need the rust toolchain or protoc. This stage runs in parallel
131
# with the main vllm-build stage.
88
-FROM ubuntu:22.04 AS rust-build
132
+FROM base AS rust-build
133
ENV DEBIAN_FRONTEND=noninteractive
134
135
RUN apt-get update -y \
136
&& apt-get install -y --no-install-recommends \
93
- ca-certificates curl git build-essential unzip python3 python3-pip \
137
+ build-essential unzip python3 python3-pip \
138
&& rm -rf /var/lib/apt/lists/*
139
140
COPY tools/install_protoc.sh /tmp/install_protoc.sh
@@ -101,6 +145,16 @@ WORKDIR /workspace
145
COPY requirements/build/rust.txt requirements/build/rust.txt
146
RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt
147
148
+ARG USE_SCCACHE
149
+ARG SCCACHE_ENDPOINT
150
+ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
151
+ARG SCCACHE_REGION_NAME=us-west-2
152
+
153
+ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
154
+ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
155
+# Avoid port collision with vllm-build's own sccache daemon.
156
+ENV SCCACHE_SERVER_PORT=4227
157
+
158
# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed
159
# by the wheel build stage.
160
COPY rust rust
@@ -117,10 +171,20 @@ ENV CARGO_BUILD_JOBS=4
171
# cache reuse.
172
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
173
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
120
- bash build_rust.sh
174
+ --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
175
+ if [ "$USE_SCCACHE" = "1" ]; then \
176
+ export RUSTC_WRAPPER=sccache; \
177
+ if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
178
+ sccache --show-stats; \
179
+ fi && \
180
+ bash build_rust.sh && \
181
+ if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
182
183
######################### BUILD IMAGE #########################
123
-FROM base AS vllm-build
184
+FROM base-arch AS vllm-build
185
+
186
+ARG USE_SCCACHE
187
+ARG SCCACHE_ENDPOINT
188
189
ARG GIT_REPO_CHECK=0
190
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
@@ -160,17 +224,27 @@ RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
224
RUN --mount=type=cache,target=/root/.cache/uv \
225
--mount=type=cache,target=/root/.cache/ccache \
226
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
163
- VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
227
+ --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
228
+ if [ "$USE_SCCACHE" = "1" ]; then \
229
+ if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
230
+ sccache --show-stats; \
231
+ fi && \
232
+ VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 && \
233
+ if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
234
235
######################### TRITON-CPU BUILD IMAGE #########################
166
-FROM base AS vllm-triton-cpu-build
236
+FROM base-arch AS vllm-triton-cpu-build
237
238
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
169
-# Re-declared here because this stage is `FROM base` (not `vllm-build`), so it
170
-# does not inherit the ARG/ENV defined there. Without it, the guard below would
171
-# see an empty value and build triton-cpu on non-x86 targets (e.g. arm64).
239
+# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`),
240
+# so it does not inherit the ARG/ENV defined there. Without it, the guard
241
+# below would see an empty value and build triton-cpu on non-x86 targets
242
+# (e.g. arm64).
243
ARG VLLM_CPU_X86=0
244
245
+ARG USE_SCCACHE
246
+ARG SCCACHE_ENDPOINT
247
+
248
WORKDIR /vllm-workspace
249
250
RUN mkdir dist
@@ -178,15 +252,22 @@ RUN mkdir dist
252
RUN --mount=type=cache,target=/root/.cache/uv \
253
--mount=type=cache,target=/root/.cache/ccache \
254
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
255
+ --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
256
if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \
257
git clone --recurse-submodules "https://github.com/triton-lang/triton-cpu.git"; \
258
cd triton-cpu; \
259
git checkout "270e696d"; \
260
+ if [ "$USE_SCCACHE" = "1" ]; then \
261
+ if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
262
+ export TRITON_APPEND_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"; \
263
+ sccache --show-stats; \
264
+ fi; \
265
uv build --wheel --out-dir=../dist; \
266
+ if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi; \
267
fi
268
269
######################### TEST DEPS #########################
189
-FROM base AS vllm-test-deps
270
+FROM base-arch AS vllm-test-deps
271
272
WORKDIR /vllm-workspace
273
@@ -260,11 +341,11 @@ ENV HF_XET_HIGH_PERFORMANCE 1
341
ENV HF_HUB_DOWNLOAD_TIMEOUT 60
342
343
######################### RELEASE IMAGE #########################
263
-FROM base AS vllm-openai
344
+FROM base-arch AS vllm-openai
345
265
-# Re-declared here because this stage is `FROM base` (not `vllm-build`), so the
266
-# RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86 would
267
-# otherwise see an empty value and try to install it on non-x86 targets.
346
+# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`),
347
+# so the RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86
348
+# would otherwise see an empty value and try to install it on non-x86 targets.
349
ARG VLLM_CPU_X86=0
350
351
WORKDIR /vllm-workspace