[CI/Build][CPU] Fix flaky rust downloads, broken prune flag, and triton-cpu cache coupling (#56247)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Sep 10, 2026 at 22:27 UTC
46bae7e98c353bc64dd762f3ebac70165f1027d9
3 files changed
+111
-68
.buildkite/scripts/hardware_ci/run-cpu-test.sh
+32
-16
@@ -13,17 +13,16 @@ TIMEOUT_VAL=$1
13
TEST_COMMAND=$2
14
15
# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
16
-# DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
17
-# BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
16
+# DISK_USAGE_THRESHOLD percent, and only prune images/cache unused for at least
17
+# CACHE_MAX_AGE so recently-built layers survive for reuse.
18
DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-80}
19
-BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-200GB}
19
+CACHE_MAX_AGE=${CACHE_MAX_AGE:-24h}
20
21
-# Reclaim disk only when the host is under pressure. We trim (not purge) the
22
-# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
23
-# touch dangling images; other agents' uniquely tagged images are left alone.
24
-# Cache mounts (exec.cachemount: uv/cargo/apt) are excluded so they survive
25
-# pruning -- they're what let installs reuse packages instead of hitting the
26
-# network, and are otherwise reclaimable like any other build cache record.
21
+# Reclaim disk only when the host is under pressure, aging out anything unused
22
+# for less than CACHE_MAX_AGE so hot layers survive -- same `--filter
23
+# until=<N>h` pattern the TPU CI scripts already rely on. `docker buildx
24
+# prune --max-used-space` is a no-op on this host's `docker` driver (BuildKit
25
+# embedded in dockerd never enforces the size cap), so we don't use it.
26
prune_if_disk_pressure() {
27
local docker_root disk_usage
28
docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
@@ -33,8 +32,7 @@ prune_if_disk_pressure() {
32
disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
33
if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
34
echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
36
- docker image prune -f || true
37
- docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" --filter type!=exec.cachemount || true
35
+ docker system prune --force --all --filter "until=${CACHE_MAX_AGE}" || true
36
else
37
echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
38
fi
@@ -61,18 +59,18 @@ docker pull ubuntu:25.04 || true
59
60
# building the docker image
61
echo "--- :docker: Building Docker image"
64
-BUILD_RETRY_PATTERN='dial tcp|i/o timeout|failed to authorize|TLS handshake timeout|connection reset|Could not resolve host|Temporary failure in name resolution|dns error: failed to lookup address information|client error \(Connect\)|error sending request for url|Failed to fetch:'
62
+BUILD_RETRY_PATTERN='dial tcp|i/o timeout|failed to authorize|TLS handshake timeout|connection reset|PROTOCOL_ERROR|Could not resolve host|Temporary failure in name resolution|dns error: failed to lookup address information|client error \(Connect\)|error sending request for url|Failed to fetch:'
63
BUILD_MAX_ATTEMPTS=4 # 1 initial + 3 retries
64
BUILD_RETRY_WAITS=(10 20 40) # seconds to wait before retry 1/2/3
65
build_log="$(mktemp)"
66
attempt=1
67
while true; do
68
if docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test \
71
- --build-arg USE_SCCACHE=1 --build-arg SCCACHE_LOCAL_ONLY=1 \
69
+ --build-arg USE_SCCACHE=1 --build-arg SCCACHE_LOCAL_ONLY=1 --build-arg max_jobs=16 \
70
-f docker/Dockerfile.cpu . 2>&1 | tee "$build_log"; then
71
break
72
fi
75
- if [ "$attempt" -ge "$BUILD_MAX_ATTEMPTS" ] || ! grep -qE "$BUILD_RETRY_PATTERN" "$build_log"; then
73
+ if [ "$attempt" -ge "$BUILD_MAX_ATTEMPTS" ] || ! grep -qiE "$BUILD_RETRY_PATTERN" "$build_log"; then
74
rm -f "$build_log"
75
exit 1
76
fi
@@ -83,6 +81,24 @@ while true; do
81
done
82
rm -f "$build_log"
83
86
-# Run the image, setting --shm-size=4g for tensor parallel.
87
-docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 --shm-size=4g "$IMAGE_NAME" \
84
+# Run the image, setting --shm-size=4g for tensor parallel. Default to
85
+# HF_HUB_OFFLINE so a warm ~/.cache/huggingface doesn't hit the network;
86
+# retry once online if the cache is missing something.
87
+OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)'
88
+run_test() {
89
+ local hf_offline=$1
90
+ docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
91
timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
92
+}
93
+
94
+test_log="$(mktemp)"
95
+if run_test 1 2>&1 | tee "$test_log"; then
96
+ rm -f "$test_log"
97
+elif grep -qE "$OFFLINE_RETRY_PATTERN" "$test_log"; then
98
+ rm -f "$test_log"
99
+ echo "--- :warning: HF_HUB_OFFLINE caused a cache miss, retrying with online fallback"
100
+ run_test 0
101
+else
102
+ rm -f "$test_log"
103
+ exit 1
104
+fi
docker/Dockerfile.cpu
+60
-49
@@ -105,6 +105,60 @@ ENV UV_HTTP_TIMEOUT=500
105
ENV UV_INDEX_STRATEGY="unsafe-best-match"
106
ENV UV_LINK_MODE="copy"
107
108
+ENV TARGETARCH=${TARGETARCH}
109
+
110
+######################### TRITON-CPU BUILD IMAGE #########################
111
+# Branches off base-common directly (before requirements/cpu.txt is installed
112
+# below) so a torch/etc. version bump doesn't invalidate this stage's cache --
113
+# it builds triton-cpu in an isolated `uv build` environment and never touches
114
+# these packages.
115
+FROM base-common AS vllm-triton-cpu-build
116
+
117
+# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
118
+# Re-declared here because this stage is `FROM base-common` (not `vllm-build`),
119
+# so it does not inherit the ARG/ENV defined there. Without it, the guard
120
+# below would see an empty value and build triton-cpu on non-x86 targets
121
+# (e.g. arm64).
122
+ARG VLLM_CPU_X86=0
123
+
124
+ARG USE_SCCACHE
125
+ARG SCCACHE_ENDPOINT
126
+ARG SCCACHE_LOCAL_ONLY=0
127
+
128
+WORKDIR /vllm-workspace
129
+
130
+RUN mkdir dist
131
+
132
+RUN --mount=type=cache,target=/root/.cache/uv \
133
+ --mount=type=cache,target=/root/.cache/ccache \
134
+ --mount=type=cache,target=/root/.cache/sccache-local,sharing=locked \
135
+ --mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
136
+ --mount=type=cache,target=/root/.triton \
137
+ --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
138
+ set -ex; \
139
+ if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \
140
+ git clone --recurse-submodules --shallow-submodules --filter=blob:none "https://github.com/triton-lang/triton-cpu.git"; \
141
+ cd triton-cpu; \
142
+ git checkout "270e696d"; \
143
+ if [ "$USE_SCCACHE" = "1" ]; then \
144
+ if [ "$SCCACHE_LOCAL_ONLY" = "1" ]; then \
145
+ unset SCCACHE_BUCKET SCCACHE_REGION; \
146
+ export SCCACHE_DIR=/root/.cache/sccache-local SCCACHE_CACHE_SIZE=20G; \
147
+ elif [ -n "${SCCACHE_ENDPOINT}" ]; then \
148
+ export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; \
149
+ fi; \
150
+ export TRITON_APPEND_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"; \
151
+ sccache --show-stats; \
152
+ fi; \
153
+ uv build --wheel --out-dir=../dist; \
154
+ if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi; \
155
+ cd ..; \
156
+ rm -rf triton-cpu; \
157
+ fi
158
+
159
+######################### COMMON BASE IMAGE + PYTHON DEPS #########################
160
+FROM base-common AS base-common-pydeps
161
+
162
# PyTorch provides its own indexes for standard and nightly builds.
163
# PYTORCH_NIGHTLY=1 builds the CPU image against torch nightly (parity with the
164
# GPU torch-nightly lane); default 0 keeps the pinned release CPU build.
@@ -133,15 +187,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \
187
uv pip install -r requirements/cpu.txt --torch-backend cpu; \
188
fi
189
136
-ENV TARGETARCH=${TARGETARCH}
137
-
190
######################### x86_64 BASE IMAGE #########################
139
-FROM base-common AS base-amd64
191
+FROM base-common-pydeps AS base-amd64
192
193
ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/opt/venv/lib/libiomp5.so"
194
195
######################### arm64 BASE IMAGE #########################
144
-FROM base-common AS base-arm64
196
+FROM base-common-pydeps AS base-arm64
197
198
ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"
199
@@ -210,6 +262,10 @@ RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
262
SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+docker.cache" bash build_rust.sh && \
263
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
264
265
+# build_rust.sh installed rustup here via rustup.rs; put it on PATH so the
266
+# child stage below finds it on disk instead of re-downloading it.
267
+ENV PATH="/root/.cargo/bin:${PATH}"
268
+
269
# Relink with the exact Git-derived package version.
270
FROM rust-build-cache AS rust-build
271
@@ -308,51 +364,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
364
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 && \
365
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
366
311
-######################### TRITON-CPU BUILD IMAGE #########################
312
-FROM base-arch AS vllm-triton-cpu-build
313
-
314
-# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
315
-# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`),
316
-# so it does not inherit the ARG/ENV defined there. Without it, the guard
317
-# below would see an empty value and build triton-cpu on non-x86 targets
318
-# (e.g. arm64).
319
-ARG VLLM_CPU_X86=0
320
-
321
-ARG USE_SCCACHE
322
-ARG SCCACHE_ENDPOINT
323
-ARG SCCACHE_LOCAL_ONLY=0
324
-
325
-WORKDIR /vllm-workspace
326
-
327
-RUN mkdir dist
328
-
329
-RUN --mount=type=cache,target=/root/.cache/uv \
330
- --mount=type=cache,target=/root/.cache/ccache \
331
- --mount=type=cache,target=/root/.cache/sccache-local,sharing=locked \
332
- --mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
333
- --mount=type=cache,target=/root/.triton \
334
- --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
335
- set -ex; \
336
- if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \
337
- git clone --recurse-submodules --shallow-submodules --filter=blob:none "https://github.com/triton-lang/triton-cpu.git"; \
338
- cd triton-cpu; \
339
- git checkout "270e696d"; \
340
- if [ "$USE_SCCACHE" = "1" ]; then \
341
- if [ "$SCCACHE_LOCAL_ONLY" = "1" ]; then \
342
- unset SCCACHE_BUCKET SCCACHE_REGION; \
343
- export SCCACHE_DIR=/root/.cache/sccache-local SCCACHE_CACHE_SIZE=20G; \
344
- elif [ -n "${SCCACHE_ENDPOINT}" ]; then \
345
- export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; \
346
- fi; \
347
- export TRITON_APPEND_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"; \
348
- sccache --show-stats; \
349
- fi; \
350
- uv build --wheel --out-dir=../dist; \
351
- if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi; \
352
- cd ..; \
353
- rm -rf triton-cpu; \
354
- fi
355
-
367
######################### TEST DEPS #########################
368
FROM base-arch AS vllm-test-deps
369
vllm/v1/worker/cpu_worker.py
+19
-3
@@ -12,7 +12,8 @@ from typing import Any
12
import psutil
13
import torch
14
15
-from vllm.config import VllmConfig
15
+from vllm import envs
16
+from vllm.config import CompilationMode, VllmConfig
17
from vllm.logger import init_logger
18
from vllm.platforms import CpuArchEnum, current_platform
19
from vllm.profiler.wrapper import TorchProfilerWrapper
@@ -197,8 +198,23 @@ class CPUWorker(Worker):
198
logger.warning("sleep mode is not supported on CPU, ignore it.")
199
pass
200
201
+ def _should_warm_up_model(self) -> bool:
202
+ # VLLM_CPU_CI_ENV always skips warmup to save CI time.
203
+ if envs.VLLM_CPU_CI_ENV:
204
+ return False
205
+ # With eager (CompilationMode.NONE) execution and an explicit KV
206
+ # cache size, warmup serves no purpose: there's no compiled graph to
207
+ # prewarm, and the auto KV-cache-size calc below -- which relies on
208
+ # a warmup forward pass having already bumped RSS to a realistic
209
+ # steady-state value -- is bypassed whenever the size is explicit.
210
+ return not (
211
+ self.compilation_config.mode == CompilationMode.NONE
212
+ and self.cache_config.kv_cache_memory_bytes is not None
213
+ )
214
+
215
def determine_available_memory(self) -> int:
201
- self.model_runner.warming_up_model()
216
+ if self._should_warm_up_model():
217
+ self.model_runner.warming_up_model()
218
219
allowed_cpu_list = get_allowed_cpu_list()
220
cpu_core = allowed_cpu_list[0]
@@ -259,7 +275,7 @@ class CPUWorker(Worker):
275
def compile_or_warm_up_model(self) -> CompilationTimes:
276
# Note: the model has been compiled in determine_available_memory(),
277
# Only compile here for models without kv cache
262
- if len(self.model_runner.kv_caches) == 0:
278
+ if len(self.model_runner.kv_caches) == 0 and self._should_warm_up_model():
279
self.model_runner.warming_up_model()
280
# Reset the seed to ensure that the random state is not affected by
281
# the model initialization and profiling.