[CI/Build][CPU] Add test image cache clean-up (#46831)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Jun 26, 2026 at 23:28 UTC
abc71548ef029132c3316b902207f254a246d593
1 file changed
+38
.buildkite/scripts/hardware_ci/run-cpu-test.sh
+38
@@ -12,6 +12,44 @@ IMAGE_NAME="cpu-test-${NUMA_NODE}${AGENT_SLOT:+-${AGENT_SLOT}}"
12
TIMEOUT_VAL=$1
13
TEST_COMMAND=$2
14
15
+# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
16
+# DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
17
+# BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
18
+DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-70}
19
+BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-80GB}
20
+
21
+# Reclaim disk only when the host is under pressure. We trim (not purge) the
22
+# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
23
+# touch dangling images; other agents' uniquely tagged images are left alone.
24
+prune_if_disk_pressure() {
25
+ local docker_root disk_usage
26
+ docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
27
+ if [ -z "$docker_root" ]; then
28
+ return 0
29
+ fi
30
+ disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
31
+ if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
32
+ echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
33
+ docker image prune -f || true
34
+ docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" || true
35
+ else
36
+ echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
37
+ fi
38
+}
39
+
40
+# Always drop this agent's image once the job ends (the default builder never
41
+# uses it as a cache source, so removing it costs no rebuild speed), then
42
+# reclaim space if needed. Guard every docker call with `|| true` so the trap
43
+# never overrides the test's exit code.
44
+cleanup() {
45
+ docker image rm -f "$IMAGE_NAME" || true
46
+ prune_if_disk_pressure
47
+}
48
+trap cleanup EXIT
49
+
50
+# Free space up front so a nearly-full host doesn't fail the build.
51
+prune_if_disk_pressure
52
+
53
# building the docker image
54
echo "--- :docker: Building Docker image"
55
docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .