A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 16:15 UTC
[XPU][CI] Fix Docker cleanup races on Intel CI runners (#40761)
Signed-off-by: zengxian <xiangdong.zeng@intel.com>
xiangdong committed
Apr 24, 2026 at 14:08 UTC
01acf96c6f57914479e6bfe79d7bd5777a2fc49f
2 files changed
+85
-11
.buildkite/scripts/hardware_ci/run-intel-test.sh
+84
-8
@@ -25,22 +25,100 @@ export PYTHONPATH=".."
25
###############################################################################
26
27
cleanup_docker() {
28
+ # Share the same lock with image pull to avoid cleanup/pull races on one node.
29
+ local docker_lock="/tmp/docker-pull.lock"
30
+ exec 9>"$docker_lock"
31
+ flock 9
32
+
33
docker_root=$(docker info -f '{{.DockerRootDir}}')
34
if [ -z "$docker_root" ]; then
35
echo "Failed to determine Docker root directory." >&2
31
- exit 1
36
+ flock -u 9
37
+ return 1
38
fi
39
echo "Docker root directory: $docker_root"
40
41
disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
42
threshold=70
43
if [ "$disk_usage" -gt "$threshold" ]; then
38
- echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
39
- docker image prune -f
40
- docker volume prune -f && docker system prune --force --filter "until=72h" --all
41
- echo "Docker images and volumes cleanup completed."
44
+ echo "Disk usage is above $threshold%. Running aggressive CI image cleanup..."
45
+ cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 1
46
+ else
47
+ echo "Disk usage is below $threshold%. Checking old CI images anyway."
48
+ cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 0
49
+ fi
50
+ echo "Old CI image cleanup completed."
51
+
52
+ flock -u 9
53
+}
54
+
55
+cleanup_old_ci_images() {
56
+ local repo_prefix="$1"
57
+ local current_image_ref="$2"
58
+ local ttl_hours="$3"
59
+ local aggressive_cleanup="$4"
60
+
61
+ if [[ -z "$repo_prefix" || "$repo_prefix" == "/" ]]; then
62
+ echo "Skip old-image cleanup: invalid repo prefix '${repo_prefix}'"
63
+ return 0
64
+ fi
65
+
66
+ if ! [[ "$ttl_hours" =~ ^[0-9]+$ ]]; then
67
+ echo "Invalid DOCKER_IMAGE_CLEANUP_HOURS='${ttl_hours}', fallback to 72"
68
+ ttl_hours=72
69
+ fi
70
+
71
+ local now_epoch cutoff_epoch
72
+ now_epoch=$(date +%s)
73
+ cutoff_epoch=$((now_epoch - ttl_hours * 3600))
74
+
75
+ local -a used_image_ids
76
+ mapfile -t used_image_ids < <(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' | sort -u)
77
+
78
+ local removed_count=0
79
+ local examined_count=0
80
+ declare -A seen_ids=()
81
+
82
+ while read -r image_ref image_id; do
83
+ [[ -z "$image_ref" || -z "$image_id" ]] && continue
84
+ ((examined_count++))
85
+
86
+ # Keep the image this job is going to use.
87
+ if [[ "$image_ref" == "$current_image_ref" ]]; then
88
+ continue
89
+ fi
90
+
91
+ # Avoid duplicate deletes when multiple tags point to same image id.
92
+ if [[ -n "${seen_ids[$image_id]:-}" ]]; then
93
+ continue
94
+ fi
95
+ seen_ids[$image_id]=1
96
+
97
+ # Never delete images that are used by any container on this node.
98
+ if printf '%s\n' "${used_image_ids[@]}" | grep -qx "$image_id"; then
99
+ continue
100
+ fi
101
+
102
+ local created created_epoch
103
+ created=$(docker image inspect -f '{{.Created}}' "$image_id" 2>/dev/null || true)
104
+ [[ -z "$created" ]] && continue
105
+ created_epoch=$(date -d "$created" +%s 2>/dev/null || true)
106
+ [[ -z "$created_epoch" ]] && continue
107
+
108
+ if (( created_epoch < cutoff_epoch )) || [[ "$aggressive_cleanup" == "1" ]]; then
109
+ if docker image rm -f "$image_id" >/dev/null 2>&1; then
110
+ ((removed_count++))
111
+ fi
112
+ fi
113
+ done < <(docker image ls --no-trunc "$repo_prefix" --format '{{.Repository}}:{{.Tag}} {{.ID}}')
114
+
115
+ # Also trim old dangling layers; this is safe and does not remove referenced images.
116
+ docker image prune -f --filter "until=${ttl_hours}h" >/dev/null 2>&1 || true
117
+
118
+ if [[ "$aggressive_cleanup" == "1" ]]; then
119
+ echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} unused images under disk pressure."
120
else
43
- echo "Disk usage is below $threshold%. No cleanup needed."
121
+ echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} old images (>${ttl_hours}h)."
122
fi
123
}
124
@@ -265,8 +343,6 @@ fi
343
344
remove_docker_container() {
345
docker rm -f "${container_name}" || true
268
- docker image rm -f "${image_name}" || true
269
- docker system prune -f || true
346
}
347
trap remove_docker_container EXIT
348
.buildkite/scripts/hardware_ci/run-xpu-test.sh
+1
-3
@@ -12,9 +12,7 @@ docker build -t "${image_name}" -f docker/Dockerfile.xpu .
12
13
# Setup cleanup
14
remove_docker_container() {
15
- docker rm -f "${container_name}" || true;
16
- docker image rm -f "${image_name}" || true;
17
- docker system prune -f || true;
15
+ docker rm -f "${container_name}" || true
16
}
17
trap remove_docker_container EXIT
18