A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[TPU][CI/CD] Clean up docker for TPU tests. (#18926)
Signed-off-by: Carol Zheng <cazheng@google.com>
Carol Zheng committed
May 30, 2025 at 02:24 UTC
3132290a14a66dc73c9f15ec9cd9f8909c978e11
1 file changed
+35
-4
.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh
+35
-4
@@ -2,15 +2,46 @@
2
3
set -xu
4
5
-# Build the docker image.
6
-docker build -f docker/Dockerfile.tpu -t vllm-tpu .
5
8
-# Set up cleanup.
9
-remove_docker_container() { docker rm -f tpu-test || true; }
6
+remove_docker_container() {
7
+ docker rm -f tpu-test || true;
8
+ docker rm -f vllm-tpu || true;
9
+}
10
+
11
trap remove_docker_container EXIT
12
+
13
# Remove the container that might not be cleaned up in the previous run.
14
remove_docker_container
15
16
+# Build the docker image.
17
+docker build -f docker/Dockerfile.tpu -t vllm-tpu .
18
+
19
+# Set up cleanup.
20
+cleanup_docker() {
21
+ # Get Docker's root directory
22
+ docker_root=$(docker info -f '{{.DockerRootDir}}')
23
+ if [ -z "$docker_root" ]; then
24
+ echo "Failed to determine Docker root directory."
25
+ exit 1
26
+ fi
27
+ echo "Docker root directory: $docker_root"
28
+ # Check disk usage of the filesystem where Docker's root directory is located
29
+ disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
30
+ # Define the threshold
31
+ threshold=70
32
+ if [ "$disk_usage" -gt "$threshold" ]; then
33
+ echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
34
+ # Remove dangling images (those that are not tagged and not used by any container)
35
+ docker image prune -f
36
+ # Remove unused volumes / force the system prune for old images as well.
37
+ docker volume prune -f && docker system prune --force --filter "until=72h" --all
38
+ echo "Docker images and volumes cleanup completed."
39
+ else
40
+ echo "Disk usage is below $threshold%. No cleanup needed."
41
+ fi
42
+}
43
+cleanup_docker
44
+
45
# For HF_TOKEN.
46
source /etc/environment
47