@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e07aff9e52
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[CI/Build] Docker cleanup functionality for amd servers (#5112)

Co-authored-by: Alexey Kondratiev <alexey.kondratiev@amd.com> Co-authored-by: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com> Co-authored-by: Alexei V. Ivanov <alexei.ivanov@amd.com> Co-authored-by: omkarkakarparthi <okakarpa>

omkar kakarparthi committed May 29, 2024 at 22:27 UTC e07aff9e52342dc82b73c803ba69601242801bc4
1 file changed +28
.buildkite/run-amd-test.sh
+28
@@ -5,6 +5,34 @@ set -ex
5 echo "--- ROCm info"
6 rocminfo
7
8 +# cleanup older docker images
9 +cleanup_docker() {
10 + # Get Docker's root directory
11 + docker_root=$(docker info -f '{{.DockerRootDir}}')
12 + if [ -z "$docker_root" ]; then
13 + echo "Failed to determine Docker root directory."
14 + exit 1
15 + fi
16 + echo "Docker root directory: $docker_root"
17 + # Check disk usage of the filesystem where Docker's root directory is located
18 + disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
19 + # Define the threshold
20 + threshold=70
21 + if [ "$disk_usage" -gt "$threshold" ]; then
22 + echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
23 + # Remove dangling images (those that are not tagged and not used by any container)
24 + docker image prune -f
25 + # Remove unused volumes
26 + docker volume prune -f
27 + echo "Docker images and volumes cleanup completed."
28 + else
29 + echo "Disk usage is below $threshold%. No cleanup needed."
30 + fi
31 +}
32 +
33 +# Call the cleanup docker function
34 +cleanup_docker
35 +
36 echo "--- Resetting GPUs"
37
38 echo "reset" > /opt/amdgpu/etc/gpu_state