A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[CI/Build] Docker cleanup functionality for amd servers (#5112)
Co-authored-by: Alexey Kondratiev <alexey.kondratiev@amd.com> Co-authored-by: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com> Co-authored-by: Alexei V. Ivanov <alexei.ivanov@amd.com> Co-authored-by: omkarkakarparthi <okakarpa>
omkar kakarparthi committed
May 29, 2024 at 22:27 UTC
e07aff9e52342dc82b73c803ba69601242801bc4
1 file changed
+28
.buildkite/run-amd-test.sh
+28
@@ -5,6 +5,34 @@ set -ex
5
echo "--- ROCm info"
6
rocminfo
7
8
+# cleanup older docker images
9
+cleanup_docker() {
10
+ # Get Docker's root directory
11
+ docker_root=$(docker info -f '{{.DockerRootDir}}')
12
+ if [ -z "$docker_root" ]; then
13
+ echo "Failed to determine Docker root directory."
14
+ exit 1
15
+ fi
16
+ echo "Docker root directory: $docker_root"
17
+ # Check disk usage of the filesystem where Docker's root directory is located
18
+ disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
19
+ # Define the threshold
20
+ threshold=70
21
+ if [ "$disk_usage" -gt "$threshold" ]; then
22
+ echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
23
+ # Remove dangling images (those that are not tagged and not used by any container)
24
+ docker image prune -f
25
+ # Remove unused volumes
26
+ docker volume prune -f
27
+ echo "Docker images and volumes cleanup completed."
28
+ else
29
+ echo "Disk usage is below $threshold%. No cleanup needed."
30
+ fi
31
+}
32
+
33
+# Call the cleanup docker function
34
+cleanup_docker
35
+
36
echo "--- Resetting GPUs"
37
38
echo "reset" > /opt/amdgpu/etc/gpu_state