A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 17:01 UTC
[ROCm][CI] Stabilize Cargo cache and pre-test image checks (#43815)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
May 27, 2026 at 22:24 UTC
33e94fc3adcb888a8c6d62285f70bc78fb068ec7
3 files changed
+46
-29
.buildkite/hardware_tests/amd.yaml
+20
@@ -17,6 +17,26 @@ steps:
17
--target test
18
--no-cache
19
--progress plain .
20
+ - |
21
+ docker run --rm --network=none --entrypoint /bin/bash "rocm/vllm-ci:${BUILDKITE_COMMIT}" -ec '
22
+ if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
23
+ if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
24
+ if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
25
+ if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
26
+ command -v python3
27
+ command -v uv
28
+ command -v pytest
29
+ if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
30
+ echo No ROCm CLI found in image >&2
31
+ exit 1
32
+ fi
33
+ python3 - <<PY
34
+ import torch, vllm
35
+ print(torch.__version__)
36
+ print(vllm.__version__)
37
+ PY
38
+ echo AMD image smoke OK
39
+ '
40
- docker push "rocm/vllm-ci:${BUILDKITE_COMMIT}"
41
env:
42
DOCKER_BUILDKIT: "1"
.buildkite/scripts/hardware_ci/run-amd-test.sh
+15
-23
@@ -35,25 +35,9 @@ export PYTHONPATH=".."
35
# Helper Functions
36
###############################################################################
37
38
-cleanup_docker() {
39
- # Get Docker's root directory
40
- docker_root=$(docker info -f '{{.DockerRootDir}}')
41
- if [ -z "$docker_root" ]; then
42
- echo "Failed to determine Docker root directory."
43
- exit 1
44
- fi
45
- echo "Docker root directory: $docker_root"
46
-
47
- disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
48
- threshold=70
49
- if [ "$disk_usage" -gt "$threshold" ]; then
50
- echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
51
- docker image prune -f
52
- docker volume prune -f && docker system prune --force --filter "until=72h" --all
53
- echo "Docker images and volumes cleanup completed."
54
- else
55
- echo "Disk usage is below $threshold%. No cleanup needed."
56
- fi
38
+report_docker_usage() {
39
+ echo "--- Docker usage"
40
+ docker system df || true
41
}
42
43
cleanup_network() {
@@ -254,8 +238,8 @@ re_quote_pytest_markers() {
238
echo "--- ROCm info"
239
rocminfo
240
257
-# --- Docker housekeeping ---
258
-cleanup_docker
241
+# --- Docker status ---
242
+report_docker_usage
243
244
# --- Pull test image ---
245
echo "--- Pulling container"
@@ -264,9 +248,17 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea
248
docker pull "${image_name}"
249
250
remove_docker_container() {
267
- docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
251
+ # docker run uses --rm, so the container is normally already gone when the
252
+ # EXIT trap runs. Cleanup is best-effort and must not affect the test result.
253
+ docker rm -f "${container_name}" >/dev/null 2>&1 || true
254
+}
255
+
256
+on_exit() {
257
+ local exit_code=$?
258
+ remove_docker_container
259
+ exit "$exit_code"
260
}
269
-trap remove_docker_container EXIT
261
+trap on_exit EXIT
262
263
# --- Prepare commands ---
264
echo "--- Running container"
docker/Dockerfile.rocm
+11
-6
@@ -135,13 +135,18 @@ ENV PATH="/root/.cargo/bin:${PATH}"
135
# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit
136
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
137
ENV CARGO_BUILD_JOBS=4
138
-
139
-# Build the release binary. Cache cargo registry/git, and copy the binary out
140
-# so it persists into the image layer for later COPY --from=rust-build.
141
-RUN --mount=type=cache,target=/root/.cargo/registry \
142
- --mount=type=cache,target=/root/.cargo/git \
138
+ENV CARGO_NET_RETRY=10
139
+ENV RUSTUP_MAX_RETRIES=10
140
+
141
+# Build the release binary. Cargo's registry/git caches can be written by
142
+# concurrent BuildKit jobs on shared workers, so lock those cache mounts while
143
+# keeping the cache benefit. Copy the binary out so it persists into the image
144
+# layer for later COPY --from=rust-build.
145
+RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \
146
+ --mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
147
cd ${COMMON_WORKDIR}/vllm \
144
- && VLLM_RS_TARGET_PATH=/tmp/vllm-rs bash build_rust.sh
148
+ && VLLM_RS_TARGET_PATH=/tmp/vllm-rs bash build_rust.sh \
149
+ && test -x /tmp/vllm-rs
150
151
# -----------------------
152
# vLLM build stages