[ROCm][CI] Cache Rust builds by source inputs (#46527)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
Andreas Karatzas committed
Jul 13, 2026 at 01:14 UTC
b7b58d1eba73be5cceeb778212c02999e05c6f7c
3 files changed
+337
-37
.buildkite/scripts/ci-bake-rocm.sh
+181
-18
@@ -15,9 +15,9 @@ set -euo pipefail
15
16
DEFAULT_REPO_SLUG="vllm-project/vllm"
17
DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
18
-DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh"
18
+DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh"
19
DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
20
-DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base"
20
+DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust-toolchain build_rixl build_rocshmem build_deepep mori_base ci_base"
21
DEFAULT_CI_BASE_METADATA_VERSION="1"
22
IMAGE_EXISTED_BEFORE_BUILD=0
23
@@ -764,7 +764,7 @@ configure_ci_base_image_refs() {
764
fi
765
set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}"
766
set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}"
767
- set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT:-}"
767
+ set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-}"
768
set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
769
return 0
770
fi
@@ -1211,12 +1211,24 @@ uses_rocm_csrc_cache() {
1211
esac
1212
}
1213
1214
+uses_rocm_rust_cache() {
1215
+ case "${TARGET}" in
1216
+ rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
1217
+ return 0
1218
+ ;;
1219
+ *)
1220
+ return 1
1221
+ ;;
1222
+ esac
1223
+}
1224
+
1225
compute_rocm_csrc_content_hash() {
1226
local bake_dir=""
1227
local dockerfile_rocm=""
1228
local -a content_paths=(
1229
"requirements/common.txt"
1230
"requirements/rocm.txt"
1231
+ "pyproject.toml"
1232
"setup.py"
1233
"CMakeLists.txt"
1234
"cmake"
@@ -1260,6 +1272,56 @@ compute_rocm_csrc_content_hash_if_needed() {
1272
echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}"
1273
}
1274
1275
+compute_rocm_rust_content_hash() {
1276
+ local bake_dir=""
1277
+ local dockerfile_rocm=""
1278
+ local -a content_paths=(
1279
+ "requirements/build/rust.txt"
1280
+ "rust/Cargo.lock"
1281
+ "rust/Cargo.toml"
1282
+ "rust/proto"
1283
+ "rust/src"
1284
+ "rust-toolchain.toml"
1285
+ "tools/build_rust.py"
1286
+ "tools/install_protoc.sh"
1287
+ "build_rust.sh"
1288
+ )
1289
+ local -a content_args=()
1290
+
1291
+ bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1292
+ dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
1293
+ mapfile -t content_args < <(
1294
+ get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
1295
+ )
1296
+
1297
+ {
1298
+ printf 'rust-input-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
1299
+ printf 'dockerfile:%s\n' "${dockerfile_rocm}"
1300
+ printf 'resolved-build-args:\n'
1301
+ hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
1302
+ printf 'dockerfile-stages:base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build\n'
1303
+ if [[ -f "${dockerfile_rocm}" ]]; then
1304
+ hash_dockerfile_stages "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build"
1305
+ else
1306
+ printf 'missing:%s\n' "${dockerfile_rocm}"
1307
+ fi
1308
+ } | sha256sum | cut -d' ' -f1
1309
+}
1310
+
1311
+compute_rocm_rust_content_hash_if_needed() {
1312
+ local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
1313
+
1314
+ if [[ "${ROCM_RUST_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_rust_cache; then
1315
+ return 0
1316
+ fi
1317
+
1318
+ ROCM_RUST_CONTENT_HASH=$(compute_rocm_rust_content_hash)
1319
+ ROCM_RUST_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}"
1320
+ export ROCM_RUST_CONTENT_HASH
1321
+ export ROCM_RUST_CONTENT_CACHE_REF
1322
+ echo "ROCm Rust content cache ref: ${ROCM_RUST_CONTENT_CACHE_REF}"
1323
+}
1324
+
1325
write_hcl_string_list_entries() {
1326
local indent="$1"
1327
local value=""
@@ -1317,6 +1379,7 @@ write_rocm_build_arg_override() {
1379
"${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \
1380
"${CI_BASE_CONTENT_ARGS:-}"
1381
get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}"
1382
+ get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
1383
} | awk 'NF && !seen[$0]++'
1384
)
1385
@@ -1365,46 +1428,133 @@ validate_cache_export_mode() {
1428
esac
1429
}
1430
1431
+validate_content_cache_export_mode() {
1432
+ local mode="$1"
1433
+ local env_name="$2"
1434
+
1435
+ case "${mode}" in
1436
+ missing|always|never)
1437
+ ;;
1438
+ *)
1439
+ echo "Error: ${env_name} must be one of: missing, always, never"
1440
+ exit 1
1441
+ ;;
1442
+ esac
1443
+}
1444
+
1445
+should_export_content_cache_ref() {
1446
+ local cache_ref="$1"
1447
+ local cache_name="$2"
1448
+ local mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
1449
+
1450
+ case "${mode}" in
1451
+ always)
1452
+ echo "${cache_name} content cache export mode is always; exporting ${cache_ref}"
1453
+ return 0
1454
+ ;;
1455
+ never)
1456
+ echo "${cache_name} content cache export mode is never; not exporting ${cache_ref}"
1457
+ return 1
1458
+ ;;
1459
+ missing|"")
1460
+ if docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1; then
1461
+ echo "${cache_name} content cache exists; not re-exporting ${cache_ref}"
1462
+ return 1
1463
+ fi
1464
+ echo "${cache_name} content cache missing; will export ${cache_ref}"
1465
+ return 0
1466
+ ;;
1467
+ *)
1468
+ echo "Error: ROCM_CONTENT_CACHE_EXPORT_MODE must be one of: missing, always, never"
1469
+ exit 1
1470
+ ;;
1471
+ esac
1472
+}
1473
+
1474
write_rocm_cache_override() {
1475
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
1476
+ local content_cache_export_mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
1477
local csrc_cache_to_mode="${ROCM_CSRC_CACHE_TO_MODE:-max}"
1478
+ local rust_cache_to_mode="${ROCM_RUST_CACHE_TO_MODE:-max}"
1479
local rocm_cache_to_mode="${ROCM_FINAL_CACHE_TO_MODE:-min}"
1372
- local -a content_cache_from=()
1480
+ local -a csrc_content_cache_from=()
1481
+ local -a rust_content_cache_from=()
1482
+ local -a combined_content_cache_from=()
1483
local -a csrc_cache_to=()
1484
+ local -a rust_cache_to=()
1485
local -a rocm_cache_to=()
1486
local -a export_wheel_cache_to=()
1487
+ local export_csrc_cache=1
1488
+ local export_rust_cache=1
1489
1377
- if ! uses_rocm_csrc_cache; then
1490
+ if ! uses_rocm_csrc_cache && ! uses_rocm_rust_cache; then
1491
return 0
1492
fi
1493
1494
+ validate_content_cache_export_mode \
1495
+ "${content_cache_export_mode}" \
1496
+ "ROCM_CONTENT_CACHE_EXPORT_MODE"
1497
validate_cache_export_mode "${csrc_cache_to_mode}" "ROCM_CSRC_CACHE_TO_MODE"
1498
+ validate_cache_export_mode "${rust_cache_to_mode}" "ROCM_RUST_CACHE_TO_MODE"
1499
validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE"
1500
+ echo "ROCm content cache export mode: ${content_cache_export_mode}"
1501
echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}"
1502
+ echo "ROCm Rust cache export mode: ${rust_cache_to_mode}"
1503
echo "ROCm final image cache export mode: ${rocm_cache_to_mode}"
1504
1505
if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then
1387
- content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
1388
- csrc_cache_to+=(
1389
- "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
1390
- )
1506
+ csrc_content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
1507
+ if should_export_content_cache_ref "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc"; then
1508
+ csrc_cache_to+=(
1509
+ "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
1510
+ )
1511
+ else
1512
+ export_csrc_cache=0
1513
+ fi
1514
fi
1515
1516
+ if [[ -n "${ROCM_RUST_CONTENT_CACHE_REF:-}" ]]; then
1517
+ rust_content_cache_from+=("type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}")
1518
+ if should_export_content_cache_ref "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust"; then
1519
+ rust_cache_to+=(
1520
+ "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true"
1521
+ )
1522
+ else
1523
+ export_rust_cache=0
1524
+ fi
1525
+ fi
1526
+
1527
+ combined_content_cache_from=("${csrc_content_cache_from[@]}" "${rust_content_cache_from[@]}")
1528
+
1529
# Docker Hub cache exports are best-effort. A cache-only target failure can
1530
# otherwise cancel the sibling image target before its manifest is pushed.
1531
if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
1396
- csrc_cache_to+=(
1397
- "type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
1398
- )
1532
+ if [[ ${export_csrc_cache} -eq 1 ]]; then
1533
+ csrc_cache_to+=(
1534
+ "type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
1535
+ )
1536
+ fi
1537
+ if [[ ${export_rust_cache} -eq 1 ]]; then
1538
+ rust_cache_to+=(
1539
+ "type=registry,ref=${cache_repo}:rust-rocm-${BUILDKITE_COMMIT},mode=${rust_cache_to_mode},ignore-error=true"
1540
+ )
1541
+ fi
1542
rocm_cache_to+=(
1543
"type=registry,ref=${cache_repo}:rocm-${BUILDKITE_COMMIT},mode=${rocm_cache_to_mode},ignore-error=true"
1544
)
1545
fi
1546
1547
if [[ -n "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then
1405
- csrc_cache_to+=(
1406
- "type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
1407
- )
1548
+ if [[ ${export_csrc_cache} -eq 1 ]]; then
1549
+ csrc_cache_to+=(
1550
+ "type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
1551
+ )
1552
+ fi
1553
+ if [[ ${export_rust_cache} -eq 1 ]]; then
1554
+ rust_cache_to+=(
1555
+ "type=registry,ref=${cache_repo}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rust_cache_to_mode},ignore-error=true"
1556
+ )
1557
+ fi
1558
rocm_cache_to+=(
1559
"type=registry,ref=${cache_repo}:rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rocm_cache_to_mode},ignore-error=true"
1560
)
@@ -1422,7 +1572,7 @@ target "csrc-rocm-ci" {
1572
cache-from = concat(
1573
get_cache_from_rocm_csrc(),
1574
EOF
1425
- write_hcl_string_list " " "${content_cache_from[@]}"
1575
+ write_hcl_string_list " " "${csrc_content_cache_from[@]}"
1576
cat <<EOF
1577
)
1578
EOF
@@ -1430,11 +1580,23 @@ EOF
1580
cat <<EOF
1581
}
1582
1583
+target "rust-rocm-ci" {
1584
+ cache-from = concat(
1585
+ get_cache_from_rocm_rust(),
1586
+EOF
1587
+ write_hcl_string_list " " "${rust_content_cache_from[@]}"
1588
+ cat <<EOF
1589
+ )
1590
+EOF
1591
+ write_hcl_string_list_attr " " "cache-to" "${rust_cache_to[@]}"
1592
+ cat <<EOF
1593
+}
1594
+
1595
target "test-rocm-ci" {
1596
cache-from = concat(
1597
get_cache_from_rocm(),
1598
EOF
1437
- write_hcl_string_list " " "${content_cache_from[@]}"
1599
+ write_hcl_string_list " " "${combined_content_cache_from[@]}"
1600
cat <<EOF
1601
)
1602
EOF
@@ -1446,7 +1608,7 @@ target "export-wheel-rocm" {
1608
cache-from = concat(
1609
get_cache_from_rocm(),
1610
EOF
1449
- write_hcl_string_list " " "${content_cache_from[@]}"
1611
+ write_hcl_string_list " " "${combined_content_cache_from[@]}"
1612
cat <<EOF
1613
)
1614
EOF
@@ -1920,6 +2082,7 @@ main() {
2082
compute_dependency_cache_keys
2083
write_ci_base_label_override
2084
compute_rocm_csrc_content_hash_if_needed
2085
+ compute_rocm_rust_content_hash_if_needed
2086
write_rocm_cache_override
2087
resolve_ci_base_dependency_targets
2088
print_bake_config
docker/Dockerfile.rocm
+111
-17
@@ -1,5 +1,7 @@
1
# default base image
2
ARG REMOTE_VLLM="0"
3
+ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
4
+ARG VLLM_BRANCH="main"
5
ARG COMMON_WORKDIR=/app
6
ARG BASE_IMAGE=rocm/vllm-dev:base
7
ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
@@ -110,10 +112,8 @@ WORKDIR ${COMMON_WORKDIR}
112
FROM base AS fetch_vllm_0
113
ONBUILD COPY ./ vllm/
114
FROM base AS fetch_vllm_1
113
-ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
114
-ARG VLLM_BRANCH="main"
115
-ENV VLLM_REPO=${VLLM_REPO}
116
-ENV VLLM_BRANCH=${VLLM_BRANCH}
115
+ARG VLLM_REPO
116
+ARG VLLM_BRANCH
117
ONBUILD RUN git clone ${VLLM_REPO} \
118
&& cd vllm \
119
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
@@ -124,34 +124,118 @@ ONBUILD RUN git clone ${VLLM_REPO} \
124
FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
125
126
# -----------------------
127
-# Rust build stage
128
-# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
129
-# don't need the rust toolchain or protoc.
130
-FROM fetch_vllm AS rust-build
131
-ARG COMMON_WORKDIR
132
-ARG USE_SCCACHE
127
+# Rust/protoc toolchain inputs
128
+#
129
+# Keep these separate from Rust source inputs so Rust source-only changes do not
130
+# invalidate the long-lived ci_base image that carries the toolchain.
131
+FROM scratch AS rust_toolchain_input_0
132
+COPY rust-toolchain.toml /rust-toolchain-input/vllm/rust-toolchain.toml
133
+COPY tools/install_protoc.sh /rust-toolchain-input/vllm/tools/install_protoc.sh
134
+
135
+FROM base AS rust_toolchain_input_1
136
+ARG VLLM_REPO
137
+ARG VLLM_BRANCH
138
+RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-toolchain-input/vllm \
139
+ && cd /rust-toolchain-input/vllm \
140
+ && git fetch -v --prune -- origin ${VLLM_BRANCH} \
141
+ && git sparse-checkout init --no-cone \
142
+ && git sparse-checkout set \
143
+ rust-toolchain.toml \
144
+ tools/install_protoc.sh \
145
+ && git checkout FETCH_HEAD
146
+
147
+FROM rust_toolchain_input_${REMOTE_VLLM} AS rust-toolchain-input
148
+
149
+# -----------------------
150
+# Rust build inputs
151
+#
152
+# REMOTE_VLLM=0: use only the local files that can affect Rust artifacts.
153
+# REMOTE_VLLM=1: fetch only those same files from the requested remote ref.
154
+FROM scratch AS rust_input_0
155
+COPY requirements/build/rust.txt /rust-input/vllm/requirements/build/rust.txt
156
+COPY rust/Cargo.lock rust/Cargo.toml /rust-input/vllm/rust/
157
+COPY rust/proto /rust-input/vllm/rust/proto
158
+COPY rust/src /rust-input/vllm/rust/src
159
+COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml
160
+COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py
161
+COPY build_rust.sh /rust-input/vllm/build_rust.sh
162
+
163
+FROM base AS rust_input_1
164
+ARG VLLM_REPO
165
+ARG VLLM_BRANCH
166
+RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \
167
+ && cd /rust-input/vllm \
168
+ && git fetch -v --prune -- origin ${VLLM_BRANCH} \
169
+ && git sparse-checkout init --no-cone \
170
+ && git sparse-checkout set \
171
+ requirements/build/rust.txt \
172
+ rust/Cargo.lock \
173
+ rust/Cargo.toml \
174
+ rust/proto/** \
175
+ rust/src/** \
176
+ rust-toolchain.toml \
177
+ tools/build_rust.py \
178
+ build_rust.sh \
179
+ && git checkout FETCH_HEAD
180
+
181
+FROM rust_input_${REMOTE_VLLM} AS rust-input
182
+
183
+# -----------------------
184
+# Rust/protoc toolchain
185
+FROM base AS rust-toolchain
186
+
187
+ENV CARGO_HOME=/root/.cargo
188
+ENV RUSTUP_HOME=/root/.rustup
189
+ENV PATH=${CARGO_HOME}/bin:${PATH}
190
191
# protoc is used by tonic-build/prost-build.
192
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
193
ca-certificates curl unzip \
194
&& rm -rf /var/lib/apt/lists/*
195
139
-COPY tools/install_protoc.sh /tmp/install_protoc.sh
196
+COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/tools/install_protoc.sh /tmp/install_protoc.sh
197
RUN /tmp/install_protoc.sh && rm /tmp/install_protoc.sh
198
199
+COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/rust-toolchain.toml /tmp/rust-toolchain.toml
200
+RUN TOOLCHAIN="$(grep '^channel' /tmp/rust-toolchain.toml | sed 's/.*= *"\(.*\)"/\1/')" \
201
+ && if ! command -v rustup >/dev/null 2>&1; then \
202
+ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
203
+ | sh -s -- -y --default-toolchain none; \
204
+ fi \
205
+ && rustup toolchain install "${TOOLCHAIN}" \
206
+ && rustup default "${TOOLCHAIN}" \
207
+ && rustc --version \
208
+ && cargo --version
209
+
210
+# -----------------------
211
+# Rust build stage
212
+# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
213
+# don't need the rust toolchain or protoc.
214
+FROM rust-toolchain AS rust-build
215
+ARG COMMON_WORKDIR
216
+ARG USE_SCCACHE
217
+
218
# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit
219
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
220
ENV CARGO_BUILD_JOBS=4
221
ENV CARGO_NET_RETRY=10
222
ENV RUSTUP_MAX_RETRIES=10
223
224
+WORKDIR ${COMMON_WORKDIR}/vllm
225
+
226
+COPY --from=rust-input /rust-input/vllm/requirements/build/rust.txt requirements/build/rust.txt
227
+
228
# BuildKit can run this stage in parallel with ROCm native builds. Keep Rust on
229
# a separate local sccache daemon while sharing the same remote cache backend.
230
ENV SCCACHE_SERVER_PORT=4227
231
232
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
153
- cd ${COMMON_WORKDIR}/vllm \
154
- && uv pip install --system -r requirements/build/rust.txt
233
+ uv pip install --system -r requirements/build/rust.txt
234
+
235
+COPY --from=rust-input /rust-input/vllm/rust rust
236
+COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml
237
+COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py
238
+COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh
239
240
# Build the release binary. Cargo's registry/git caches can be written by
241
# concurrent BuildKit jobs on shared workers, so lock those cache mounts while
@@ -159,8 +243,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
243
# can outlive source updates across BuildKit cache reuse.
244
RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \
245
--mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
162
- cd ${COMMON_WORKDIR}/vllm \
163
- && if [ "$USE_SCCACHE" = "1" ]; then \
246
+ if [ "$USE_SCCACHE" = "1" ]; then \
247
export RUSTC_WRAPPER=sccache \
248
&& sccache --show-stats; \
249
fi \
@@ -185,8 +268,8 @@ COPY requirements/common.txt requirements/common.txt
268
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
269
uv pip install --system -r requirements/rocm.txt
270
188
-# pyproject.toml is bind-mounted in the RUN step so metadata-only changes do
189
-# not invalidate the expensive native build layer.
271
+# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
272
+# but it still participates in the BuildKit cache key for this RUN step.
273
COPY setup.py CMakeLists.txt ./
274
COPY tools/build_rust.py tools/build_rust.py
275
COPY cmake cmake/
@@ -556,6 +639,17 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1
639
FROM mori_base AS ci_base
640
ARG COMMON_WORKDIR
641
642
+ENV CARGO_HOME=/root/.cargo
643
+ENV RUSTUP_HOME=/root/.rustup
644
+ENV PATH=${CARGO_HOME}/bin:${PATH}
645
+
646
+# Bake the Rust/protoc toolchain into ci_base so Rust build/test steps do not
647
+# need to fetch it on each per-commit image build.
648
+COPY --from=rust-toolchain /root/.cargo /root/.cargo
649
+COPY --from=rust-toolchain /root/.rustup /root/.rustup
650
+COPY --from=rust-toolchain /usr/local/bin/protoc /usr/local/bin/protoc
651
+COPY --from=rust-toolchain /usr/local/include/google /usr/local/include/google
652
+
653
# Update rdma-core to support latest rocshmem.
654
ARG DEEPEP_NIC
655
RUN if [ "${DEEPEP_NIC}" = "cx7" ] || [ "${DEEPEP_NIC}" = "io" ]; then \
docker/ci-rocm.hcl
+45
-2
@@ -141,6 +141,10 @@ variable "ROCM_CSRC_CACHE_TO_MODE" {
141
default = "max"
142
}
143
144
+variable "ROCM_RUST_CACHE_TO_MODE" {
145
+ default = "max"
146
+}
147
+
148
variable "ROCM_FINAL_CACHE_TO_MODE" {
149
default = "min"
150
}
@@ -164,6 +168,13 @@ function "get_cache_from_rocm" {
168
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
169
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
170
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
171
+ # Import the source-scoped Rust frontend cache so non-Rust changes do not
172
+ # force a fresh cargo release build.
173
+ BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "",
174
+ PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "",
175
+ VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
176
+ ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
177
+ ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
178
# Branch-scoped full image cache - fallback when parent-commit cache is evicted
179
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
180
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
@@ -204,6 +215,27 @@ function "get_cache_to_rocm_csrc" {
215
])
216
}
217
218
+function "get_cache_from_rocm_rust" {
219
+ params = []
220
+ result = compact([
221
+ BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "",
222
+ PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "",
223
+ VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
224
+ ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
225
+ ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
226
+ ])
227
+}
228
+
229
+function "get_cache_to_rocm_rust" {
230
+ params = []
231
+ result = compact([
232
+ # Export exact-commit and branch-scoped Rust caches. A content-addressed
233
+ # cache ref is appended by ci-bake-rocm.sh when that wrapper is used.
234
+ BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT},mode=${ROCM_RUST_CACHE_TO_MODE}" : "",
235
+ ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${ROCM_RUST_CACHE_TO_MODE}" : "",
236
+ ])
237
+}
238
+
239
# Cache functions for upstream dependency stages (RIXL/UCX, ROCShmem, DeepEP).
240
# These stages are pinned to specific upstream commit hashes, so cache keys use
241
# those hashes rather than the Buildkite commit. This means the cache persists
@@ -276,6 +308,17 @@ target "csrc-rocm-ci" {
308
output = ["type=cacheonly"]
309
}
310
311
+# Cache-only target for the Rust frontend build stage. Final-image cache
312
+# exports use mode=min and do not reliably persist intermediate cargo layers,
313
+# so Rust gets its own source-scoped cache target.
314
+target "rust-rocm-ci" {
315
+ inherits = ["_common-rocm", "_ci-rocm"]
316
+ target = "rust-build"
317
+ cache-from = get_cache_from_rocm_rust()
318
+ cache-to = get_cache_to_rocm_rust()
319
+ output = ["type=cacheonly"]
320
+}
321
+
322
# Keep wheel export on the same CI graph as the test image build so the
323
# shared build_vllm/export_vllm stages resolve identically within one bake
324
# invocation. Without this, export-wheel-rocm uses the plain local target
@@ -292,13 +335,13 @@ target "export-wheel-rocm" {
335
# Artifact-only vLLM build. GPU test jobs consume this artifact on top of
336
# ci_base, avoiding a per-commit multi-GB image push/pull.
337
group "test-rocm-ci-with-artifacts" {
295
- targets = ["csrc-rocm-ci", "export-wheel-rocm"]
338
+ targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"]
339
}
340
341
# Full test image + wheel export. Kept for fallback/debugging when a pushed
342
# per-commit image is useful.
343
group "test-rocm-ci-with-wheel" {
301
- targets = ["csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
344
+ targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
345
}
346
347
# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG