[CI][ROCm] Stabilize ci_base hash calculation and image handoff (#48746)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jul 15, 2026 at 10:56 UTC
0885b51981d09089e7a067276046b04a20e5b59d
2 files changed
+69
-6
.buildkite/scripts/ci-bake-rocm.sh
+55
-3
@@ -285,7 +285,7 @@ get_content_arg_names() {
285
fi | awk 'NF && !seen[$0]++'
286
}
287
288
-compute_ci_base_content_hash() {
288
+compute_ci_base_content_hash_once() {
289
local -a content_paths=()
290
local -a content_args=()
291
local dockerfile="${CI_BASE_DOCKERFILE:-}"
@@ -301,7 +301,8 @@ compute_ci_base_content_hash() {
301
if [[ -n "${dockerfile}" ]]; then
302
printf 'dockerfile:%s\n' "${dockerfile}"
303
printf 'resolved-build-args:\n'
304
- hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}"
304
+ hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}" \
305
+ || return 1
306
if [[ -n "${stages}" ]]; then
307
printf 'dockerfile-stages:%s\n' "${stages}"
308
if [[ -f "${dockerfile}" ]]; then
@@ -314,6 +315,53 @@ compute_ci_base_content_hash() {
315
} | sha256sum | cut -d' ' -f1
316
}
317
318
+compute_ci_base_content_hash() {
319
+ local attempts="${CI_BASE_HASH_ATTEMPTS:-3}"
320
+ local delay_secs="${CI_BASE_HASH_RETRY_DELAY:-5}"
321
+ local attempt=0
322
+ local hash=""
323
+ local failed=0
324
+ local -a hashes=()
325
+
326
+ if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then
327
+ echo "Invalid CI_BASE_HASH_ATTEMPTS: ${attempts}" >&2
328
+ return 1
329
+ fi
330
+ if [[ ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
331
+ echo "Invalid CI_BASE_HASH_RETRY_DELAY: ${delay_secs}" >&2
332
+ return 1
333
+ fi
334
+
335
+ for ((attempt = 1; attempt <= attempts; attempt++)); do
336
+ if ! hash=$(compute_ci_base_content_hash_once); then
337
+ echo "ci_base content hash calculation ${attempt}/${attempts} failed" >&2
338
+ failed=1
339
+ else
340
+ hashes+=("${hash}")
341
+ echo "ci_base content hash calculation ${attempt}/${attempts}: ${hash}" >&2
342
+ fi
343
+
344
+ if ((attempt < attempts)); then
345
+ sleep "${delay_secs}"
346
+ fi
347
+ done
348
+
349
+ if ((failed)) || ((${#hashes[@]} != attempts)); then
350
+ echo "Could not calculate a reliable ci_base content hash" >&2
351
+ return 1
352
+ fi
353
+
354
+ for hash in "${hashes[@]:1}"; do
355
+ if [[ "${hash}" != "${hashes[0]}" ]]; then
356
+ echo "ci_base content hash changed between calculations" >&2
357
+ printf ' observed: %s\n' "${hashes[@]}" >&2
358
+ return 1
359
+ fi
360
+ done
361
+
362
+ printf '%s\n' "${hashes[0]}"
363
+}
364
+
365
extract_dockerfile_arg_default() {
366
local dockerfile="$1"
367
local arg_name="$2"
@@ -366,7 +414,11 @@ hash_dockerfile_arg_values() {
414
printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
415
if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
416
digest=$(resolve_image_digest "${arg_value}")
369
- printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest:-unknown}"
417
+ if [[ -z "${digest}" ]]; then
418
+ echo "Failed to resolve digest for BASE_IMAGE=${arg_value}" >&2
419
+ return 1
420
+ fi
421
+ printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest}"
422
fi
423
done
424
}
.buildkite/scripts/rocm/build-test-image.sh
+14
-3
@@ -13,6 +13,18 @@ metadata_get() {
13
fi
14
}
15
16
+use_ci_base_if_present() {
17
+ local ci_base_image=""
18
+
19
+ ci_base_image="$(metadata_get rocm-ci-base-image)"
20
+ if [[ -z "${ci_base_image}" ]]; then
21
+ return 1
22
+ fi
23
+
24
+ export CI_BASE_IMAGE="${ci_base_image}"
25
+ echo "Using ROCm ci_base image selected by the preceding build step: ${CI_BASE_IMAGE}"
26
+}
27
+
28
use_refreshed_base_if_present() {
29
local base_refreshed=""
30
@@ -22,15 +34,12 @@ use_refreshed_base_if_present() {
34
fi
35
36
export BASE_IMAGE
25
- export CI_BASE_IMAGE
37
export IMAGE_TAG_LATEST
38
39
BASE_IMAGE="$(metadata_get rocm-base-image)"
29
- CI_BASE_IMAGE="$(metadata_get rocm-ci-base-image)"
40
IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)"
41
42
echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}"
33
- echo "Using refreshed ROCm ci_base image for test image: ${CI_BASE_IMAGE}"
43
if [[ -n "${IMAGE_TAG_LATEST}" ]]; then
44
echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}"
45
fi
@@ -41,6 +50,8 @@ use_refreshed_base_if_present() {
50
main() {
51
local base_refreshed=0
52
53
+ use_ci_base_if_present || true
54
+
55
if use_refreshed_base_if_present; then
56
base_refreshed=1
57
fi