[CI] Mint the CRCR report's OIDC token after the wait, not before (#59259)
Andrey Talman committed
Sep 29, 2026 at 14:50 UTC
35cd03ebff344cc933ef7f42e45cfb19c76d0db7
1 file changed
+16
-3
.buildkite/scripts/crcr-report.sh
+16
-3
@@ -71,9 +71,11 @@ if [[ -z "${BK_TOKEN}" ]]; then
71
fi
72
73
AUDIENCE="pytorch-cross-repo-ci-relay"
74
+# Pre-flight only; the token minted here is discarded. Checking now means an
75
+# unusable OIDC setup costs seconds rather than being discovered after hours of
76
+# polling. The token that is actually sent is minted after the wait, below.
77
# OIDC redaction also requires the unavailable Job API socket.
75
-OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)"
76
-if [[ -z "${OIDC_TOKEN}" ]]; then
78
+if [[ -z "$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)" ]]; then
79
echo "could not mint a Buildkite OIDC token -- skipping report"
80
exit 0
81
fi
@@ -142,7 +144,7 @@ POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}"
144
# meaningless whenever the step was itself delayed.
145
#
146
# Resolved after the first fetch, from the build's own created_at.
145
-MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-25200}"
147
+MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-14400}"
148
WAIT_DEADLINE=""
149
# Used until created_at is known. Without it, an API outage on the very first
150
# request leaves the deadline unresolved and the loop spins to the step timeout.
@@ -220,6 +222,17 @@ done
222
# Resolved from this script's location: the pipeline runs it from
223
# /vllm-workspace/tests, so a repo-relative path would not resolve.
224
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
225
+
226
+# Minted here, not before the poll loop: a Buildkite OIDC token defaults to a
227
+# five-minute lifetime and the loop above can run to the build-age deadline.
228
+# Build 91827 waited 6h48m and then took a 401 on all 360 callbacks from a
229
+# token that had expired hours earlier.
230
+OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)"
231
+if [[ -z "${OIDC_TOKEN}" ]]; then
232
+ echo "could not mint a Buildkite OIDC token after the wait -- skipping report"
233
+ exit 0
234
+fi
235
+
236
python3 "${SCRIPT_DIR}/crcr_report.py" \
237
--build-json "${BUILD_JSON}" \
238
--callback-url "${CALLBACK_URL}" \