[CI] Bound the CRCR report by build age, and stop gating it on a job (#59066)
Andrey Talman committed
Sep 28, 2026 at 20:42 UTC
869278cbb02fc04a5f10d2eaba2bb251939e1eab
2 files changed
+100
-35
.buildkite/scripts/crcr-report.sh
+83
-17
@@ -79,12 +79,22 @@ if [[ -z "${OIDC_TOKEN}" ]]; then
79
fi
80
81
BUILD_JSON="$(mktemp)"
82
-trap 'rm -f "${BUILD_JSON}"' EXIT
82
+BUILD_JSON_STAGING="$(mktemp)"
83
+trap 'rm -f "${BUILD_JSON}" "${BUILD_JSON_STAGING}"' EXIT
84
BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}"
85
86
+# Staged so a failed request cannot clobber the last good snapshot: curl -o
87
+# writes the error body too, and the deadline path reports from whatever
88
+# snapshot it has.
89
fetch_build() {
86
- curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
87
- -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}"
90
+ local code
91
+ code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON_STAGING}" \
92
+ -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}")"
93
+ if [[ "${code}" == "200" ]]; then
94
+ mv -f "${BUILD_JSON_STAGING}" "${BUILD_JSON}"
95
+ BUILD_JSON_STAGING="$(mktemp)"
96
+ fi
97
+ echo "${code}"
98
}
99
100
# Count jobs that have not reached a terminal state, ignoring this job: the
@@ -113,22 +123,78 @@ PY
123
# finished and none of the eventual 152 hard failures visible, and HUD recorded
124
# that nightly as green.
125
#
116
-# Polling rather than a full `depends_on` barrier: Buildkite groups carry no
117
-# key, so a barrier means naming all ~250 step keys, which silently rots when a
118
-# step is added and fails the pipeline upload when one is removed or excluded
119
-# from this lane (AMD, retired A100). The pipeline does gate this step on the
120
-# long pole so it is scheduled near the end of the build -- that keeps the agent
121
-# from idling for hours, but it is only a hint, and this loop is what actually
122
-# guarantees the snapshot is complete.
123
-POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}"
124
-# Under the step's own timeout, so a stuck build still gets a partial report
125
-# rather than the job being killed with nothing sent.
126
-WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} ))
126
+# Polling rather than a `depends_on` barrier: Buildkite groups carry no key, so
127
+# a barrier means naming all ~250 step keys, which silently rots when a step is
128
+# added and fails the pipeline upload when one is removed or excluded from this
129
+# lane (AMD, retired A100). Gating on a single step was tried and is worse: a
130
+# dependency that fails or is cancelled takes the report with it despite
131
+# allow_dependency_failure (builds 91191 and 91461 both lost their report that
132
+# way), so this step depends only on the image it runs in.
133
+# 15 min: the wait now spans the whole build, so a 60s poll made ~420 requests
134
+# against an API shared with the rest of CI to learn something that changes on
135
+# the order of minutes. The cost is up to one interval of extra latency after
136
+# the last job lands.
137
+POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}"
138
+
139
+# The deadline is measured from when the *build* started, not from when this
140
+# script did, so "report no later than N hours into the build" holds however
141
+# late this step is scheduled. Anchoring it to script start made the bound
142
+# meaningless whenever the step was itself delayed.
143
+#
144
+# Resolved after the first fetch, from the build's own created_at.
145
+MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-25200}"
146
+WAIT_DEADLINE=""
147
+# Used until created_at is known. Without it, an API outage on the very first
148
+# request leaves the deadline unresolved and the loop spins to the step timeout.
149
+FALLBACK_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S ))
150
+
151
+build_deadline() {
152
+ python3 - "${BUILD_JSON}" "${MAX_BUILD_AGE_S}" <<'PY'
153
+import datetime, json, sys
154
+build = json.load(open(sys.argv[1]))
155
+created = build.get("created_at")
156
+if not created:
157
+ raise SystemExit(1)
158
+started = datetime.datetime.fromisoformat(created.replace("Z", "+00:00"))
159
+print(int(started.timestamp()) + int(sys.argv[2]))
160
+PY
161
+}
162
+
163
while :; do
164
http_code="$(fetch_build)"
165
if [[ "${http_code}" != "200" ]]; then
130
- echo "buildkite API returned ${http_code} -- skipping report"
131
- exit 0
166
+ # A transient 429/5xx must not cost the whole report: treating one
167
+ # failure as fatal would drop a nightly for a blip. Keep polling on the
168
+ # last good snapshot; only give up if the deadline passes having never
169
+ # fetched one.
170
+ echo "buildkite API returned ${http_code}"
171
+ if [[ ! -s "${BUILD_JSON}" ]]; then
172
+ if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then
173
+ echo "no build data was ever fetched -- nothing to report"
174
+ exit 0
175
+ fi
176
+ sleep "${POLL_INTERVAL_S}"
177
+ continue
178
+ fi
179
+ if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then
180
+ echo "deadline reached while the API is unavailable --" \
181
+ "reporting from the last good snapshot"
182
+ break
183
+ fi
184
+ sleep "${POLL_INTERVAL_S}"
185
+ continue
186
+ fi
187
+ if [[ -z "${WAIT_DEADLINE}" ]]; then
188
+ WAIT_DEADLINE="$(build_deadline)" || WAIT_DEADLINE=""
189
+ if [[ -z "${WAIT_DEADLINE}" ]]; then
190
+ # No created_at to anchor to; fall back to script start so the loop
191
+ # still has a bound rather than running until the step times out.
192
+ WAIT_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S ))
193
+ echo "build created_at unavailable -- deadline measured from now"
194
+ fi
195
+ echo "reporting deadline: $(date -u -d "@${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null \
196
+ || date -u -r "${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ')" \
197
+ "(build age limit ${MAX_BUILD_AGE_S}s)"
198
fi
199
remaining="$(outstanding_jobs)" || remaining=""
200
if [[ -z "${remaining}" ]]; then
@@ -140,7 +206,7 @@ while :; do
206
break
207
fi
208
if (( $(date +%s) >= WAIT_DEADLINE )); then
143
- echo "still ${remaining} job(s) running at the wait deadline --" \
209
+ echo "still ${remaining} job(s) running at the build-age deadline --" \
210
"reporting a partial view"
211
break
212
fi
.buildkite/test_areas/crcr_report.yaml
+17
-18
@@ -1,31 +1,30 @@
1
group: CRCR Report
2
-# The report waits for the rest of the build (see crcr-report.sh), so gate it on
3
-# the long pole rather than letting it wait on an agent. A torch-nightly build
4
-# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three
5
-# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in
6
-# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours.
2
+# Only image-build-cpu: this step must always run.
3
#
8
-# Depending on it means Buildkite does not schedule this step until the build is
9
-# nearly over, so the agent polls briefly instead of idling ~13h. It is a
10
-# scheduling hint, not the correctness mechanism: the script still waits for
11
-# every job, so if this stops being the slowest step the report is late to start
12
-# but still correct.
4
+# It was briefly gated on the build's long pole (lm-eval-large-models-4xh100)
5
+# so the agent would not idle. That backfired on the first nightly to use it,
6
+# build 91191: the long pole OOMed, hung, and was cancelled -- and a cancelled
7
+# dependency propagates past `allow_dependency_failure`, so the step went
8
+# `waiting_failed` and that nightly got no CRCR row at all. Nothing in this
9
+# pipeline sets timeout_in_minutes (the pipeline default and org maximum are
10
+# both unset), so a hung job has no terminal state to reach either: gating on
11
+# any single job can suppress the report indefinitely.
12
+#
13
+# The bound now lives in the script, measured from the build's created_at, so
14
+# the report lands within CRCR_MAX_BUILD_AGE_S of the build starting whatever
15
+# else happens.
16
depends_on:
17
- image-build-cpu
15
- - lm-eval-large-models-4xh100
18
steps:
19
- label: ":satellite: Report nightly results to CRCR"
20
key: crcr-report
21
if: build.env("TORCH_NIGHTLY") == "1"
22
soft_fail: true
23
allow_dependency_failure: true
22
- # depends_on defers the start to near the end of the build, so in the normal
23
- # case the script polls for minutes. This only has to cover the case where
24
- # the long pole finishes early and something else is still running, so it is
25
- # sized against the whole build rather than the expected wait. The script's
26
- # own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial
27
- # report; this is the backstop that stops the job outliving the build.
28
- timeout_in_minutes: 840
24
+ # The script polls from the start of the build now, so this covers the whole
25
+ # wait. It sits above the script's own build-age deadline (7h) so the script
26
+ # sends its partial report rather than being killed with nothing sent.
27
+ timeout_in_minutes: 480
28
device: cpu-small
29
commands:
30
- bash /vllm-workspace/.buildkite/scripts/crcr-report.sh