[CI] Report to CRCR after all jobs finish, gated on the build's long pole (#58628)
Andrey Talman committed
Sep 24, 2026 at 19:02 UTC
f5b40fcdfefb66138cb20031f82c86ac0f53722a
2 files changed
+86
-8
.buildkite/scripts/crcr-report.sh
+67
-7
@@ -80,13 +80,73 @@ fi
80
81
BUILD_JSON="$(mktemp)"
82
trap 'rm -f "${BUILD_JSON}"' EXIT
83
-http_code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
84
- -H "Authorization: Bearer ${BK_TOKEN}" \
85
- "https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")"
86
-if [[ "${http_code}" != "200" ]]; then
87
- echo "buildkite API returned ${http_code} -- skipping report"
88
- exit 0
89
-fi
83
+BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}"
84
+
85
+fetch_build() {
86
+ curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
87
+ -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}"
88
+}
89
+
90
+# Count jobs that have not reached a terminal state, ignoring this job: the
91
+# report is itself part of the build and cannot wait for itself to finish.
92
+outstanding_jobs() {
93
+ python3 - "${BUILD_JSON}" "${BUILDKITE_JOB_ID:-}" <<'PY'
94
+import json, sys
95
+TERMINAL = {
96
+ "passed", "failed", "blocked", "canceled", "skipped", "not_run",
97
+ "broken", "timed_out", "waiting_failed", "finished", "expired",
98
+}
99
+build = json.load(open(sys.argv[1]))
100
+self_id = sys.argv[2]
101
+print(sum(
102
+ 1 for j in build.get("jobs", [])
103
+ if j.get("type") == "script"
104
+ and j.get("id") != self_id
105
+ and j.get("state") not in TERMINAL
106
+))
107
+PY
108
+}
109
+
110
+# Wait for the rest of the build. The report is a snapshot of the Buildkite
111
+# API's view of this build, so taking it early does not merely omit jobs -- it
112
+# actively reports them as clean. Build 90830 reported with 14 of 347 jobs
113
+# finished and none of the eventual 152 hard failures visible, and HUD recorded
114
+# that nightly as green.
115
+#
116
+# Polling rather than a full `depends_on` barrier: Buildkite groups carry no
117
+# key, so a barrier means naming all ~250 step keys, which silently rots when a
118
+# step is added and fails the pipeline upload when one is removed or excluded
119
+# from this lane (AMD, retired A100). The pipeline does gate this step on the
120
+# long pole so it is scheduled near the end of the build -- that keeps the agent
121
+# from idling for hours, but it is only a hint, and this loop is what actually
122
+# guarantees the snapshot is complete.
123
+POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}"
124
+# Under the step's own timeout, so a stuck build still gets a partial report
125
+# rather than the job being killed with nothing sent.
126
+WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} ))
127
+while :; do
128
+ http_code="$(fetch_build)"
129
+ if [[ "${http_code}" != "200" ]]; then
130
+ echo "buildkite API returned ${http_code} -- skipping report"
131
+ exit 0
132
+ fi
133
+ remaining="$(outstanding_jobs)" || remaining=""
134
+ if [[ -z "${remaining}" ]]; then
135
+ echo "could not parse build JSON -- reporting what we have"
136
+ break
137
+ fi
138
+ if (( remaining == 0 )); then
139
+ echo "all other jobs have finished -- reporting"
140
+ break
141
+ fi
142
+ if (( $(date +%s) >= WAIT_DEADLINE )); then
143
+ echo "still ${remaining} job(s) running at the wait deadline --" \
144
+ "reporting a partial view"
145
+ break
146
+ fi
147
+ echo "waiting for ${remaining} job(s) to finish..."
148
+ sleep "${POLL_INTERVAL_S}"
149
+done
150
151
# One callback per job, matching HUD's per-job crcr_workflow_job schema.
152
# delivery_id is synthetic: the nightly path has no upstream dispatch to borrow
.buildkite/test_areas/crcr_report.yaml
+19
-1
@@ -1,13 +1,31 @@
1
group: CRCR Report
2
+# The report waits for the rest of the build (see crcr-report.sh), so gate it on
3
+# the long pole rather than letting it wait on an agent. A torch-nightly build
4
+# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three
5
+# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in
6
+# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours.
7
+#
8
+# Depending on it means Buildkite does not schedule this step until the build is
9
+# nearly over, so the agent polls briefly instead of idling ~13h. It is a
10
+# scheduling hint, not the correctness mechanism: the script still waits for
11
+# every job, so if this stops being the slowest step the report is late to start
12
+# but still correct.
13
depends_on:
14
- image-build-cpu
15
+ - lm-eval-large-models-4xh100
16
steps:
17
- label: ":satellite: Report nightly results to CRCR"
18
key: crcr-report
19
if: build.env("TORCH_NIGHTLY") == "1"
20
soft_fail: true
21
allow_dependency_failure: true
10
- timeout_in_minutes: 15
22
+ # depends_on defers the start to near the end of the build, so in the normal
23
+ # case the script polls for minutes. This only has to cover the case where
24
+ # the long pole finishes early and something else is still running, so it is
25
+ # sized against the whole build rather than the expected wait. The script's
26
+ # own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial
27
+ # report; this is the backstop that stops the job outliving the build.
28
+ timeout_in_minutes: 840
29
device: cpu-small
30
commands:
31
- bash /vllm-workspace/.buildkite/scripts/crcr-report.sh