| 1 | #!/bin/bash |
| 2 | # Report this build's per-job results to the PyTorch Cross-Repo CI Relay (CRCR). |
| 3 | # |
| 4 | # Only runs in the torch-nightly lane on main. CRCR's nightly path is a |
| 5 | # self-report: unlike PR-triggered callbacks there is no upstream dispatch to |
| 6 | # correlate with, so the relay accepts a single "completed" callback per job and |
| 7 | # forwards it to HUD (hud.pytorch.org/crcr). |
| 8 | # |
| 9 | # Reporting is best-effort. A relay outage, an expired mapping or a missing |
| 10 | # token must never fail the nightly build, so every failure path here exits 0 |
| 11 | # after logging. The step is also marked soft_fail in the pipeline. |
| 12 | |
| 13 | set -uo pipefail |
| 14 | |
| 15 | # --- Gating ----------------------------------------------------------------- |
| 16 | # Matches the image_build.sh convention: the lane is selected inside the script |
| 17 | # rather than in pipeline YAML. |
| 18 | if [[ "${TORCH_NIGHTLY:-0}" != "1" ]]; then |
| 19 | echo "TORCH_NIGHTLY != 1 -- not the nightly lane, nothing to report" |
| 20 | exit 0 |
| 21 | fi |
| 22 | |
| 23 | # Defence in depth. The relay decides what a Buildkite pipeline may claim via |
| 24 | # ci_providers.yml, and a pipeline that builds fork PRs should constrain |
| 25 | # build_branch there. Refusing to even mint a token off main means a fork PR |
| 26 | # cannot report as vllm-project/vllm even if that mapping is ever relaxed. |
| 27 | if [[ "${BUILDKITE_BRANCH:-}" != "main" ]]; then |
| 28 | echo "branch '${BUILDKITE_BRANCH:-}' is not main -- refusing to report" |
| 29 | exit 0 |
| 30 | fi |
| 31 | |
| 32 | CALLBACK_URL="${CRCR_CALLBACK_URL:-}" |
| 33 | if [[ -z "${CALLBACK_URL}" ]]; then |
| 34 | echo "CRCR_CALLBACK_URL unset -- skipping report" |
| 35 | exit 0 |
| 36 | fi |
| 37 | |
| 38 | # read_builds only. Needed because a job cannot see its siblings' outcomes: |
| 39 | # the agent exposes only its own step, so the job list comes from the REST API. |
| 40 | TOKEN_SECRET_KEY="${CRCR_BUILDKITE_TOKEN_SECRET_KEY:-CRCR_BUILDKITE_API_TOKEN}" |
| 41 | BK_TOKEN="${BUILDKITE_API_TOKEN:-}" |
| 42 | if [[ -z "${BK_TOKEN}" ]]; then |
| 43 | # Not in the job environment, so read it from a Buildkite secret. |
| 44 | # |
| 45 | # Report why a lookup failed. Swallowing stderr made a missing secret, a |
| 46 | # denied policy and an unusable agent indistinguishable, all surfacing as the |
| 47 | # same "no token" line. Only stderr is echoed -- stdout is the secret. |
| 48 | if ! command -v buildkite-agent >/dev/null 2>&1; then |
| 49 | echo "buildkite-agent is not on PATH; cannot read secret '${TOKEN_SECRET_KEY}'" |
| 50 | else |
| 51 | secret_err="$(mktemp)" |
| 52 | # Requires agent >= 3.107.0: the Docker plugin does not mount the Job API |
| 53 | # socket needed for redaction. Capture the value without logging it. |
| 54 | if BK_TOKEN="$(buildkite-agent secret get --skip-redaction "${TOKEN_SECRET_KEY}" 2>"${secret_err}")"; then |
| 55 | if [[ -z "${BK_TOKEN}" ]]; then |
| 56 | echo "secret '${TOKEN_SECRET_KEY}' resolved but is empty" |
| 57 | fi |
| 58 | else |
| 59 | BK_TOKEN="" |
| 60 | echo "buildkite-agent secret get '${TOKEN_SECRET_KEY}' failed" \ |
| 61 | "(agent $(buildkite-agent --version 2>&1 | head -1)):" |
| 62 | sed 's/^/ /' "${secret_err}" |
| 63 | fi |
| 64 | rm -f "${secret_err}" |
| 65 | fi |
| 66 | fi |
| 67 | if [[ -z "${BK_TOKEN}" ]]; then |
| 68 | echo "no Buildkite API token (env BUILDKITE_API_TOKEN or secret" \ |
| 69 | "'${TOKEN_SECRET_KEY}') -- skipping report" |
| 70 | exit 0 |
| 71 | fi |
| 72 | |
| 73 | AUDIENCE="pytorch-cross-repo-ci-relay" |
| 74 | # Pre-flight only; the token minted here is discarded. Checking now means an |
| 75 | # unusable OIDC setup costs seconds rather than being discovered after hours of |
| 76 | # polling. The token that is actually sent is minted after the wait, below. |
| 77 | # OIDC redaction also requires the unavailable Job API socket. |
| 78 | if [[ -z "$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)" ]]; then |
| 79 | echo "could not mint a Buildkite OIDC token -- skipping report" |
| 80 | exit 0 |
| 81 | fi |
| 82 | |
| 83 | BUILD_JSON="$(mktemp)" |
| 84 | BUILD_JSON_STAGING="$(mktemp)" |
| 85 | trap 'rm -f "${BUILD_JSON}" "${BUILD_JSON_STAGING}"' EXIT |
| 86 | BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}" |
| 87 | |
| 88 | # Staged so a failed request cannot clobber the last good snapshot: curl -o |
| 89 | # writes the error body too, and the deadline path reports from whatever |
| 90 | # snapshot it has. |
| 91 | fetch_build() { |
| 92 | local code |
| 93 | code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON_STAGING}" \ |
| 94 | -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}")" |
| 95 | if [[ "${code}" == "200" ]]; then |
| 96 | mv -f "${BUILD_JSON_STAGING}" "${BUILD_JSON}" |
| 97 | BUILD_JSON_STAGING="$(mktemp)" |
| 98 | fi |
| 99 | echo "${code}" |
| 100 | } |
| 101 | |
| 102 | # Count jobs that have not reached a terminal state, ignoring this job: the |
| 103 | # report is itself part of the build and cannot wait for itself to finish. |
| 104 | outstanding_jobs() { |
| 105 | python3 - "${BUILD_JSON}" "${BUILDKITE_JOB_ID:-}" <<'PY' |
| 106 | import json, sys |
| 107 | TERMINAL = { |
| 108 | "passed", "failed", "blocked", "canceled", "skipped", "not_run", |
| 109 | "broken", "timed_out", "waiting_failed", "finished", "expired", |
| 110 | } |
| 111 | build = json.load(open(sys.argv[1])) |
| 112 | self_id = sys.argv[2] |
| 113 | print(sum( |
| 114 | 1 for j in build.get("jobs", []) |
| 115 | if j.get("type") == "script" |
| 116 | and j.get("id") != self_id |
| 117 | and j.get("state") not in TERMINAL |
| 118 | )) |
| 119 | PY |
| 120 | } |
| 121 | |
| 122 | # Wait for the rest of the build. The report is a snapshot of the Buildkite |
| 123 | # API's view of this build, so taking it early does not merely omit jobs -- it |
| 124 | # actively reports them as clean. Build 90830 reported with 14 of 347 jobs |
| 125 | # finished and none of the eventual 152 hard failures visible, and HUD recorded |
| 126 | # that nightly as green. |
| 127 | # |
| 128 | # Polling rather than a `depends_on` barrier: Buildkite groups carry no key, so |
| 129 | # a barrier means naming all ~250 step keys, which silently rots when a step is |
| 130 | # added and fails the pipeline upload when one is removed or excluded from this |
| 131 | # lane (AMD, retired A100). Gating on a single step was tried and is worse: a |
| 132 | # dependency that fails or is cancelled takes the report with it despite |
| 133 | # allow_dependency_failure (builds 91191 and 91461 both lost their report that |
| 134 | # way), so this step depends only on the image it runs in. |
| 135 | # 15 min: the wait now spans the whole build, so a 60s poll made ~420 requests |
| 136 | # against an API shared with the rest of CI to learn something that changes on |
| 137 | # the order of minutes. The cost is up to one interval of extra latency after |
| 138 | # the last job lands. |
| 139 | POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}" |
| 140 | |
| 141 | # The deadline is measured from when the *build* started, not from when this |
| 142 | # script did, so "report no later than N hours into the build" holds however |
| 143 | # late this step is scheduled. Anchoring it to script start made the bound |
| 144 | # meaningless whenever the step was itself delayed. |
| 145 | # |
| 146 | # Resolved after the first fetch, from the build's own created_at. |
| 147 | MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-14400}" |
| 148 | WAIT_DEADLINE="" |
| 149 | # Used until created_at is known. Without it, an API outage on the very first |
| 150 | # request leaves the deadline unresolved and the loop spins to the step timeout. |
| 151 | FALLBACK_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S )) |
| 152 | |
| 153 | build_deadline() { |
| 154 | python3 - "${BUILD_JSON}" "${MAX_BUILD_AGE_S}" <<'PY' |
| 155 | import datetime, json, sys |
| 156 | build = json.load(open(sys.argv[1])) |
| 157 | created = build.get("created_at") |
| 158 | if not created: |
| 159 | raise SystemExit(1) |
| 160 | started = datetime.datetime.fromisoformat(created.replace("Z", "+00:00")) |
| 161 | print(int(started.timestamp()) + int(sys.argv[2])) |
| 162 | PY |
| 163 | } |
| 164 | |
| 165 | while :; do |
| 166 | http_code="$(fetch_build)" |
| 167 | if [[ "${http_code}" != "200" ]]; then |
| 168 | # A transient 429/5xx must not cost the whole report: treating one |
| 169 | # failure as fatal would drop a nightly for a blip. Keep polling on the |
| 170 | # last good snapshot; only give up if the deadline passes having never |
| 171 | # fetched one. |
| 172 | echo "buildkite API returned ${http_code}" |
| 173 | if [[ ! -s "${BUILD_JSON}" ]]; then |
| 174 | if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then |
| 175 | echo "no build data was ever fetched -- nothing to report" |
| 176 | exit 0 |
| 177 | fi |
| 178 | sleep "${POLL_INTERVAL_S}" |
| 179 | continue |
| 180 | fi |
| 181 | if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then |
| 182 | echo "deadline reached while the API is unavailable --" \ |
| 183 | "reporting from the last good snapshot" |
| 184 | break |
| 185 | fi |
| 186 | sleep "${POLL_INTERVAL_S}" |
| 187 | continue |
| 188 | fi |
| 189 | if [[ -z "${WAIT_DEADLINE}" ]]; then |
| 190 | WAIT_DEADLINE="$(build_deadline)" || WAIT_DEADLINE="" |
| 191 | if [[ -z "${WAIT_DEADLINE}" ]]; then |
| 192 | # No created_at to anchor to; fall back to script start so the loop |
| 193 | # still has a bound rather than running until the step times out. |
| 194 | WAIT_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S )) |
| 195 | echo "build created_at unavailable -- deadline measured from now" |
| 196 | fi |
| 197 | echo "reporting deadline: $(date -u -d "@${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null \ |
| 198 | || date -u -r "${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ')" \ |
| 199 | "(build age limit ${MAX_BUILD_AGE_S}s)" |
| 200 | fi |
| 201 | remaining="$(outstanding_jobs)" || remaining="" |
| 202 | if [[ -z "${remaining}" ]]; then |
| 203 | echo "could not parse build JSON -- reporting what we have" |
| 204 | break |
| 205 | fi |
| 206 | if (( remaining == 0 )); then |
| 207 | echo "all other jobs have finished -- reporting" |
| 208 | break |
| 209 | fi |
| 210 | if (( $(date +%s) >= WAIT_DEADLINE )); then |
| 211 | echo "still ${remaining} job(s) running at the build-age deadline --" \ |
| 212 | "reporting a partial view" |
| 213 | break |
| 214 | fi |
| 215 | echo "waiting for ${remaining} job(s) to finish..." |
| 216 | sleep "${POLL_INTERVAL_S}" |
| 217 | done |
| 218 | |
| 219 | # One callback per job, matching HUD's per-job crcr_workflow_job schema. |
| 220 | # delivery_id is synthetic: the nightly path has no upstream dispatch to borrow |
| 221 | # one from, so build+job is used as the idempotency key. |
| 222 | # Resolved from this script's location: the pipeline runs it from |
| 223 | # /vllm-workspace/tests, so a repo-relative path would not resolve. |
| 224 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 225 | |
| 226 | # Minted here, not before the poll loop: a Buildkite OIDC token defaults to a |
| 227 | # five-minute lifetime and the loop above can run to the build-age deadline. |
| 228 | # Build 91827 waited 6h48m and then took a 401 on all 360 callbacks from a |
| 229 | # token that had expired hours earlier. |
| 230 | OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)" |
| 231 | if [[ -z "${OIDC_TOKEN}" ]]; then |
| 232 | echo "could not mint a Buildkite OIDC token after the wait -- skipping report" |
| 233 | exit 0 |
| 234 | fi |
| 235 | |
| 236 | python3 "${SCRIPT_DIR}/crcr_report.py" \ |
| 237 | --build-json "${BUILD_JSON}" \ |
| 238 | --callback-url "${CALLBACK_URL}" \ |
| 239 | --oidc-token "${OIDC_TOKEN}" \ |
| 240 | || echo "crcr report failed -- continuing (reporting is best-effort)" |
| 241 | |
| 242 | exit 0 |