| 1 | group: CRCR Report |
| 2 | # Only image-build-cpu: this step must always run. |
| 3 | # |
| 4 | # It was briefly gated on the build's long pole (lm-eval-large-models-4xh100) |
| 5 | # so the agent would not idle. That backfired on the first nightly to use it, |
| 6 | # build 91191: the long pole OOMed, hung, and was cancelled -- and a cancelled |
| 7 | # dependency propagates past `allow_dependency_failure`, so the step went |
| 8 | # `waiting_failed` and that nightly got no CRCR row at all. Nothing in this |
| 9 | # pipeline sets timeout_in_minutes (the pipeline default and org maximum are |
| 10 | # both unset), so a hung job has no terminal state to reach either: gating on |
| 11 | # any single job can suppress the report indefinitely. |
| 12 | # |
| 13 | # The bound now lives in the script, measured from the build's created_at, so |
| 14 | # the report lands within CRCR_MAX_BUILD_AGE_S of the build starting whatever |
| 15 | # else happens. |
| 16 | depends_on: |
| 17 | - image-build-cpu |
| 18 | steps: |
| 19 | - label: ":satellite: Report nightly results to CRCR" |
| 20 | key: crcr-report |
| 21 | if: build.env("TORCH_NIGHTLY") == "1" |
| 22 | soft_fail: true |
| 23 | allow_dependency_failure: true |
| 24 | # The script polls from the start of the build now, so this covers the whole |
| 25 | # wait. It sits above the script's own build-age deadline (7h) so the script |
| 26 | # sends its partial report rather than being killed with nothing sent. |
| 27 | timeout_in_minutes: 480 |
| 28 | device: cpu-small |
| 29 | commands: |
| 30 | - bash /vllm-workspace/.buildkite/scripts/crcr-report.sh |