[CI] Add TORCH_NIGHTLY=1 build mode (run full suite on torch nightly) (#47180)
Co-authored-by: Andrey Talman <atalman@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Andrey Talman committed
Jul 10, 2026 at 04:38 UTC
f36284a8d21d3748df30c8f1b30357be711350b5
5 files changed
+82
-14
.buildkite/image_build/image_build.sh
+12
@@ -177,6 +177,18 @@ BRANCH=$4
177
IMAGE_TAG=$5
178
IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional
179
180
+# When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the
181
+# entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate
182
+# to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the
183
+# normal IMAGE_TAG that every test step already pulls -- no separate image tag,
184
+# no duplicate "vLLM Against PyTorch Nightly" pipeline section.
185
+if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
186
+ echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly"
187
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
188
+ exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \
189
+ "${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}"
190
+fi
191
+
192
# build config
193
TARGET="test-ci"
194
VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}"
.buildkite/scripts/check-ray-compatibility.sh
+5
-1
@@ -29,7 +29,11 @@ if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
29
TORCH_INDEX_URL=""
30
fi
31
else
32
- TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
32
+ if [ "${TORCH_NIGHTLY:-0}" = "1" ]; then
33
+ TORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/cu130"
34
+ else
35
+ TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
36
+ fi
37
fi
38
echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
39
.buildkite/scripts/trigger-ci-build.sh
+32
-11
@@ -23,6 +23,7 @@ NC='\033[0m' # No Color
23
# Default configuration
24
PIPELINE="ci"
25
DRY_RUN=true
26
+TORCH_NIGHTLY=false
27
28
usage() {
29
cat <<EOF
@@ -34,12 +35,14 @@ Sets RUN_ALL=1 and NIGHTLY=1 environment variables.
35
SAFETY: Dry-run by default. Use --execute to actually trigger a build.
36
37
Options:
37
- --execute Actually trigger the build (default: dry-run)
38
- --pipeline Buildkite pipeline slug (default: ${PIPELINE})
39
- --commit Override commit SHA (default: current HEAD)
40
- --branch Override branch name (default: current branch)
41
- --message Custom build message (default: auto-generated)
42
- --help Show this help message
38
+ --execute Actually trigger the build (default: dry-run)
39
+ --pipeline Buildkite pipeline slug (default: ${PIPELINE})
40
+ --commit Override commit SHA (default: current HEAD)
41
+ --branch Override branch name (default: current branch)
42
+ --message Custom build message (default: auto-generated)
43
+ --torch-nightly Also build and run the full suite against torch nightly
44
+ (sets TORCH_NIGHTLY=1)
45
+ --help Show this help message
46
47
Prerequisites:
48
- bk CLI installed: brew tap buildkite/buildkite && brew install buildkite/buildkite/bk
@@ -49,6 +52,7 @@ Examples:
52
$(basename "$0") # Dry-run, show what would happen
53
$(basename "$0") --execute # Actually trigger the build
54
$(basename "$0") --pipeline ci-shadow # Dry-run with different pipeline
55
+ $(basename "$0") --torch-nightly # Dry-run a full torch-nightly run
56
EOF
57
exit 1
58
}
@@ -96,6 +100,10 @@ while [[ $# -gt 0 ]]; do
100
MESSAGE="$2"
101
shift 2
102
;;
103
+ --torch-nightly)
104
+ TORCH_NIGHTLY=true
105
+ shift
106
+ ;;
107
--help|-h)
108
usage
109
;;
@@ -171,11 +179,17 @@ if [[ $(echo "$REMOTE_BRANCHES" | wc -l) -gt 5 ]]; then
179
fi
180
echo ""
181
182
+# Environment variables passed to the build.
183
+BUILD_ENV=("RUN_ALL=1" "NIGHTLY=1")
184
+if [[ "$TORCH_NIGHTLY" == true ]]; then
185
+ BUILD_ENV+=("TORCH_NIGHTLY=1")
186
+fi
187
+
188
log_info "Pipeline: ${PIPELINE}"
189
log_info "Branch: ${BRANCH}"
190
log_info "Commit: ${COMMIT}"
191
log_info "Message: ${MESSAGE}"
178
-log_info "Environment: RUN_ALL=1, NIGHTLY=1"
192
+log_info "Environment: ${BUILD_ENV[*]}"
193
echo ""
194
195
# Build the command
@@ -187,9 +201,10 @@ CMD=(bk build create
201
--commit "${COMMIT}"
202
--branch "${BRANCH}"
203
--message "${MESSAGE}"
190
- --env "RUN_ALL=1"
191
- --env "NIGHTLY=1"
204
)
205
+for env_var in "${BUILD_ENV[@]}"; do
206
+ CMD+=(--env "${env_var}")
207
+done
208
209
if [[ "$DRY_RUN" == true ]]; then
210
echo "=========================================="
@@ -210,8 +225,14 @@ if [[ "$DRY_RUN" == true ]]; then
225
echo " --commit '$(escape_for_shell "${COMMIT}")' \\"
226
echo " --branch '$(escape_for_shell "${BRANCH}")' \\"
227
echo " --message '$(escape_for_shell "${MESSAGE}")' \\"
213
- echo " --env 'RUN_ALL=1' \\"
214
- echo " --env 'NIGHTLY=1'"
228
+ last_idx=$(( ${#BUILD_ENV[@]} - 1 ))
229
+ for i in "${!BUILD_ENV[@]}"; do
230
+ if [[ $i -eq $last_idx ]]; then
231
+ echo " --env '$(escape_for_shell "${BUILD_ENV[$i]}")'"
232
+ else
233
+ echo " --env '$(escape_for_shell "${BUILD_ENV[$i]}")' \\"
234
+ fi
235
+ done
236
echo ""
237
echo "=========================================="
238
echo -e "${YELLOW}To actually trigger this build, run:${NC}"
CMakeLists.txt
+11
-2
@@ -70,6 +70,15 @@ endif()
70
#
71
set(TORCH_SUPPORTED_VERSION_CUDA "2.11.0")
72
set(TORCH_SUPPORTED_VERSION_ROCM "2.11.0")
73
+# TORCH_NIGHTLY=1 builds run against unpinned nightly wheels, so the supported-
74
+# version check would always warn. Only treat it as a nightly build when the
75
+# value is exactly "1" (the bootstrap exports TORCH_NIGHTLY=0 by default, which
76
+# must NOT suppress the warning for normal builds).
77
+if (DEFINED ENV{TORCH_NIGHTLY} AND "$ENV{TORCH_NIGHTLY}" STREQUAL "1")
78
+ set(TORCH_NIGHTLY_BUILD TRUE)
79
+else()
80
+ set(TORCH_NIGHTLY_BUILD FALSE)
81
+endif()
82
83
#
84
# Try to find python package with an executable that exactly matches
@@ -175,7 +184,7 @@ endif()
184
if (NOT HIP_FOUND AND NOT PYTORCH_FOUND_HIP AND CUDA_FOUND)
185
set(VLLM_GPU_LANG "CUDA")
186
178
- if (NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_CUDA})
187
+ if (NOT TORCH_NIGHTLY_BUILD AND NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_CUDA})
188
message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_CUDA} "
189
"expected for CUDA build, saw ${Torch_VERSION} instead.")
190
endif()
@@ -188,7 +197,7 @@ elseif(HIP_FOUND OR PYTORCH_FOUND_HIP)
197
enable_language(HIP)
198
199
# ROCm 5.X and 6.X
191
- if (ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
200
+ if (NOT TORCH_NIGHTLY_BUILD AND ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
201
Torch_VERSION VERSION_LESS ${TORCH_SUPPORTED_VERSION_ROCM})
202
message(WARNING "Pytorch version >= ${TORCH_SUPPORTED_VERSION_ROCM} "
203
"expected for ROCm build, saw ${Torch_VERSION} instead.")
docs/contributing/ci/update_pytorch_version.md
+22
@@ -93,6 +93,28 @@ To address this, manually trigger a build on Buildkite to accomplish two objecti
93
<img width="60%" alt="Buildkite new build popup" src="https://github.com/user-attachments/assets/3b07f71b-bb18-4ca3-aeaf-da0fe79d315f" />
94
</p>
95
96
+You can also trigger this build from the command line with
97
+[`.buildkite/scripts/trigger-ci-build.sh`](../../../.buildkite/scripts/trigger-ci-build.sh)
98
+(dry-run by default; pass `--execute` to actually trigger it).
99
+
100
+## Test against PyTorch nightly
101
+
102
+The steps above test against a specific PyTorch RC/stable wheel pinned in the
103
+requirements files. To instead build and run the CI suite against the latest
104
+PyTorch **nightly** wheels, set the `TORCH_NIGHTLY=1` environment variable on
105
+the build (or apply the `ready-torch-nightly` label to the PR).
106
+
107
+When `TORCH_NIGHTLY=1`, the base CI image is built against PyTorch nightly
108
+(`image_build_torch_nightly.sh`, `PYTORCH_NIGHTLY=1`, CUDA 13.0) and tagged at
109
+the normal image tag, so the entire existing pipeline runs on nightly torch --
110
+there is no separate pipeline section to trigger. Combine it with `RUN_ALL=1`
111
+to run the full suite (the `ready-torch-nightly` label and
112
+`trigger-ci-build.sh --torch-nightly` both set this for you). This is the
113
+configuration to use for a scheduled "vLLM vs PyTorch nightly" run.
114
+
115
+Use `.buildkite/scripts/trigger-ci-build.sh --torch-nightly` to trigger it from
116
+the command line.
117
+
118
## Update all the different vLLM platforms
119
120
Rather than attempting to update all vLLM platforms in a single pull request, it's more manageable