| 1 | #!/bin/bash |
| 2 | set -euo pipefail |
| 3 | |
| 4 | # replace invalid characters in Docker image tags and truncate to 128 chars |
| 5 | clean_docker_tag() { |
| 6 | local input="$1" |
| 7 | echo "$input" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128 |
| 8 | } |
| 9 | |
| 10 | print_usage_and_exit() { |
| 11 | echo "Usage: $0 <registry> <repo> <commit> <branch> <image_tag> [<image_tag_latest>]" |
| 12 | exit 1 |
| 13 | } |
| 14 | |
| 15 | print_instance_info() { |
| 16 | echo "" |
| 17 | echo "=== Debug: Instance Information ===" |
| 18 | # Get IMDSv2 token |
| 19 | if TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \ |
| 20 | -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" 2>/dev/null); then |
| 21 | AMI_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ |
| 22 | http://169.254.169.254/latest/meta-data/ami-id 2>/dev/null || echo "unknown") |
| 23 | INSTANCE_TYPE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ |
| 24 | http://169.254.169.254/latest/meta-data/instance-type 2>/dev/null || echo "unknown") |
| 25 | INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ |
| 26 | http://169.254.169.254/latest/meta-data/instance-id 2>/dev/null || echo "unknown") |
| 27 | AZ=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ |
| 28 | http://169.254.169.254/latest/meta-data/placement/availability-zone 2>/dev/null || echo "unknown") |
| 29 | echo "AMI ID: ${AMI_ID}" |
| 30 | echo "Instance Type: ${INSTANCE_TYPE}" |
| 31 | echo "Instance ID: ${INSTANCE_ID}" |
| 32 | echo "AZ: ${AZ}" |
| 33 | else |
| 34 | echo "Not running on EC2 or IMDS not available" |
| 35 | fi |
| 36 | # Check for warm cache AMI (marker file baked into custom AMI) |
| 37 | if [[ -f /etc/vllm-ami-info ]]; then |
| 38 | echo "Cache: warm (custom vLLM AMI)" |
| 39 | cat /etc/vllm-ami-info |
| 40 | else |
| 41 | echo "Cache: cold (standard AMI)" |
| 42 | fi |
| 43 | echo "===================================" |
| 44 | echo "" |
| 45 | } |
| 46 | |
| 47 | setup_buildx_builder() { |
| 48 | echo "--- :buildkite: Setting up buildx builder" |
| 49 | if [[ -S "${BUILDKIT_SOCKET}" ]]; then |
| 50 | # Custom AMI with standalone buildkitd - use remote driver for warm cache |
| 51 | echo "✅ Found local buildkitd socket at ${BUILDKIT_SOCKET}" |
| 52 | echo "Using remote driver to connect to buildkitd (warm cache available)" |
| 53 | if docker buildx inspect baked-vllm-builder >/dev/null 2>&1; then |
| 54 | echo "Using existing baked-vllm-builder" |
| 55 | docker buildx use baked-vllm-builder |
| 56 | else |
| 57 | echo "Creating baked-vllm-builder with remote driver" |
| 58 | docker buildx create \ |
| 59 | --name baked-vllm-builder \ |
| 60 | --driver remote \ |
| 61 | --use \ |
| 62 | "unix://${BUILDKIT_SOCKET}" |
| 63 | fi |
| 64 | docker buildx inspect --bootstrap |
| 65 | elif docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then |
| 66 | # Existing builder available |
| 67 | echo "Using existing builder: ${BUILDER_NAME}" |
| 68 | docker buildx use "${BUILDER_NAME}" |
| 69 | docker buildx inspect --bootstrap |
| 70 | else |
| 71 | # No local buildkitd, no existing builder - create new docker-container builder |
| 72 | echo "No local buildkitd found, using docker-container driver" |
| 73 | docker buildx create --name "${BUILDER_NAME}" --driver docker-container --use |
| 74 | docker buildx inspect --bootstrap |
| 75 | fi |
| 76 | |
| 77 | # builder info |
| 78 | echo "Active builder:" |
| 79 | docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls |
| 80 | } |
| 81 | |
| 82 | download_ci_hcl() { |
| 83 | # Returns non-zero instead of exiting: the optional export path must be |
| 84 | # able to give up without failing a build whose image already exists. |
| 85 | echo "--- :arrow_down: Downloading ci.hcl" |
| 86 | curl -sSfL -o "${CI_HCL_PATH}" "${CI_HCL_URL}" || return $? |
| 87 | echo "Downloaded to ${CI_HCL_PATH}" |
| 88 | |
| 89 | if [[ ! -f "${CI_HCL_PATH}" ]]; then |
| 90 | echo "Error: ci.hcl not found at ${CI_HCL_PATH}" |
| 91 | return 1 |
| 92 | fi |
| 93 | } |
| 94 | |
| 95 | export_kernel_symbol_map_from_cache() { |
| 96 | # The existing-image path: everything here is best effort. Called as |
| 97 | # `... || echo`, so errexit is suspended inside and any failure just |
| 98 | # returns; the build keeps the image it already has. |
| 99 | local rc=0 |
| 100 | download_ci_hcl \ |
| 101 | && setup_buildx_builder \ |
| 102 | && resolve_parent_commit \ |
| 103 | && export PARENT_COMMIT \ |
| 104 | && BUILD_TMP_DIR="$(mktemp -d)" \ |
| 105 | && export_kernel_symbol_map || rc=$? |
| 106 | rm -rf -- "${BUILD_TMP_DIR:-}" |
| 107 | return "${rc}" |
| 108 | } |
| 109 | |
| 110 | export_kernel_symbol_map() { |
| 111 | # Only when the build asked for it (nightly/post-merge). The map was |
| 112 | # produced inside the csrc-build stage during the main bake; this second |
| 113 | # bake of the scratch stage is a cache hit that just writes the file out. |
| 114 | if [[ "${VLLM_KERNEL_SYMBOL_MAP:-0}" != "1" ]]; then |
| 115 | return 0 |
| 116 | fi |
| 117 | echo "--- :world_map: Exporting kernel symbol map" |
| 118 | local out_dir="${BUILD_TMP_DIR}/kernel-symbol-map" |
| 119 | mkdir -p "${out_dir}" |
| 120 | if docker buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain \ |
| 121 | --set "kernel-symbol-map.output=type=local,dest=${out_dir}" kernel-symbol-map \ |
| 122 | && [[ -s "${out_dir}/kernel_symbol_map.json.gz" ]]; then |
| 123 | ls -la "${out_dir}" |
| 124 | (cd "${out_dir}" && buildkite-agent artifact upload "kernel_symbol_map.json.gz") |
| 125 | else |
| 126 | echo "kernel symbol map export failed; continuing without it" >&2 |
| 127 | fi |
| 128 | } |
| 129 | |
| 130 | annotate_image_tags() { |
| 131 | .buildkite/scripts/annotate-image-build.sh \ |
| 132 | "${IMAGE_TAG:-}" "${IMAGE_TAG_LATEST:-}" |
| 133 | } |
| 134 | |
| 135 | check_and_skip_if_image_exists() { |
| 136 | if [[ -n "${IMAGE_TAG:-}" ]]; then |
| 137 | echo "--- :mag: Checking if image exists" |
| 138 | if docker manifest inspect "${IMAGE_TAG}" >/dev/null 2>&1; then |
| 139 | echo "Image already exists: ${IMAGE_TAG}" |
| 140 | echo "Skipping build" |
| 141 | if [[ "${VLLM_KERNEL_SYMBOL_MAP:-0}" == "1" ]]; then |
| 142 | # The image is reused, but the symbol map is an artifact of |
| 143 | # this build. Bake just the export target: its layers come |
| 144 | # from the registry cache, and the map step itself reruns |
| 145 | # if the cached image was built without the arg. Never |
| 146 | # fails the build: the image is already there. |
| 147 | export_kernel_symbol_map_from_cache \ |
| 148 | || echo "kernel symbol map: export from the cached image failed; continuing" >&2 |
| 149 | fi |
| 150 | annotate_image_tags |
| 151 | exit 0 |
| 152 | fi |
| 153 | echo "Image not found, proceeding with build" |
| 154 | fi |
| 155 | } |
| 156 | |
| 157 | ecr_login() { |
| 158 | aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true |
| 159 | aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true |
| 160 | } |
| 161 | |
| 162 | prepare_cache_tags() { |
| 163 | # resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN |
| 164 | TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache" |
| 165 | MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache" |
| 166 | |
| 167 | if [[ "$BUILDKITE_PULL_REQUEST" == "false" ]]; then |
| 168 | if [[ "$BUILDKITE_BRANCH" == "main" ]]; then |
| 169 | cache="${MAIN_CACHE_ECR}:latest" |
| 170 | else |
| 171 | clean_branch=$(clean_docker_tag "$BUILDKITE_BRANCH") |
| 172 | cache="${TEST_CACHE_ECR}:${clean_branch}" |
| 173 | fi |
| 174 | CACHE_TO="$cache" |
| 175 | CACHE_FROM="$cache" |
| 176 | CACHE_FROM_BASE_BRANCH="$cache" |
| 177 | else |
| 178 | CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}" |
| 179 | CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}" |
| 180 | if [[ "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" == "main" ]]; then |
| 181 | CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest" |
| 182 | else |
| 183 | clean_base=$(clean_docker_tag "$BUILDKITE_PULL_REQUEST_BASE_BRANCH") |
| 184 | CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}" |
| 185 | fi |
| 186 | fi |
| 187 | |
| 188 | CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest" |
| 189 | export CACHE_TO CACHE_FROM CACHE_FROM_BASE_BRANCH CACHE_FROM_MAIN |
| 190 | } |
| 191 | |
| 192 | resolve_parent_commit() { |
| 193 | if [[ -z "${PARENT_COMMIT:-}" ]]; then |
| 194 | PARENT_COMMIT=$(git rev-parse HEAD~1 2>/dev/null || echo "") |
| 195 | if [[ -n "${PARENT_COMMIT}" ]]; then |
| 196 | echo "Computed parent commit for cache fallback: ${PARENT_COMMIT}" |
| 197 | export PARENT_COMMIT |
| 198 | else |
| 199 | echo "Could not determine parent commit (may be first commit in repo)" |
| 200 | fi |
| 201 | else |
| 202 | echo "Using provided PARENT_COMMIT: ${PARENT_COMMIT}" |
| 203 | fi |
| 204 | } |
| 205 | |
| 206 | print_bake_config() { |
| 207 | echo "--- :page_facing_up: Resolved bake configuration" |
| 208 | # Write to a temp directory to avoid polluting the repo root (which is the |
| 209 | # Docker build context). Files left in the repo root get COPY'd into the |
| 210 | # image and can cause duplicate artifact uploads from downstream steps. |
| 211 | local bake_tmp |
| 212 | bake_tmp="$(mktemp -d)" |
| 213 | BAKE_CONFIG_FILE="${bake_tmp}/bake-config-build-${BUILDKITE_BUILD_NUMBER:-local}.json" |
| 214 | docker buildx bake "${BAKE_FILES[@]}" --print "${TARGET}" | tee "${BAKE_CONFIG_FILE}" || true |
| 215 | echo "Saved bake config to ${BAKE_CONFIG_FILE}" |
| 216 | echo "--- :arrow_down: Uploading bake config to Buildkite" |
| 217 | (cd "$(dirname "${BAKE_CONFIG_FILE}")" && buildkite-agent artifact upload "$(basename "${BAKE_CONFIG_FILE}")") |
| 218 | } |
| 219 | |
| 220 | record_buildkit_trace() ( |
| 221 | local metadata_file="$1" |
| 222 | local helper=".buildkite/scripts/ci-otel/ci_otel.py" |
| 223 | local trace_file="${BUILD_TMP_DIR}/buildkit-trace.json" |
| 224 | local build_ref record_id |
| 225 | |
| 226 | if [[ "${BUILDKITE:-}" != "true" || ! -f "${helper}" ]] || |
| 227 | ! command -v timeout >/dev/null 2>&1 || |
| 228 | ! command -v python3 >/dev/null 2>&1; then |
| 229 | return 0 |
| 230 | fi |
| 231 | build_ref="$(timeout 3s python3 "${helper}" build-ref "${metadata_file}" "${TARGET}" 2>/dev/null)" || { |
| 232 | echo "vLLM CI OTel: BuildKit metadata unavailable; trace skipped" >&2 |
| 233 | return 0 |
| 234 | } |
| 235 | record_id="${build_ref##*/}" |
| 236 | if [[ -z "${record_id}" ]]; then |
| 237 | return 0 |
| 238 | fi |
| 239 | |
| 240 | echo "--- :mag: Recording BuildKit trace" |
| 241 | if ! timeout 20s docker buildx history trace "${record_id}" >"${trace_file}"; then |
| 242 | echo "vLLM CI OTel: BuildKit history trace unavailable; upload skipped" >&2 |
| 243 | return 0 |
| 244 | fi |
| 245 | |
| 246 | export CI_INFRA_BUILDX_REF="${build_ref}" |
| 247 | export CI_INFRA_OTEL_SPOOL_DIR="${BUILD_TMP_DIR}/otel-spans" |
| 248 | timeout 5s python3 "${helper}" record-buildkit "${trace_file}" || { |
| 249 | echo "vLLM CI OTel: BuildKit trace conversion skipped" >&2 |
| 250 | return 0 |
| 251 | } |
| 252 | timeout 5s python3 "${helper}" flush || |
| 253 | echo "vLLM CI OTel: BuildKit trace upload skipped" >&2 |
| 254 | ) |
| 255 | |
| 256 | ################################# |
| 257 | # Main Script # |
| 258 | ################################# |
| 259 | print_instance_info |
| 260 | |
| 261 | if [[ $# -lt 5 ]]; then |
| 262 | print_usage_and_exit |
| 263 | fi |
| 264 | |
| 265 | # input args |
| 266 | REGISTRY=$1 |
| 267 | REPO=$2 |
| 268 | BUILDKITE_COMMIT=$3 |
| 269 | BRANCH=$4 |
| 270 | IMAGE_TAG=$5 |
| 271 | IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional |
| 272 | |
| 273 | # When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the |
| 274 | # entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate |
| 275 | # to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the |
| 276 | # normal IMAGE_TAG that every test step already pulls -- no separate image tag, |
| 277 | # no duplicate "vLLM Against PyTorch Nightly" pipeline section. |
| 278 | if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then |
| 279 | echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly" |
| 280 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 281 | exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \ |
| 282 | "${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}" |
| 283 | fi |
| 284 | |
| 285 | # build config |
| 286 | TARGET="test-ci" |
| 287 | VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}" |
| 288 | BUILDER_NAME="${BUILDER_NAME:-vllm-builder}" |
| 289 | CI_HCL_URL="${CI_HCL_URL:-https://raw.githubusercontent.com/vllm-project/ci-infra/main/docker/ci.hcl}" |
| 290 | CI_HCL_PATH="${CI_HCL_PATH:-/tmp/ci.hcl}" |
| 291 | ZSTD_HCL_PATH="${ZSTD_HCL_PATH:-.buildkite/image_build/zstd.hcl}" |
| 292 | BUILDKIT_SOCKET="/run/buildkit/buildkitd.sock" |
| 293 | BAKE_FILES=(-f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" -f "${ZSTD_HCL_PATH}") |
| 294 | |
| 295 | prepare_cache_tags |
| 296 | ecr_login |
| 297 | |
| 298 | # Environment info (for docs and human readers) |
| 299 | # VLLM_CI_BRANCH - ci-infra branch to use (default: main) |
| 300 | # VLLM_BAKE_FILE_PATH - Path to vLLM's bake file (default: docker/docker-bake.hcl) |
| 301 | # BUILDER_NAME - Name for buildx builder (default: vllm-builder) |
| 302 | # |
| 303 | # Build configuration (exported as environment variables for bake): |
| 304 | export BUILDKITE_COMMIT |
| 305 | export PARENT_COMMIT |
| 306 | export IMAGE_TAG |
| 307 | export IMAGE_TAG_LATEST |
| 308 | export COMMIT="${COMMIT:-${BUILDKITE_COMMIT}}" |
| 309 | export CACHE_FROM |
| 310 | export CACHE_FROM_BASE_BRANCH |
| 311 | export CACHE_FROM_MAIN |
| 312 | export CACHE_TO |
| 313 | |
| 314 | # print args |
| 315 | echo "--- :mag: Arguments" |
| 316 | echo "REGISTRY: ${REGISTRY}" |
| 317 | echo "REPO: ${REPO}" |
| 318 | echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}" |
| 319 | echo "BRANCH: ${BRANCH}" |
| 320 | echo "IMAGE_TAG: ${IMAGE_TAG}" |
| 321 | echo "IMAGE_TAG_LATEST: ${IMAGE_TAG_LATEST}" |
| 322 | |
| 323 | # print build configuration |
| 324 | echo "--- :mag: Build configuration" |
| 325 | echo "TARGET: ${TARGET}" |
| 326 | echo "vLLM bake file: ${VLLM_BAKE_FILE_PATH}" |
| 327 | echo "BUILDER_NAME: ${BUILDER_NAME}" |
| 328 | echo "CI_HCL_URL: ${CI_HCL_URL}" |
| 329 | echo "BUILDKIT_SOCKET: ${BUILDKIT_SOCKET}" |
| 330 | |
| 331 | echo "--- :mag: Cache tags" |
| 332 | echo "CACHE_TO: ${CACHE_TO}" |
| 333 | echo "CACHE_FROM: ${CACHE_FROM}" |
| 334 | echo "CACHE_FROM_BASE_BRANCH: ${CACHE_FROM_BASE_BRANCH}" |
| 335 | echo "CACHE_FROM_MAIN: ${CACHE_FROM_MAIN}" |
| 336 | |
| 337 | check_and_skip_if_image_exists |
| 338 | |
| 339 | echo "--- :docker: Setting up Docker buildx bake" |
| 340 | echo "Target: ${TARGET}" |
| 341 | echo "vLLM bake file: ${VLLM_BAKE_FILE_PATH}" |
| 342 | echo "CI HCL path: ${CI_HCL_PATH}" |
| 343 | |
| 344 | if [[ ! -f "${VLLM_BAKE_FILE_PATH}" ]]; then |
| 345 | echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE_PATH}" |
| 346 | echo "Make sure you're running from the vLLM repository root" |
| 347 | exit 1 |
| 348 | fi |
| 349 | |
| 350 | download_ci_hcl || exit 1 |
| 351 | |
| 352 | setup_buildx_builder |
| 353 | |
| 354 | resolve_parent_commit |
| 355 | export PARENT_COMMIT |
| 356 | |
| 357 | print_bake_config |
| 358 | |
| 359 | echo "--- :docker: Building ${TARGET}" |
| 360 | BUILD_TMP_DIR="$(mktemp -d)" |
| 361 | trap 'rm -rf -- "${BUILD_TMP_DIR}"' EXIT |
| 362 | BUILD_METADATA_FILE="${BUILD_TMP_DIR}/build-metadata.json" |
| 363 | BUILD_STATUS=0 |
| 364 | docker --debug buildx bake "${BAKE_FILES[@]}" \ |
| 365 | --progress plain --metadata-file "${BUILD_METADATA_FILE}" "${TARGET}" || BUILD_STATUS=$? |
| 366 | |
| 367 | record_buildkit_trace "${BUILD_METADATA_FILE}" || true |
| 368 | |
| 369 | if [[ "${BUILD_STATUS}" -ne 0 ]]; then |
| 370 | exit "${BUILD_STATUS}" |
| 371 | fi |
| 372 | |
| 373 | echo "--- :white_check_mark: Build complete" |
| 374 | export_kernel_symbol_map || true |
| 375 | |
| 376 | annotate_image_tags |