A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 23:30 UTC
[Releases] [ROCm] Enable Nightly Docker Image and Wheel Releases for ROCm (#37283)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: Hongxia Yang <hongxiay.yang@amd.com>
TJian committed
Mar 27, 2026 at 00:32 UTC
60af7b967bf52aa9623aa4f224a0d281757ba9b0
6 files changed
+247
-249
.buildkite/release-pipeline.yaml
+158
-219
@@ -342,184 +342,143 @@ steps:
342
# To build a specific version, trigger the build from that branch/tag.
343
#
344
# Environment variables for ROCm builds (set via Buildkite UI or schedule):
345
- # ROCM_PYTHON_VERSION: Python version (default: 3.12)
346
- # PYTORCH_ROCM_ARCH: GPU architectures (default: gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151)
347
- # ROCM_UPLOAD_WHEELS: Upload to S3 (default: false for nightly, true for releases)
348
- # ROCM_FORCE_REBUILD: Force rebuild base wheels, ignore S3 cache (default: false)
345
#
346
# Note: ROCm version is determined by BASE_IMAGE in docker/Dockerfile.rocm_base
351
- # (currently rocm/dev-ubuntu-22.04:7.1-complete)
347
#
348
# =============================================================================
349
355
- # ROCm Input Step - Collect build configuration (manual trigger only)
356
- - input: "ROCm Wheel Release Build Configuration"
357
- key: input-rocm-config
358
- depends_on: ~
359
- if: build.source == "ui"
360
- fields:
361
- - text: "Python Version"
362
- key: "rocm-python-version"
363
- default: "3.12"
364
- hint: "Python version (e.g., 3.12)"
365
- - text: "GPU Architectures"
366
- key: "rocm-pytorch-rocm-arch"
367
- default: "gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151"
368
- hint: "Semicolon-separated GPU architectures"
369
- - select: "Upload Wheels to S3"
370
- key: "rocm-upload-wheels"
371
- default: "true"
372
- options:
373
- - label: "No - Build only (nightly/dev)"
374
- value: "false"
375
- - label: "Yes - Upload to S3 (release)"
376
- value: "true"
377
- - select: "Force Rebuild Base Wheels"
378
- key: "rocm-force-rebuild"
379
- default: "false"
380
- hint: "Ignore S3 cache and rebuild base wheels from scratch"
381
- options:
382
- - label: "No - Use cached wheels if available"
383
- value: "false"
384
- - label: "Yes - Rebuild even if cache exists"
385
- value: "true"
386
-
350
# ROCm Job 1: Build ROCm Base Wheels (with S3 caching)
388
- - label: ":rocm: Build ROCm Base Wheels"
351
+ - label: ":rocm: Build ROCm Base Image & Wheels"
352
id: build-rocm-base-wheels
390
- depends_on:
391
- - step: input-rocm-config
392
- allow_failure: true # Allow failure so non-UI builds can proceed (input step is skipped)
353
+ depends_on: ~
354
agents:
355
queue: cpu_queue_release
356
commands:
396
- # Set configuration and check cache
357
- |
358
set -euo pipefail
359
400
- # Get values from meta-data (set by input step) or use defaults
401
- PYTHON_VERSION="$$(buildkite-agent meta-data get rocm-python-version 2>/dev/null || echo '')"
402
- export PYTHON_VERSION="$${PYTHON_VERSION:-3.12}"
403
-
404
- PYTORCH_ROCM_ARCH="$$(buildkite-agent meta-data get rocm-pytorch-rocm-arch 2>/dev/null || echo '')"
405
- export PYTORCH_ROCM_ARCH="$${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151}"
406
-
407
- # Check for force rebuild flag
408
- ROCM_FORCE_REBUILD="$${ROCM_FORCE_REBUILD:-}"
409
- if [ -z "$${ROCM_FORCE_REBUILD}" ]; then
410
- ROCM_FORCE_REBUILD="$$(buildkite-agent meta-data get rocm-force-rebuild 2>/dev/null || echo '')"
411
- fi
360
+ # Generate cache key
361
+ CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
362
+ ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base"
363
364
echo "========================================"
414
- echo "ROCm Base Wheels Build Configuration"
365
+ echo "ROCm Base Build Configuration"
366
echo "========================================"
416
- echo " PYTHON_VERSION: $${PYTHON_VERSION}"
417
- echo " PYTORCH_ROCM_ARCH: $${PYTORCH_ROCM_ARCH}"
418
- echo " ROCM_FORCE_REBUILD: $${ROCM_FORCE_REBUILD:-false}"
367
+ echo " CACHE_KEY: $${CACHE_KEY}"
368
+ echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
369
echo "========================================"
370
+
371
+ # Login to ECR
372
+ aws ecr-public get-login-password --region us-east-1 | \
373
+ docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
374
+
375
+ IMAGE_EXISTS=false
376
+ WHEELS_EXIST=false
377
+
378
+ # Check ECR for Docker image
379
421
- # Save resolved config for later jobs
422
- buildkite-agent meta-data set "rocm-python-version" "$${PYTHON_VERSION}"
423
- buildkite-agent meta-data set "rocm-pytorch-rocm-arch" "$${PYTORCH_ROCM_ARCH}"
424
-
425
- # Check S3 cache for pre-built wheels
426
- CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
427
- CACHE_PATH=$$(.buildkite/scripts/cache-rocm-base-wheels.sh path)
428
- echo ""
429
- echo "Cache key: $${CACHE_KEY}"
430
- echo "Cache path: $${CACHE_PATH}"
431
-
432
- # Save cache key for downstream jobs
433
- buildkite-agent meta-data set "rocm-cache-key" "$${CACHE_KEY}"
434
-
435
- CACHE_STATUS="miss"
436
- if [ "$${ROCM_FORCE_REBUILD}" != "true" ]; then
437
- CACHE_STATUS=$$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
438
- else
439
- echo "Force rebuild requested, skipping cache check"
380
+ if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
381
+ IMAGE_EXISTS=true
382
+ echo "ECR image cache HIT"
383
+ fi
384
+
385
+ # Check S3 for wheels
386
+ WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
387
+ if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then
388
+ WHEELS_EXIST=true
389
+ echo "S3 wheels cache HIT"
390
fi
391
442
- if [ "$${CACHE_STATUS}" = "hit" ]; then
392
+
393
+ # Scenario 1: Both cached (best case)
394
+ if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then
395
echo ""
444
- echo "CACHE HIT! Downloading pre-built wheels..."
396
+ echo "FULL CACHE HIT - Reusing both image and wheels"
397
echo ""
446
- .buildkite/scripts/cache-rocm-base-wheels.sh download
447
-
448
- # Set the S3 path for the cached Docker image (for Job 2 to download)
449
- S3_ARTIFACT_PATH="s3://$${S3_BUCKET}/rocm/cache/$${CACHE_KEY}"
450
- buildkite-agent meta-data set "rocm-docker-image-s3-path" "$${S3_ARTIFACT_PATH}/rocm-base-image.tar.gz"
451
-
452
- # Mark that we used cache (for Docker image handling)
453
- buildkite-agent meta-data set "rocm-used-cache" "true"
398
399
+ # Download wheels
400
+ .buildkite/scripts/cache-rocm-base-wheels.sh download
401
+
402
+ # Save ECR tag for downstream jobs
403
+ buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
404
+
405
+ # Scenario 2: Image cached but wheels missing
406
+ elif [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "false" ]; then
407
+ echo ""
408
+ echo "PARTIAL CACHE - Image exists, extracting wheels..."
409
echo ""
456
- echo "Cache download complete. Skipping Docker build."
457
- echo "Docker image will be downloaded from: $${S3_ARTIFACT_PATH}/rocm-base-image.tar.gz"
410
+
411
+ docker pull "$${ECR_CACHE_TAG}"
412
+
413
+ # Rebuild wheel extraction stage
414
+ DOCKER_BUILDKIT=1 docker buildx build \
415
+ --file docker/Dockerfile.rocm_base \
416
+ --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
417
+ --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
418
+ --target debs_wheel_release \
419
+ --build-arg USE_SCCACHE=1 \
420
+ --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
421
+ --build-arg SCCACHE_REGION_NAME=us-west-2 \
422
+ --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
423
+ --load \
424
+ .
425
+
426
+ # Extract and upload wheels
427
+ mkdir -p artifacts/rocm-base-wheels
428
+ cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
429
+ docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
430
+ docker rm $${cid}
431
+
432
+ .buildkite/scripts/cache-rocm-base-wheels.sh upload
433
+
434
+ buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
435
+
436
+ # Scenario 3: Full rebuild needed
437
else
438
echo ""
460
- echo "CACHE MISS. Building from scratch..."
439
+ echo " CACHE MISS - Building from scratch..."
440
echo ""
462
-
463
- # Build full base image (for later vLLM build)
441
+
442
+ # Build full base image and push to ECR
443
DOCKER_BUILDKIT=1 docker buildx build \
444
--file docker/Dockerfile.rocm_base \
466
- --tag rocm/vllm-dev:base-$${BUILDKITE_BUILD_NUMBER} \
467
- --build-arg PYTORCH_ROCM_ARCH="$${PYTORCH_ROCM_ARCH}" \
468
- --build-arg PYTHON_VERSION="$${PYTHON_VERSION}" \
445
+ --tag "$${ECR_CACHE_TAG}" \
446
--build-arg USE_SCCACHE=1 \
447
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
448
--build-arg SCCACHE_REGION_NAME=us-west-2 \
449
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
473
- --load \
450
+ --push \
451
.
475
-
476
- # Build debs_wheel_release stage for wheel extraction
452
+
453
+ # Build wheel extraction stage
454
DOCKER_BUILDKIT=1 docker buildx build \
455
--file docker/Dockerfile.rocm_base \
479
- --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
456
+ --tag rocm-base-debs:${BUILDKITE_BUILD_NUMBER} \
457
--target debs_wheel_release \
481
- --build-arg PYTORCH_ROCM_ARCH="$${PYTORCH_ROCM_ARCH}" \
482
- --build-arg PYTHON_VERSION="$${PYTHON_VERSION}" \
458
--build-arg USE_SCCACHE=1 \
459
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
460
--build-arg SCCACHE_REGION_NAME=us-west-2 \
461
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
462
--load \
463
.
489
-
490
- # Extract wheels from Docker image
464
+
465
+ # Extract and upload wheels
466
mkdir -p artifacts/rocm-base-wheels
492
- container_id=$$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
493
- docker cp $${container_id}:/app/debs/. artifacts/rocm-base-wheels/
494
- docker rm $${container_id}
495
- echo "Extracted base wheels:"
496
- ls -lh artifacts/rocm-base-wheels/
497
-
498
- # Upload wheels to S3 cache for future builds
499
- echo ""
500
- echo "Uploading wheels to S3 cache..."
467
+ cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
468
+ docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
469
+ docker rm $${cid}
470
+
471
.buildkite/scripts/cache-rocm-base-wheels.sh upload
472
503
- # Export base Docker image for reuse in vLLM build
504
- mkdir -p artifacts/rocm-docker-image
505
- docker save rocm/vllm-dev:base-$${BUILDKITE_BUILD_NUMBER} | gzip > artifacts/rocm-docker-image/rocm-base-image.tar.gz
506
- echo "Docker image size:"
507
- ls -lh artifacts/rocm-docker-image/
508
-
509
- # Upload large Docker image to S3 (also cached by cache key)
510
- S3_ARTIFACT_PATH="s3://$${S3_BUCKET}/rocm/cache/$${CACHE_KEY}"
511
- echo "Uploading Docker image to $${S3_ARTIFACT_PATH}/"
512
- aws s3 cp artifacts/rocm-docker-image/rocm-base-image.tar.gz "$${S3_ARTIFACT_PATH}/rocm-base-image.tar.gz"
513
-
514
- # Save the S3 path for downstream jobs
515
- buildkite-agent meta-data set "rocm-docker-image-s3-path" "$${S3_ARTIFACT_PATH}/rocm-base-image.tar.gz"
516
-
517
- # Mark that we did NOT use cache
518
- buildkite-agent meta-data set "rocm-used-cache" "false"
519
-
473
+ # Cache base docker image to ECR
474
+ docker push "$${ECR_CACHE_TAG}"
475
+
476
+ buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
477
+
478
echo ""
521
- echo "Build complete. Wheels cached for future builds."
479
+ echo " Build complete - Image and wheels cached"
480
fi
481
+
482
artifact_paths:
483
- "artifacts/rocm-base-wheels/*.whl"
484
env:
@@ -563,31 +522,25 @@ steps:
522
echo "Downloading wheel artifacts from current build"
523
buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
524
566
- # Download Docker image from S3 (too large for Buildkite artifacts)
567
- DOCKER_IMAGE_S3_PATH="$$(buildkite-agent meta-data get rocm-docker-image-s3-path 2>/dev/null || echo '')"
568
- if [ -z "$${DOCKER_IMAGE_S3_PATH}" ]; then
569
- echo "ERROR: rocm-docker-image-s3-path metadata not found"
525
+ # Get ECR image tag from metadata (set by build-rocm-base-wheels)
526
+ ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
527
+ if [ -z "$${ECR_IMAGE_TAG}" ]; then
528
+ echo "ERROR: rocm-base-image-tag metadata not found"
529
echo "This should have been set by the build-rocm-base-wheels job"
530
exit 1
531
fi
573
- echo "Downloading Docker image from $${DOCKER_IMAGE_S3_PATH}"
574
- mkdir -p artifacts/rocm-docker-image
575
- aws s3 cp "$${DOCKER_IMAGE_S3_PATH}" artifacts/rocm-docker-image/rocm-base-image.tar.gz
576
-
577
- # Load base Docker image and capture the tag
578
- echo "Loading base Docker image..."
579
- LOAD_OUTPUT=$$(gunzip -c artifacts/rocm-docker-image/rocm-base-image.tar.gz | docker load)
580
- echo "$${LOAD_OUTPUT}"
581
- # Extract the actual loaded image tag from "Loaded image: <tag>" output
582
- # This avoids picking up stale images (like rocm/vllm-dev:nightly) already on the agent
583
- BASE_IMAGE_TAG=$$(echo "$${LOAD_OUTPUT}" | grep "Loaded image:" | sed 's/Loaded image: //')
584
- if [ -z "$${BASE_IMAGE_TAG}" ]; then
585
- echo "ERROR: Failed to extract image tag from docker load output"
586
- echo "Load output was: $${LOAD_OUTPUT}"
587
- exit 1
588
- fi
589
- echo "Loaded base image: $${BASE_IMAGE_TAG}"
590
-
532
+
533
+ echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
534
+
535
+ # Login to ECR
536
+ aws ecr-public get-login-password --region us-east-1 | \
537
+ docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
538
+
539
+ # Pull base Docker image from ECR
540
+ docker pull "$${ECR_IMAGE_TAG}"
541
+
542
+ echo "Loaded base image: $${ECR_IMAGE_TAG}"
543
+
544
# Prepare base wheels for Docker build context
545
mkdir -p docker/context/base-wheels
546
touch docker/context/base-wheels/.keep
@@ -595,16 +548,11 @@ steps:
548
echo "Base wheels for vLLM build:"
549
ls -lh docker/context/base-wheels/
550
598
- # Get GPU architectures from meta-data
599
- PYTORCH_ROCM_ARCH="$$(buildkite-agent meta-data get rocm-pytorch-rocm-arch 2>/dev/null || echo '')"
600
- PYTORCH_ROCM_ARCH="$${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151}"
601
-
551
echo "========================================"
552
echo "Building vLLM wheel with:"
553
echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
554
echo " BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}"
606
- echo " PYTORCH_ROCM_ARCH: $${PYTORCH_ROCM_ARCH}"
607
- echo " BASE_IMAGE: $${BASE_IMAGE_TAG}"
555
+ echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
556
echo "========================================"
557
558
# Build vLLM wheel using local checkout (REMOTE_VLLM=0)
@@ -612,8 +560,7 @@ steps:
560
--file docker/Dockerfile.rocm \
561
--target export_vllm_wheel_release \
562
--output type=local,dest=rocm-dist \
615
- --build-arg BASE_IMAGE="$${BASE_IMAGE_TAG}" \
616
- --build-arg ARG_PYTORCH_ROCM_ARCH="$${PYTORCH_ROCM_ARCH}" \
563
+ --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
564
--build-arg REMOTE_VLLM=0 \
565
--build-arg GIT_REPO_CHECK=1 \
566
--build-arg USE_SCCACHE=1 \
@@ -621,10 +568,8 @@ steps:
568
--build-arg SCCACHE_REGION_NAME=us-west-2 \
569
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
570
.
624
-
571
echo "Built vLLM wheel:"
572
ls -lh rocm-dist/*.whl
627
-
573
# Copy wheel to artifacts directory
574
mkdir -p artifacts/rocm-vllm-wheel
575
cp rocm-dist/*.whl artifacts/rocm-vllm-wheel/
@@ -650,28 +595,6 @@ steps:
595
- |
596
set -euo pipefail
597
653
- # Check if upload is enabled (from env var, meta-data, or release branch)
654
- ROCM_UPLOAD_WHEELS="$${ROCM_UPLOAD_WHEELS:-}"
655
- if [ -z "$${ROCM_UPLOAD_WHEELS}" ]; then
656
- # Try to get from meta-data (input form)
657
- ROCM_UPLOAD_WHEELS="$$(buildkite-agent meta-data get rocm-upload-wheels 2>/dev/null || echo '')"
658
- fi
659
-
660
- echo "========================================"
661
- echo "Upload check:"
662
- echo " ROCM_UPLOAD_WHEELS: $${ROCM_UPLOAD_WHEELS}"
663
- echo " BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}"
664
- echo "========================================"
665
-
666
- # Skip upload if not enabled
667
- if [ "$${ROCM_UPLOAD_WHEELS}" != "true" ]; then
668
- echo "Skipping S3 upload (ROCM_UPLOAD_WHEELS != true, NIGHTLY != 1, not a release branch)"
669
- echo "To enable upload, set 'Upload Wheels to S3' to 'Yes' in the build configuration"
670
- exit 0
671
- fi
672
-
673
- echo "Upload enabled, proceeding..."
674
-
598
# Download artifacts from current build
599
echo "Downloading artifacts from current build"
600
buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
@@ -687,10 +610,7 @@ steps:
610
- label: ":memo: Annotate ROCm wheel release"
611
id: annotate-rocm-release
612
depends_on:
690
- - step: upload-rocm-wheels
691
- allow_failure: true
692
- - step: input-release-version
693
- allow_failure: true
613
+ - upload-rocm-wheels
614
agents:
615
queue: cpu_queue_release
616
commands:
@@ -716,7 +636,7 @@ steps:
636
S3_BUCKET: "vllm-wheels"
637
VARIANT: "rocm700"
638
719
- # ROCm Job 5: Build ROCm Release Docker Image
639
+ # ROCm Job 6: Build ROCm Release Docker Image
640
- label: ":docker: Build release image - x86_64 - ROCm"
641
id: build-rocm-release-image
642
depends_on:
@@ -728,42 +648,39 @@ steps:
648
commands:
649
- |
650
set -euo pipefail
731
-
651
+
652
# Login to ECR
653
aws ecr-public get-login-password --region us-east-1 | \
654
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
735
-
736
- # Download Docker image from S3 (set by build-rocm-base-wheels)
737
- DOCKER_IMAGE_S3_PATH="$$(buildkite-agent meta-data get rocm-docker-image-s3-path 2>/dev/null || echo '')"
738
- if [ -z "$${DOCKER_IMAGE_S3_PATH}" ]; then
739
- echo "ERROR: rocm-docker-image-s3-path metadata not found"
655
+
656
+ # Get ECR image tag from metadata (set by build-rocm-base-wheels)
657
+ ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
658
+ if [ -z "$${ECR_IMAGE_TAG}" ]; then
659
+ echo "ERROR: rocm-base-image-tag metadata not found"
660
+ echo "This should have been set by the build-rocm-base-wheels job"
661
exit 1
662
fi
742
-
743
- echo "Downloading base image from $${DOCKER_IMAGE_S3_PATH}"
744
- mkdir -p artifacts/rocm-docker-image
745
- aws s3 cp "$${DOCKER_IMAGE_S3_PATH}" artifacts/rocm-docker-image/rocm-base-image.tar.gz
746
-
747
- # Load base Docker image
748
- echo "Loading base Docker image..."
749
- LOAD_OUTPUT=$$(gunzip -c artifacts/rocm-docker-image/rocm-base-image.tar.gz | docker load)
750
- BASE_IMAGE_TAG=$$(echo "$${LOAD_OUTPUT}" | grep "Loaded image:" | sed 's/Loaded image: //')
751
- echo "Loaded base image: $${BASE_IMAGE_TAG}"
752
-
753
- # Tag and push the base image to ECR
754
- docker tag "$${BASE_IMAGE_TAG}" public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm-base
755
- docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm-base
756
- echo "Pushed base image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm-base"
757
-
758
- # Get GPU architectures from meta-data
759
- PYTORCH_ROCM_ARCH="$$(buildkite-agent meta-data get rocm-pytorch-rocm-arch 2>/dev/null || echo '')"
760
- PYTORCH_ROCM_ARCH="$${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151}"
761
-
663
+
664
+ echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
665
+
666
+ # Pull base Docker image from ECR
667
+ docker pull "$${ECR_IMAGE_TAG}"
668
+
669
+ echo "Loaded base image: $${ECR_IMAGE_TAG}"
670
+
671
+ # Pass the base image ECR tag to downstream steps (nightly publish)
672
+ buildkite-agent meta-data set "rocm-base-ecr-tag" "$${ECR_IMAGE_TAG}"
673
+
674
+ echo "========================================"
675
+ echo "Building vLLM ROCm release image with:"
676
+ echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
677
+ echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
678
+ echo "========================================"
679
+
680
# Build vLLM ROCm release image using cached base
681
DOCKER_BUILDKIT=1 docker build \
682
--build-arg max_jobs=16 \
765
- --build-arg BASE_IMAGE="$${BASE_IMAGE_TAG}" \
766
- --build-arg ARG_PYTORCH_ROCM_ARCH="$${PYTORCH_ROCM_ARCH}" \
683
+ --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
684
--build-arg USE_SCCACHE=1 \
685
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
686
--build-arg SCCACHE_REGION_NAME=us-west-2 \
@@ -772,10 +689,32 @@ steps:
689
--target vllm-openai \
690
--progress plain \
691
-f docker/Dockerfile.rocm .
775
-
692
+
693
# Push to ECR
694
docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm
778
- echo "Pushed: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm"
695
+
696
+ echo ""
697
+ echo " Successfully built and pushed ROCm release image"
698
+ echo " Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm"
699
+ echo ""
700
env:
701
DOCKER_BUILDKIT: "1"
702
S3_BUCKET: "vllm-wheels"
703
+
704
+ - label: "Publish nightly ROCm image to DockerHub"
705
+ depends_on:
706
+ - build-rocm-release-image
707
+ agents:
708
+ queue: small_cpu_queue_release
709
+ commands:
710
+ - "bash .buildkite/scripts/push-nightly-builds-rocm.sh"
711
+ # Clean up old nightly builds (keep only last 14)
712
+ - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-rocm"
713
+ - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly- vllm/vllm-openai-rocm"
714
+ plugins:
715
+ - docker-login#v3.0.0:
716
+ username: vllmbot
717
+ password-env: DOCKERHUB_TOKEN
718
+ env:
719
+ DOCKER_BUILDKIT: "1"
720
+ DOCKERHUB_USERNAME: "vllmbot"
.buildkite/scripts/annotate-release.sh
+4
-2
@@ -8,6 +8,8 @@ if [ -z "${RELEASE_VERSION}" ]; then
8
RELEASE_VERSION="1.0.0.dev"
9
fi
10
11
+ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
12
+
13
buildkite-agent annotate --style 'info' --context 'release-workflow' << EOF
14
To download the wheel (by commit):
15
\`\`\`
@@ -33,7 +35,7 @@ docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64
35
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64
36
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130
37
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130
36
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base
38
+docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base
39
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
40
docker pull public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION}
41
docker pull public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:v${RELEASE_VERSION}
@@ -74,7 +76,7 @@ docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:v${RE
76
docker push vllm/vllm-openai-rocm:latest
77
docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}
78
77
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
79
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
80
docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base
81
docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
82
docker push vllm/vllm-openai-rocm:latest-base
.buildkite/scripts/annotate-rocm-release.sh
+6
-5
@@ -5,20 +5,21 @@
5
# Generate Buildkite annotation for ROCm wheel release
6
set -ex
7
8
-# Get build configuration from meta-data
8
+# Extract build configuration from Dockerfile.rocm_base (single source of truth)
9
# Extract ROCm version dynamically from Dockerfile.rocm_base
10
# BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.0-complete -> extracts "7.0"
11
ROCM_VERSION=$(grep -E '^ARG BASE_IMAGE=' docker/Dockerfile.rocm_base | sed -E 's/.*:([0-9]+\.[0-9]+).*/\1/' || echo "unknown")
12
-PYTHON_VERSION=$(buildkite-agent meta-data get rocm-python-version 2>/dev/null || echo "3.12")
13
-PYTORCH_ROCM_ARCH=$(buildkite-agent meta-data get rocm-pytorch-rocm-arch 2>/dev/null || echo "gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151")
12
+PYTHON_VERSION=$(grep '^ARG PYTHON_VERSION=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTHON_VERSION=//')
13
+PYTORCH_ROCM_ARCH=$(grep '^ARG PYTORCH_ROCM_ARCH=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTORCH_ROCM_ARCH=//')
14
15
-# TODO: Enable the nightly build for ROCm
15
# Get release version, default to 1.0.0.dev for nightly/per-commit builds
16
RELEASE_VERSION=$(buildkite-agent meta-data get release-version 2>/dev/null || echo "")
17
if [ -z "${RELEASE_VERSION}" ]; then
18
RELEASE_VERSION="1.0.0.dev"
19
fi
20
21
+ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
22
+
23
# S3 URLs
24
S3_BUCKET="${S3_BUCKET:-vllm-wheels}"
25
S3_REGION="${AWS_DEFAULT_REGION:-us-west-2}"
@@ -96,7 +97,7 @@ To download and upload the image:
97
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base
98
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
99
99
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
100
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
101
docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base
102
docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
103
docker push vllm/vllm-openai-rocm:latest-base
.buildkite/scripts/cache-rocm-base-wheels.sh
+7
-16
@@ -15,8 +15,6 @@
15
#
16
# Environment variables:
17
# S3_BUCKET - S3 bucket name (default: vllm-wheels)
18
-# PYTHON_VERSION - Python version (affects cache key)
19
-# PYTORCH_ROCM_ARCH - GPU architectures (affects cache key)
18
#
19
# Note: ROCm version is determined by BASE_IMAGE in Dockerfile.rocm_base,
20
# so changes to ROCm version are captured by the Dockerfile hash.
@@ -36,13 +34,7 @@ generate_cache_key() {
34
fi
35
local dockerfile_hash=$(sha256sum "$DOCKERFILE" | cut -c1-16)
36
39
- # Include key build args that affect the output
40
- # These should match the ARGs in Dockerfile.rocm_base that change the build output
41
- # Note: ROCm version is determined by BASE_IMAGE in the Dockerfile, so it's captured by dockerfile_hash
42
- local args_string="${PYTHON_VERSION:-}|${PYTORCH_ROCM_ARCH:-}"
43
- local args_hash=$(echo "$args_string" | sha256sum | cut -c1-8)
44
-
45
- echo "${dockerfile_hash}-${args_hash}"
37
+ echo "${dockerfile_hash}"
38
}
39
40
CACHE_KEY=$(generate_cache_key)
@@ -52,9 +44,6 @@ case "${1:-}" in
44
check)
45
echo "Checking cache for key: ${CACHE_KEY}" >&2
46
echo "Cache path: ${CACHE_PATH}" >&2
55
- echo "Variables used in cache key:" >&2
56
- echo " PYTHON_VERSION: ${PYTHON_VERSION:-<not set>}" >&2
57
- echo " PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH:-<not set>}" >&2
47
48
# Check if cache exists by listing objects
49
# We look for at least one .whl file
@@ -104,14 +93,16 @@ case "${1:-}" in
93
echo "Cache key: ${CACHE_KEY}"
94
echo "Cache path: ${CACHE_PATH}"
95
echo ""
107
-
96
mkdir -p artifacts/rocm-base-wheels
109
- aws s3 cp --recursive "${CACHE_PATH}" artifacts/rocm-base-wheels/
110
-
97
+
98
+ # Use sync with include/exclude to only download .whl files
99
+ aws s3 sync "${CACHE_PATH}" artifacts/rocm-base-wheels/ \
100
+ --exclude "*" \
101
+ --include "*.whl"
102
+
103
echo ""
104
echo "Downloaded wheels:"
105
find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' -exec ls -lh {} \;
114
-
106
WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
107
echo ""
108
echo "Total: $WHEEL_COUNT wheels"
.buildkite/scripts/cleanup-nightly-builds.sh
+10
-7
@@ -4,16 +4,19 @@ set -ex
4
5
# Clean up old nightly builds from DockerHub, keeping only the last 14 builds
6
# This script uses DockerHub API to list and delete old tags with specified prefix
7
-# Usage: cleanup-nightly-builds.sh [TAG_PREFIX]
8
-# Example: cleanup-nightly-builds.sh "nightly-" or cleanup-nightly-builds.sh "cu130-nightly-"
7
+# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO]
8
+# Example: cleanup-nightly-builds.sh "nightly-"
9
+# Example: cleanup-nightly-builds.sh "cu130-nightly-"
10
+# Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm"
11
10
-# Get tag prefix from argument, default to "nightly-" if not provided
12
+# Get tag prefix and repo from arguments
13
TAG_PREFIX="${1:-nightly-}"
14
+REPO="${2:-vllm/vllm-openai}"
15
13
-echo "Cleaning up tags with prefix: $TAG_PREFIX"
16
+echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
17
15
-# DockerHub API endpoint for vllm/vllm-openai repository
16
-REPO_API_URL="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags"
18
+# DockerHub API endpoint for the repository
19
+REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags"
20
21
# Get DockerHub credentials from environment
22
if [ -z "$DOCKERHUB_TOKEN" ]; then
@@ -70,7 +73,7 @@ delete_tag() {
73
local tag_name="$1"
74
echo "Deleting tag: $tag_name"
75
73
- local delete_url="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags/$tag_name"
76
+ local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name"
77
set +x
78
local response=$(curl -s -X DELETE -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url")
79
set -x
.buildkite/scripts/push-nightly-builds-rocm.sh
new
+62
@@ -0,0 +1,62 @@
1
+#!/bin/bash
2
+# SPDX-License-Identifier: Apache-2.0
3
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4
+#
5
+# Push ROCm nightly base image and nightly image from ECR
6
+# to Docker Hub as vllm/vllm-openai-rocm:base-nightly and vllm/vllm-openai-rocm:nightly
7
+# and vllm/vllm-openai-rocm:base-nightly-<commit> and vllm/vllm-openai-rocm:nightly-<commit>.
8
+# Run when NIGHTLY=1 after build-rocm-release-image has pushed to ECR.
9
+#
10
+# Local testing (no push to Docker Hub):
11
+# BUILDKITE_COMMIT=<commit-with-rocm-image-in-ecr> DRY_RUN=1 bash .buildkite/scripts/push-nightly-builds-rocm.sh
12
+# Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login.
13
+
14
+set -ex
15
+
16
+# Use BUILDKITE_COMMIT from env (required; set to a commit that has ROCm image in ECR for local test)
17
+BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the ROCm image in ECR (e.g. from a previous release pipeline run)}"
18
+DRY_RUN="${DRY_RUN:-0}"
19
+
20
+# Get the base image ECR tag (set by build-rocm-release-image pipeline step)
21
+BASE_ORIG_TAG="$(buildkite-agent meta-data get rocm-base-ecr-tag 2>/dev/null || echo "")"
22
+if [ -z "$BASE_ORIG_TAG" ]; then
23
+ echo "WARNING: rocm-base-ecr-tag metadata not found, falling back to commit-based tag"
24
+ BASE_ORIG_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base"
25
+fi
26
+
27
+ORIG_TAG="${BUILDKITE_COMMIT}-rocm"
28
+BASE_TAG_NAME="base-nightly"
29
+TAG_NAME="nightly"
30
+BASE_TAG_NAME_COMMIT="base-nightly-${BUILDKITE_COMMIT}"
31
+TAG_NAME_COMMIT="nightly-${BUILDKITE_COMMIT}"
32
+
33
+echo "Pushing ROCm base image from ECR: $BASE_ORIG_TAG"
34
+echo "Pushing ROCm release image from ECR tag: $ORIG_TAG to Docker Hub as $TAG_NAME and $TAG_NAME_COMMIT"
35
+[[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub"
36
+
37
+# Login to ECR and pull the image built by build-rocm-release-image
38
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
39
+docker pull "$BASE_ORIG_TAG"
40
+docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG"
41
+
42
+# Tag for Docker Hub (base-nightly and base-nightly-<commit>, nightly and nightly-<commit>)
43
+docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
44
+docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
45
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME"
46
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
47
+
48
+if [[ "$DRY_RUN" == "1" ]]; then
49
+ echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
50
+ echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
51
+ echo "[DRY_RUN] Local tags created. Exiting without push."
52
+ exit 0
53
+fi
54
+
55
+# Push to Docker Hub (docker-login plugin runs before this step in CI)
56
+docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
57
+docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
58
+docker push vllm/vllm-openai-rocm:"$TAG_NAME"
59
+docker push vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
60
+
61
+echo "Pushed vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
62
+echo "Pushed vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"