@tduncan / Nvidia-Optimization-Stack-VLLM / commits / ef0aa7ca2f

[ROCm] [Release] [Per-commit] Reenable per commit rocm wheel (#49044)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>

TJian committed Jul 19, 2026 at 11:38 UTC ef0aa7ca2feb75051b30ea3cef4e9950252b1441
1 file changed +354 -357
.buildkite/release-pipeline.yaml
+354 -357
@@ -590,373 +590,370 @@ steps:
590 #
591 # =============================================================================
592
593 - - block: "Unblock ROCm wheel/image prerequisites"
593 + - group: "Build ROCm Wheel / Image "
594 + key: "build-rocm-wheel-image"
595 depends_on: ~
595 - key: block-build-rocm
596 - if: build.env("NIGHTLY") != "1"
596 + steps:
597 + # ROCm Job 1: Build ROCm Base Wheels (with S3 caching)
598 + - label: ":rocm: Build ROCm Base Image & Wheels"
599 + id: build-rocm-base-wheels
600 + depends_on: ~
601 + agents:
602 + queue: cpu_queue_release
603 + commands:
604 + - |
605 + set -euo pipefail
606 +
607 + # Generate cache key
608 + CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
609 + ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base"
610 +
611 + echo "========================================"
612 + echo "ROCm Base Build Configuration"
613 + echo "========================================"
614 + echo " CACHE_KEY: $${CACHE_KEY}"
615 + echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
616 + echo "========================================"
617 +
618 + # Login to ECR
619 + aws ecr-public get-login-password --region us-east-1 | \
620 + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
621 +
622 + IMAGE_EXISTS=false
623 + WHEELS_EXIST=false
624 +
625 + # Check ECR for Docker image
626 +
627 + if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
628 + IMAGE_EXISTS=true
629 + echo "ECR image cache HIT"
630 + fi
631 +
632 + # Check S3 for wheels
633 + WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
634 + if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then
635 + WHEELS_EXIST=true
636 + echo "S3 wheels cache HIT"
637 + fi
638 +
639 +
640 + # Scenario 1: Both cached (best case)
641 + if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then
642 + echo ""
643 + echo "FULL CACHE HIT - Reusing both image and wheels"
644 + echo ""
645 +
646 + # Download wheels
647 + .buildkite/scripts/cache-rocm-base-wheels.sh download
648 +
649 + # Save ECR tag for downstream jobs
650 + buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
651 +
652 + # Scenario 2: Full rebuild needed
653 + else
654 + echo ""
655 + echo " CACHE MISS - Building from scratch..."
656 + echo ""
657 +
658 + # Build full base image and push to ECR
659 + DOCKER_BUILDKIT=1 docker buildx build \
660 + --file docker/Dockerfile.rocm_base \
661 + --tag "$${ECR_CACHE_TAG}" \
662 + --build-arg USE_SCCACHE=1 \
663 + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
664 + --build-arg SCCACHE_REGION_NAME=us-west-2 \
665 + --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
666 + --push \
667 + .
668 +
669 + # Build wheel extraction stage
670 + DOCKER_BUILDKIT=1 docker buildx build \
671 + --file docker/Dockerfile.rocm_base \
672 + --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
673 + --target debs_wheel_release \
674 + --build-arg USE_SCCACHE=1 \
675 + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
676 + --build-arg SCCACHE_REGION_NAME=us-west-2 \
677 + --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
678 + --load \
679 + .
680 +
681 + # Extract and upload wheels
682 + mkdir -p artifacts/rocm-base-wheels
683 + cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
684 + docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
685 + docker rm $${cid}
686 +
687 + .buildkite/scripts/cache-rocm-base-wheels.sh upload
688 +
689 + # Cache base docker image to ECR
690 + docker push "$${ECR_CACHE_TAG}"
691 +
692 + buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
693 +
694 + echo ""
695 + echo " Build complete - Image and wheels cached"
696 + fi
697
598 - # ROCm Job 1: Build ROCm Base Wheels (with S3 caching)
599 - - label: ":rocm: Build ROCm Base Image & Wheels"
600 - id: build-rocm-base-wheels
601 - depends_on:
602 - - step: block-build-rocm
603 - allow_failure: true
604 - agents:
605 - queue: cpu_queue_release
606 - commands:
607 - - |
608 - set -euo pipefail
609 -
610 - # Generate cache key
611 - CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
612 - ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base"
613 -
614 - echo "========================================"
615 - echo "ROCm Base Build Configuration"
616 - echo "========================================"
617 - echo " CACHE_KEY: $${CACHE_KEY}"
618 - echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
619 - echo "========================================"
620 -
621 - # Login to ECR
622 - aws ecr-public get-login-password --region us-east-1 | \
623 - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
624 -
625 - IMAGE_EXISTS=false
626 - WHEELS_EXIST=false
627 -
628 - # Check ECR for Docker image
629 -
630 - if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
631 - IMAGE_EXISTS=true
632 - echo "ECR image cache HIT"
633 - fi
634 -
635 - # Check S3 for wheels
636 - WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
637 - if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then
638 - WHEELS_EXIST=true
639 - echo "S3 wheels cache HIT"
640 - fi
641 -
642 -
643 - # Scenario 1: Both cached (best case)
644 - if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then
645 - echo ""
646 - echo "FULL CACHE HIT - Reusing both image and wheels"
647 - echo ""
648 -
649 - # Download wheels
650 - .buildkite/scripts/cache-rocm-base-wheels.sh download
651 -
652 - # Save ECR tag for downstream jobs
653 - buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
654 -
655 - # Scenario 2: Full rebuild needed
656 - else
657 - echo ""
658 - echo " CACHE MISS - Building from scratch..."
659 - echo ""
660 -
661 - # Build full base image and push to ECR
662 - DOCKER_BUILDKIT=1 docker buildx build \
663 - --file docker/Dockerfile.rocm_base \
664 - --tag "$${ECR_CACHE_TAG}" \
665 - --build-arg USE_SCCACHE=1 \
666 - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
667 - --build-arg SCCACHE_REGION_NAME=us-west-2 \
668 - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
669 - --push \
670 - .
671 -
672 - # Build wheel extraction stage
673 - DOCKER_BUILDKIT=1 docker buildx build \
674 - --file docker/Dockerfile.rocm_base \
675 - --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
676 - --target debs_wheel_release \
677 - --build-arg USE_SCCACHE=1 \
678 - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
679 - --build-arg SCCACHE_REGION_NAME=us-west-2 \
680 - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
681 - --load \
682 - .
683 -
684 - # Extract and upload wheels
685 - mkdir -p artifacts/rocm-base-wheels
686 - cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
687 - docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
688 - docker rm $${cid}
689 -
690 - .buildkite/scripts/cache-rocm-base-wheels.sh upload
691 -
692 - # Cache base docker image to ECR
693 - docker push "$${ECR_CACHE_TAG}"
694 -
695 - buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
696 -
697 - echo ""
698 - echo " Build complete - Image and wheels cached"
699 - fi
700 -
701 - artifact_paths:
702 - - "artifacts/rocm-base-wheels/*.whl"
703 - env:
704 - DOCKER_BUILDKIT: "1"
705 - S3_BUCKET: "vllm-wheels"
698 + artifact_paths:
699 + - "artifacts/rocm-base-wheels/*.whl"
700 + env:
701 + DOCKER_BUILDKIT: "1"
702 + S3_BUCKET: "vllm-wheels"
703
707 - # ROCm Job 2: Build vLLM ROCm Wheel
708 - - label: ":python: Build vLLM ROCm Wheel - x86_64"
709 - id: build-rocm-vllm-wheel
710 - depends_on:
711 - - step: build-rocm-base-wheels
712 - allow_failure: false
713 - agents:
714 - queue: cpu_queue_release
715 - timeout_in_minutes: 180
716 - commands:
717 - # Download artifacts and prepare Docker image
718 - - |
719 - set -euo pipefail
720 -
721 - # Ensure git tags are up-to-date (Buildkite's default fetch doesn't update tags)
722 - # This fixes version detection when tags are moved/force-pushed
723 - echo "Fetching latest tags from origin..."
724 - git fetch --tags --force origin
725 -
726 - # Log tag information for debugging version detection
727 - echo "========================================"
728 - echo "Git Tag Verification"
729 - echo "========================================"
730 - echo "Current HEAD: $(git rev-parse HEAD)"
731 - echo "git describe --tags: $(git describe --tags 2>/dev/null || echo 'No tags found')"
732 - echo ""
733 - echo "Recent tags (pointing to commits near HEAD):"
734 - git tag -l --sort=-creatordate | head -5
735 - echo "setuptools_scm version detection:"
736 - pip install -q setuptools_scm 2>/dev/null || true
737 - python3 -c "import setuptools_scm; print(' Detected version:', setuptools_scm.get_version())" 2>/dev/null || echo " (setuptools_scm not available in this environment)"
738 - echo "========================================"
739 -
740 - # Download wheel artifacts from current build
741 - echo "Downloading wheel artifacts from current build"
742 - buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
743 -
744 - # Get ECR image tag from metadata (set by build-rocm-base-wheels)
745 - ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
746 - if [ -z "$${ECR_IMAGE_TAG}" ]; then
747 - echo "ERROR: rocm-base-image-tag metadata not found"
748 - echo "This should have been set by the build-rocm-base-wheels job"
749 - exit 1
750 - fi
751 -
752 - echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
753 -
754 - # Login to ECR
755 - aws ecr-public get-login-password --region us-east-1 | \
756 - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
757 -
758 - # Pull base Docker image from ECR
759 - docker pull "$${ECR_IMAGE_TAG}"
760 -
761 - echo "Loaded base image: $${ECR_IMAGE_TAG}"
762 -
763 - # Prepare base wheels for Docker build context
764 - mkdir -p docker/context/base-wheels
765 - touch docker/context/base-wheels/.keep
766 - cp artifacts/rocm-base-wheels/*.whl docker/context/base-wheels/
767 - echo "Base wheels for vLLM build:"
768 - ls -lh docker/context/base-wheels/
769 -
770 - echo "========================================"
771 - echo "Building vLLM wheel with:"
772 - echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
773 - echo " BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}"
774 - echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
775 - echo "========================================"
776 -
777 - # Build vLLM wheel using local checkout (REMOTE_VLLM=0)
778 - DOCKER_BUILDKIT=1 docker build \
779 - --file docker/Dockerfile.rocm \
780 - --target export_vllm_wheel_release \
781 - --output type=local,dest=rocm-dist \
782 - --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
783 - --build-arg REMOTE_VLLM=0 \
784 - --build-arg GIT_REPO_CHECK=1 \
785 - --build-arg USE_SCCACHE=1 \
786 - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
787 - --build-arg SCCACHE_REGION_NAME=us-west-2 \
788 - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
789 - .
790 - echo "Built vLLM wheel:"
791 - ls -lh rocm-dist/*.whl
792 - # Copy wheel to artifacts directory
793 - mkdir -p artifacts/rocm-vllm-wheel
794 - cp rocm-dist/*.whl artifacts/rocm-vllm-wheel/
795 - echo "Final vLLM wheel:"
796 - ls -lh artifacts/rocm-vllm-wheel/
797 - artifact_paths:
798 - - "artifacts/rocm-vllm-wheel/*.whl"
799 - env:
800 - DOCKER_BUILDKIT: "1"
801 - S3_BUCKET: "vllm-wheels"
704 + # ROCm Job 2: Build vLLM ROCm Wheel
705 + - label: ":python: Build vLLM ROCm Wheel - x86_64"
706 + id: build-rocm-vllm-wheel
707 + depends_on:
708 + - step: build-rocm-base-wheels
709 + allow_failure: false
710 + agents:
711 + queue: cpu_queue_release
712 + timeout_in_minutes: 180
713 + commands:
714 + # Download artifacts and prepare Docker image
715 + - |
716 + set -euo pipefail
717 +
718 + # Ensure git tags are up-to-date (Buildkite's default fetch doesn't update tags)
719 + # This fixes version detection when tags are moved/force-pushed
720 + echo "Fetching latest tags from origin..."
721 + git fetch --tags --force origin
722 +
723 + # Log tag information for debugging version detection
724 + echo "========================================"
725 + echo "Git Tag Verification"
726 + echo "========================================"
727 + echo "Current HEAD: $(git rev-parse HEAD)"
728 + echo "git describe --tags: $(git describe --tags 2>/dev/null || echo 'No tags found')"
729 + echo ""
730 + echo "Recent tags (pointing to commits near HEAD):"
731 + git tag -l --sort=-creatordate | head -5
732 + echo "setuptools_scm version detection:"
733 + pip install -q setuptools_scm 2>/dev/null || true
734 + python3 -c "import setuptools_scm; print(' Detected version:', setuptools_scm.get_version())" 2>/dev/null || echo " (setuptools_scm not available in this environment)"
735 + echo "========================================"
736 +
737 + # Download wheel artifacts from current build
738 + echo "Downloading wheel artifacts from current build"
739 + buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
740 +
741 + # Get ECR image tag from metadata (set by build-rocm-base-wheels)
742 + ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
743 + if [ -z "$${ECR_IMAGE_TAG}" ]; then
744 + echo "ERROR: rocm-base-image-tag metadata not found"
745 + echo "This should have been set by the build-rocm-base-wheels job"
746 + exit 1
747 + fi
748 +
749 + echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
750 +
751 + # Login to ECR
752 + aws ecr-public get-login-password --region us-east-1 | \
753 + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
754 +
755 + # Pull base Docker image from ECR
756 + docker pull "$${ECR_IMAGE_TAG}"
757 +
758 + echo "Loaded base image: $${ECR_IMAGE_TAG}"
759 +
760 + # Prepare base wheels for Docker build context
761 + mkdir -p docker/context/base-wheels
762 + touch docker/context/base-wheels/.keep
763 + cp artifacts/rocm-base-wheels/*.whl docker/context/base-wheels/
764 + echo "Base wheels for vLLM build:"
765 + ls -lh docker/context/base-wheels/
766 +
767 + echo "========================================"
768 + echo "Building vLLM wheel with:"
769 + echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
770 + echo " BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}"
771 + echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
772 + echo "========================================"
773 +
774 + # Build vLLM wheel using local checkout (REMOTE_VLLM=0)
775 + DOCKER_BUILDKIT=1 docker build \
776 + --file docker/Dockerfile.rocm \
777 + --target export_vllm_wheel_release \
778 + --output type=local,dest=rocm-dist \
779 + --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
780 + --build-arg REMOTE_VLLM=0 \
781 + --build-arg GIT_REPO_CHECK=1 \
782 + --build-arg USE_SCCACHE=1 \
783 + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
784 + --build-arg SCCACHE_REGION_NAME=us-west-2 \
785 + --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
786 + .
787 + echo "Built vLLM wheel:"
788 + ls -lh rocm-dist/*.whl
789 + # Copy wheel to artifacts directory
790 + mkdir -p artifacts/rocm-vllm-wheel
791 + cp rocm-dist/*.whl artifacts/rocm-vllm-wheel/
792 + echo "Final vLLM wheel:"
793 + ls -lh artifacts/rocm-vllm-wheel/
794 + artifact_paths:
795 + - "artifacts/rocm-vllm-wheel/*.whl"
796 + env:
797 + DOCKER_BUILDKIT: "1"
798 + S3_BUCKET: "vllm-wheels"
799
803 - # ROCm Job 3: Upload Wheels to S3
804 - - label: ":s3: Upload ROCm Wheels to S3"
805 - id: upload-rocm-wheels
806 - depends_on:
807 - - step: build-rocm-vllm-wheel
808 - allow_failure: false
809 - agents:
810 - queue: cpu_queue_release
811 - timeout_in_minutes: 60
812 - commands:
813 - # Download all wheel artifacts and run upload
814 - - |
815 - set -euo pipefail
800 + # ROCm Job 3: Upload Wheels to S3
801 + - label: ":s3: Upload ROCm Wheels to S3"
802 + id: upload-rocm-wheels
803 + depends_on:
804 + - step: build-rocm-vllm-wheel
805 + allow_failure: false
806 + agents:
807 + queue: cpu_queue_release
808 + timeout_in_minutes: 60
809 + commands:
810 + # Download all wheel artifacts and run upload
811 + - |
812 + set -euo pipefail
813
817 - # Download artifacts from current build
818 - echo "Downloading artifacts from current build"
819 - buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
820 - buildkite-agent artifact download "artifacts/rocm-vllm-wheel/*.whl" .
814 + # Download artifacts from current build
815 + echo "Downloading artifacts from current build"
816 + # buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
817 + # buildkite-agent artifact download "artifacts/rocm-vllm-wheel/*.whl" .
818
822 - # Run upload script
823 - bash .buildkite/scripts/upload-rocm-wheels.sh
824 - env:
825 - DOCKER_BUILDKIT: "1"
826 - S3_BUCKET: "vllm-wheels"
819 + # # Run upload script
820 + bash .buildkite/scripts/upload-rocm-wheels.sh
821 + env:
822 + DOCKER_BUILDKIT: "1"
823 + S3_BUCKET: "vllm-wheels"
824
828 - # ROCm Job 4: Annotate ROCm Wheel Release
829 - - label: ":memo: Annotate ROCm wheel release"
830 - id: annotate-rocm-release
831 - depends_on:
832 - - upload-rocm-wheels
833 - agents:
834 - queue: cpu_queue_release
835 - commands:
836 - - "bash .buildkite/scripts/annotate-rocm-release.sh"
837 - env:
838 - S3_BUCKET: "vllm-wheels"
839 -
840 - # ROCm Job 5: Generate Root Index for ROCm Wheels (for release only)
841 - # This is the job to create https://wheels.vllm.ai/rocm/ index allowing
842 - # users to install with `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/`
843 - - block: "Generate Root Index for ROCm Wheels for Release"
844 - key: block-generate-root-index-rocm-wheels
845 - depends_on: upload-rocm-wheels
846 -
847 - - label: ":package: Generate Root Index for ROCm Wheels for Release"
848 - depends_on: block-generate-root-index-rocm-wheels
849 - id: generate-root-index-rocm-wheels
850 - agents:
851 - queue: cpu_queue_release
852 - commands:
853 - - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
854 - env:
855 - S3_BUCKET: "vllm-wheels"
856 - VARIANT: "rocm723"
825 + # ROCm Job 4: Annotate ROCm Wheel Release
826 + - label: ":memo: Annotate ROCm wheel release"
827 + id: annotate-rocm-release
828 + depends_on:
829 + - upload-rocm-wheels
830 + agents:
831 + queue: cpu_queue_release
832 + commands:
833 + - "bash .buildkite/scripts/annotate-rocm-release.sh"
834 + env:
835 + S3_BUCKET: "vllm-wheels"
836 +
837 + # ROCm Job 5: Generate Root Index for ROCm Wheels (for release only)
838 + # This is the job to create https://wheels.vllm.ai/rocm/ index allowing
839 + # users to install with `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/`
840 + - block: "Generate Root Index for ROCm Wheels for Release"
841 + key: block-generate-root-index-rocm-wheels
842 + depends_on: upload-rocm-wheels
843 +
844 + - label: ":package: Generate Root Index for ROCm Wheels for Release"
845 + depends_on: block-generate-root-index-rocm-wheels
846 + id: generate-root-index-rocm-wheels
847 + agents:
848 + queue: cpu_queue_release
849 + commands:
850 + - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
851 + env:
852 + S3_BUCKET: "vllm-wheels"
853 + VARIANT: "rocm723"
854
858 - # ROCm Job 6: Build ROCm Release Docker Image
859 - - label: ":docker: Build release image - x86_64 - ROCm"
860 - id: build-rocm-release-image
861 - depends_on:
862 - - step: block-build-release-images
863 - allow_failure: true
864 - - step: build-rocm-base-wheels
865 - allow_failure: false
866 - agents:
867 - queue: cpu_queue_release
868 - timeout_in_minutes: 60
869 - commands:
870 - - |
871 - set -euo pipefail
872 -
873 - # Login to ECR
874 - aws ecr-public get-login-password --region us-east-1 | \
875 - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
876 -
877 - # Get ECR image tag from metadata (set by build-rocm-base-wheels)
878 - ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
879 - if [ -z "$${ECR_IMAGE_TAG}" ]; then
880 - echo "ERROR: rocm-base-image-tag metadata not found"
881 - echo "This should have been set by the build-rocm-base-wheels job"
882 - exit 1
883 - fi
884 -
885 - echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
886 -
887 - # Pull base Docker image from ECR
888 - docker pull "$${ECR_IMAGE_TAG}"
889 -
890 - echo "Loaded base image: $${ECR_IMAGE_TAG}"
891 -
892 - # Pass the base image ECR tag to downstream steps (nightly publish)
893 - buildkite-agent meta-data set "rocm-base-ecr-tag" "$${ECR_IMAGE_TAG}"
894 -
895 - echo "========================================"
896 - echo "Building vLLM ROCm release image with:"
897 - echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
898 - echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
899 - echo "========================================"
900 -
901 - # Build vLLM ROCm release image using cached base
902 - DOCKER_BUILDKIT=1 docker build \
903 - --build-arg max_jobs=16 \
904 - --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
905 - --build-arg USE_SCCACHE=1 \
906 - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
907 - --build-arg SCCACHE_REGION_NAME=us-west-2 \
908 - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
909 - --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm \
910 - --target vllm-openai \
911 - --progress plain \
912 - -f docker/Dockerfile.rocm .
913 -
914 - # Push to ECR
915 - docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm
916 -
917 - echo ""
918 - echo " Successfully built and pushed ROCm release image"
919 - echo " Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm"
920 - echo ""
921 - env:
922 - DOCKER_BUILDKIT: "1"
923 - S3_BUCKET: "vllm-wheels"
855 + # ROCm Job 6: Build ROCm Release Docker Image
856 + - label: ":docker: Build release image - x86_64 - ROCm"
857 + id: build-rocm-release-image
858 + depends_on:
859 + - step: block-build-release-images
860 + allow_failure: true
861 + - step: build-rocm-base-wheels
862 + allow_failure: false
863 + agents:
864 + queue: cpu_queue_release
865 + timeout_in_minutes: 60
866 + commands:
867 + - |
868 + set -euo pipefail
869 +
870 + # Login to ECR
871 + aws ecr-public get-login-password --region us-east-1 | \
872 + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
873 +
874 + # Get ECR image tag from metadata (set by build-rocm-base-wheels)
875 + ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
876 + if [ -z "$${ECR_IMAGE_TAG}" ]; then
877 + echo "ERROR: rocm-base-image-tag metadata not found"
878 + echo "This should have been set by the build-rocm-base-wheels job"
879 + exit 1
880 + fi
881 +
882 + echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
883 +
884 + # Pull base Docker image from ECR
885 + docker pull "$${ECR_IMAGE_TAG}"
886 +
887 + echo "Loaded base image: $${ECR_IMAGE_TAG}"
888 +
889 + # Pass the base image ECR tag to downstream steps (nightly publish)
890 + buildkite-agent meta-data set "rocm-base-ecr-tag" "$${ECR_IMAGE_TAG}"
891 +
892 + echo "========================================"
893 + echo "Building vLLM ROCm release image with:"
894 + echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
895 + echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
896 + echo "========================================"
897 +
898 + # Build vLLM ROCm release image using cached base
899 + DOCKER_BUILDKIT=1 docker build \
900 + --build-arg max_jobs=16 \
901 + --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
902 + --build-arg USE_SCCACHE=1 \
903 + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
904 + --build-arg SCCACHE_REGION_NAME=us-west-2 \
905 + --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
906 + --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm \
907 + --target vllm-openai \
908 + --progress plain \
909 + -f docker/Dockerfile.rocm .
910 +
911 + # Push to ECR
912 + docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm
913 +
914 + echo ""
915 + echo " Successfully built and pushed ROCm release image"
916 + echo " Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm"
917 + echo ""
918 + env:
919 + DOCKER_BUILDKIT: "1"
920 + S3_BUCKET: "vllm-wheels"
921
925 - - label: "Publish nightly XPU image to DockerHub"
926 - depends_on:
927 - - create-manifest-xpu
928 - if: build.env("NIGHTLY") == "1"
929 - agents:
930 - queue: small_cpu_queue_release
931 - commands:
932 - - "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh"
933 - - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu"
934 - plugins:
935 - - docker-login#v3.0.0:
936 - username: vllmbot
937 - password-env: DOCKERHUB_TOKEN
938 - env:
939 - DOCKER_BUILDKIT: "1"
940 - DOCKERHUB_USERNAME: "vllmbot"
922 + - label: "Publish nightly XPU image to DockerHub"
923 + depends_on:
924 + - create-manifest-xpu
925 + if: build.env("NIGHTLY") == "1"
926 + agents:
927 + queue: small_cpu_queue_release
928 + commands:
929 + - "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh"
930 + - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu"
931 + plugins:
932 + - docker-login#v3.0.0:
933 + username: vllmbot
934 + password-env: DOCKERHUB_TOKEN
935 + env:
936 + DOCKER_BUILDKIT: "1"
937 + DOCKERHUB_USERNAME: "vllmbot"
938
942 - - label: "Publish nightly ROCm image to DockerHub"
943 - depends_on:
944 - - build-rocm-release-image
945 - if: build.env("NIGHTLY") == "1"
946 - agents:
947 - queue: small_cpu_queue_release
948 - commands:
949 - - "bash .buildkite/scripts/push-nightly-builds-rocm.sh"
950 - # Clean up old nightly builds (keep only last 14)
951 - - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-rocm"
952 - - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly- vllm/vllm-openai-rocm"
953 - plugins:
954 - - docker-login#v3.0.0:
955 - username: vllmbot
956 - password-env: DOCKERHUB_TOKEN
957 - env:
958 - DOCKER_BUILDKIT: "1"
959 - DOCKERHUB_USERNAME: "vllmbot"
939 + - label: "Publish nightly ROCm image to DockerHub"
940 + depends_on:
941 + - build-rocm-release-image
942 + if: build.env("NIGHTLY") == "1"
943 + agents:
944 + queue: small_cpu_queue_release
945 + commands:
946 + - "bash .buildkite/scripts/push-nightly-builds-rocm.sh"
947 + # Clean up old nightly builds (keep only last 14)
948 + - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-rocm"
949 + - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly- vllm/vllm-openai-rocm"
950 + plugins:
951 + - docker-login#v3.0.0:
952 + username: vllmbot
953 + password-env: DOCKERHUB_TOKEN
954 + env:
955 + DOCKER_BUILDKIT: "1"
956 + DOCKERHUB_USERNAME: "vllmbot"
957
958 # =============================================================================
959 # Publish to DockerHub and PyPI (at the end so all builds complete first)