| 1 | #!/usr/bin/env bash |
| 2 | # SPDX-License-Identifier: Apache-2.0 |
| 3 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 4 | # |
| 5 | # Generate S3 PyPI Root Index for Latest Version |
| 6 | # |
| 7 | # Creates a PEP 503 compatible index.html at rocm/ pointing to the latest |
| 8 | # semantic version's packages. This enables users to install with: |
| 9 | # uv pip install vllm --extra-index-url s3://vllm-wheels/rocm |
| 10 | # |
| 11 | # Usage: |
| 12 | # generate-root-index.sh [options] |
| 13 | # |
| 14 | # Options: |
| 15 | # --dry-run Preview changes without uploading |
| 16 | # --version VER Use specific version instead of auto-detecting latest |
| 17 | # |
| 18 | # Environment variables: |
| 19 | # S3_BUCKET - Bucket name (default: vllm-wheels) |
| 20 | # VARIANT - ROCm variant (default: rocm721) |
| 21 | # DRY_RUN - Set to 1 for preview mode (same as --dry-run) |
| 22 | |
| 23 | set -euo pipefail |
| 24 | |
| 25 | # ======== Configuration ======== |
| 26 | BUCKET="${S3_BUCKET:-vllm-wheels}" |
| 27 | VARIANT="${VARIANT:-rocm721}" |
| 28 | DRY_RUN="${DRY_RUN:-0}" |
| 29 | FORCE_VERSION="" |
| 30 | |
| 31 | # Parse command line arguments |
| 32 | while [[ $# -gt 0 ]]; do |
| 33 | case $1 in |
| 34 | --dry-run) |
| 35 | DRY_RUN=1 |
| 36 | shift |
| 37 | ;; |
| 38 | --version) |
| 39 | FORCE_VERSION="$2" |
| 40 | shift 2 |
| 41 | ;; |
| 42 | *) |
| 43 | echo "Unknown option: $1" |
| 44 | exit 1 |
| 45 | ;; |
| 46 | esac |
| 47 | done |
| 48 | |
| 49 | # Working directory for generated files |
| 50 | WORK_DIR=$(mktemp -d) |
| 51 | trap 'rm -rf "$WORK_DIR"' EXIT |
| 52 | |
| 53 | echo "========================================" |
| 54 | echo "Generate Root Index for Latest Version" |
| 55 | echo "========================================" |
| 56 | echo "S3 Bucket: $BUCKET" |
| 57 | echo "ROCm Variant: $VARIANT" |
| 58 | echo "Dry Run: $DRY_RUN" |
| 59 | echo "========================================" |
| 60 | echo "" |
| 61 | |
| 62 | # ======== Step 1: Find latest semantic version ======== |
| 63 | |
| 64 | echo "Step 1: Finding latest semantic version..." |
| 65 | |
| 66 | # List all directories under rocm/ |
| 67 | aws s3api list-objects-v2 \ |
| 68 | --bucket "$BUCKET" \ |
| 69 | --prefix "rocm/" \ |
| 70 | --delimiter "/" \ |
| 71 | --query 'CommonPrefixes[].Prefix' \ |
| 72 | --output text | tr '\t' '\n' > "$WORK_DIR/all_prefixes.txt" |
| 73 | |
| 74 | # Filter for semantic versions (x.y.z pattern) |
| 75 | grep -oE 'rocm/[0-9]+\.[0-9]+\.[0-9]+/' "$WORK_DIR/all_prefixes.txt" | \ |
| 76 | sed 's|rocm/||; s|/||' | \ |
| 77 | sort -V > "$WORK_DIR/versions.txt" || true |
| 78 | |
| 79 | if [[ ! -s "$WORK_DIR/versions.txt" ]]; then |
| 80 | echo "ERROR: No semantic versions found under s3://$BUCKET/rocm/" |
| 81 | exit 1 |
| 82 | fi |
| 83 | |
| 84 | echo "Found versions:" |
| 85 | cat "$WORK_DIR/versions.txt" |
| 86 | echo "" |
| 87 | |
| 88 | if [[ -n "$FORCE_VERSION" ]]; then |
| 89 | LATEST_VERSION="$FORCE_VERSION" |
| 90 | echo "Using forced version: $LATEST_VERSION" |
| 91 | else |
| 92 | LATEST_VERSION=$(tail -1 "$WORK_DIR/versions.txt") |
| 93 | echo "Latest version (auto-detected): $LATEST_VERSION" |
| 94 | fi |
| 95 | |
| 96 | # Verify the version exists |
| 97 | if ! grep -qx "$LATEST_VERSION" "$WORK_DIR/versions.txt"; then |
| 98 | echo "ERROR: Version $LATEST_VERSION not found in bucket" |
| 99 | exit 1 |
| 100 | fi |
| 101 | |
| 102 | # ======== Step 2: List packages from latest version ======== |
| 103 | |
| 104 | echo "" |
| 105 | echo "Step 2: Listing packages from rocm/$LATEST_VERSION/$VARIANT/..." |
| 106 | |
| 107 | VERSION_PREFIX="rocm/$LATEST_VERSION/$VARIANT/" |
| 108 | |
| 109 | # List package directories |
| 110 | aws s3api list-objects-v2 \ |
| 111 | --bucket "$BUCKET" \ |
| 112 | --prefix "$VERSION_PREFIX" \ |
| 113 | --delimiter "/" \ |
| 114 | --query 'CommonPrefixes[].Prefix' \ |
| 115 | --output text | tr '\t' '\n' > "$WORK_DIR/package_prefixes.txt" || true |
| 116 | |
| 117 | if [[ ! -s "$WORK_DIR/package_prefixes.txt" ]]; then |
| 118 | echo "ERROR: No packages found under s3://$BUCKET/$VERSION_PREFIX" |
| 119 | exit 1 |
| 120 | fi |
| 121 | |
| 122 | # Extract package names |
| 123 | sed "s|${VERSION_PREFIX}||; s|/||g" "$WORK_DIR/package_prefixes.txt" | \ |
| 124 | grep -v '^$' > "$WORK_DIR/packages.txt" |
| 125 | |
| 126 | echo "Found packages:" |
| 127 | cat "$WORK_DIR/packages.txt" |
| 128 | echo "" |
| 129 | |
| 130 | # ======== Step 3: Generate root index.html ======== |
| 131 | |
| 132 | echo "Step 3: Generating root index.html..." |
| 133 | |
| 134 | mkdir -p "$WORK_DIR/output" |
| 135 | |
| 136 | { |
| 137 | cat <<'EOF' |
| 138 | <!DOCTYPE html> |
| 139 | <html> |
| 140 | <head> |
| 141 | <meta name="pypi:repository-version" content="1.0"> |
| 142 | </head> |
| 143 | <body> |
| 144 | EOF |
| 145 | |
| 146 | while read -r pkg; do |
| 147 | echo " <a href=\"$pkg/\">$pkg</a><br>" |
| 148 | done < "$WORK_DIR/packages.txt" |
| 149 | |
| 150 | cat <<'EOF' |
| 151 | </body> |
| 152 | </html> |
| 153 | EOF |
| 154 | } > "$WORK_DIR/output/index.html" |
| 155 | |
| 156 | echo "Generated root index.html:" |
| 157 | cat "$WORK_DIR/output/index.html" |
| 158 | echo "" |
| 159 | |
| 160 | # ======== Step 4: Copy and adjust package index files ======== |
| 161 | |
| 162 | echo "Step 4: Copying and adjusting package index files..." |
| 163 | |
| 164 | while read -r pkg; do |
| 165 | echo "Processing package: $pkg" |
| 166 | |
| 167 | # Download existing index.html from versioned path |
| 168 | SOURCE_INDEX="s3://$BUCKET/$VERSION_PREFIX$pkg/index.html" |
| 169 | |
| 170 | mkdir -p "$WORK_DIR/output/$pkg" |
| 171 | |
| 172 | if aws s3 cp "$SOURCE_INDEX" "$WORK_DIR/output/$pkg/index.html" 2>/dev/null; then |
| 173 | # Adjust relative paths: |
| 174 | # Original: href="../../../{commit}/wheel.whl" (from rocm/0.13.0/rocm710/vllm/) |
| 175 | # New: href="../{commit}/wheel.whl" (from rocm/vllm/) |
| 176 | sed -i 's|href="\.\./\.\./\.\./|href="../|g' "$WORK_DIR/output/$pkg/index.html" |
| 177 | echo " - Downloaded and adjusted: $pkg/index.html" |
| 178 | else |
| 179 | echo " - WARNING: Could not download index for $pkg" |
| 180 | fi |
| 181 | done < "$WORK_DIR/packages.txt" |
| 182 | |
| 183 | echo "" |
| 184 | |
| 185 | # ======== Step 5: Upload to S3 ======== |
| 186 | |
| 187 | echo "Step 5: Uploading to s3://$BUCKET/rocm/..." |
| 188 | echo "" |
| 189 | |
| 190 | # List what would be uploaded |
| 191 | echo "Files to upload:" |
| 192 | find "$WORK_DIR/output" -name "*.html" -type f | while read -r file; do |
| 193 | rel_path="${file#"$WORK_DIR"/output/}" |
| 194 | echo " rocm/$rel_path" |
| 195 | done |
| 196 | echo "" |
| 197 | |
| 198 | if [[ "$DRY_RUN" == "1" ]]; then |
| 199 | echo "DRY RUN - Skipping upload" |
| 200 | echo "" |
| 201 | echo "Preview of generated files:" |
| 202 | echo "----------------------------------------" |
| 203 | echo "rocm/index.html:" |
| 204 | cat "$WORK_DIR/output/index.html" |
| 205 | echo "" |
| 206 | echo "----------------------------------------" |
| 207 | echo "Sample package index (first package):" |
| 208 | FIRST_PKG=$(head -1 "$WORK_DIR/packages.txt") |
| 209 | if [[ -f "$WORK_DIR/output/$FIRST_PKG/index.html" ]]; then |
| 210 | echo "rocm/$FIRST_PKG/index.html:" |
| 211 | cat "$WORK_DIR/output/$FIRST_PKG/index.html" |
| 212 | fi |
| 213 | else |
| 214 | # Upload all generated files |
| 215 | aws s3 cp --recursive "$WORK_DIR/output/" "s3://$BUCKET/rocm/" \ |
| 216 | --content-type "text/html" |
| 217 | |
| 218 | echo "Upload complete!" |
| 219 | fi |
| 220 | |
| 221 | # ======== Summary ======== |
| 222 | |
| 223 | echo "" |
| 224 | echo "========================================" |
| 225 | echo "Root Index Generation Complete!" |
| 226 | echo "========================================" |
| 227 | echo "" |
| 228 | echo "Latest version: $LATEST_VERSION" |
| 229 | echo "Packages indexed: $(wc -l < "$WORK_DIR/packages.txt")" |
| 230 | echo "" |
| 231 | echo "Install command:" |
| 232 | echo " uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/" |
| 233 | echo "========================================" |