| 1 | #!/bin/bash |
| 2 | # SPDX-License-Identifier: Apache-2.0 |
| 3 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 4 | # |
| 5 | # Check if Ray LLM can generate lock files that are compatible with this |
| 6 | # version of vllm. Downloads Ray's requirement files and runs a full |
| 7 | # dependency resolution with the installed vllm's constraints to see if |
| 8 | # a valid lock file can be produced. |
| 9 | # |
| 10 | # See: https://github.com/vllm-project/vllm/issues/33599 |
| 11 | |
| 12 | set -eo pipefail |
| 13 | |
| 14 | RAY_BASE_URL="https://raw.githubusercontent.com/ray-project/ray/master/python" |
| 15 | |
| 16 | WORK_DIR=$(mktemp -d) |
| 17 | trap 'rm -rf "$WORK_DIR"' EXIT |
| 18 | |
| 19 | # ── Detect PyTorch index URL ───────────────────────────────────────────── |
| 20 | |
| 21 | if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then |
| 22 | ROCM_VER=$(python3 -c "import torch; print(torch.version.hip.rsplit('.', 1)[0])") |
| 23 | CANDIDATE_URL="https://download.pytorch.org/whl/rocm${ROCM_VER}" |
| 24 | if curl -fsSL --head "${CANDIDATE_URL}/" >/dev/null 2>&1; then |
| 25 | TORCH_INDEX_URL="${CANDIDATE_URL}" |
| 26 | else |
| 27 | echo ">>> WARNING: ROCm ${ROCM_VER} wheel index not found at ${CANDIDATE_URL}" |
| 28 | echo ">>> Falling back to default PyPI (resolution may be incomplete)" |
| 29 | TORCH_INDEX_URL="" |
| 30 | fi |
| 31 | else |
| 32 | if [ "${TORCH_NIGHTLY:-0}" = "1" ]; then |
| 33 | TORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/cu130" |
| 34 | else |
| 35 | TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130" |
| 36 | fi |
| 37 | fi |
| 38 | echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}" |
| 39 | |
| 40 | # Fetch all Ray requirement files used in the LLM depset pipeline |
| 41 | echo ">>> Fetching Ray requirement files" |
| 42 | RAY_FILES=( |
| 43 | "requirements.txt" |
| 44 | "requirements/cloud-requirements.txt" |
| 45 | "requirements/base-test-requirements.txt" |
| 46 | "requirements/llm/llm-requirements.txt" |
| 47 | "requirements/llm/llm-test-requirements.txt" |
| 48 | ) |
| 49 | for FILE in "${RAY_FILES[@]}"; do |
| 50 | LOCAL_PATH="${WORK_DIR}/$(basename "$FILE")" |
| 51 | echo " ${FILE}" |
| 52 | curl -fsSL -o "$LOCAL_PATH" "${RAY_BASE_URL}/${FILE}" |
| 53 | done |
| 54 | |
| 55 | # Extract installed vllm deps |
| 56 | echo ">>> Extracting installed vllm dependency constraints" |
| 57 | python3 - "${WORK_DIR}/vllm-constraints.txt" <<'PYEOF' |
| 58 | """Write out the installed vllm's dependencies as pip constraint lines. |
| 59 | |
| 60 | Ray uses vllm[audio], so audio-extra deps are included with their extra |
| 61 | markers stripped. The resolver cannot evaluate extra markers for a |
| 62 | package that is not itself being resolved from an index, so we activate |
| 63 | them manually here. |
| 64 | """ |
| 65 | import importlib.metadata |
| 66 | import re |
| 67 | import sys |
| 68 | |
| 69 | out_path = sys.argv[1] |
| 70 | raw_reqs = importlib.metadata.requires("vllm") or [] |
| 71 | |
| 72 | # Ray uses vllm[audio] – activate that extra. |
| 73 | ACTIVE_EXTRAS = {"audio"} |
| 74 | EXTRA_RE = re.compile(r"""extra\s*==\s*['"]([^'"]+)['"]""") |
| 75 | |
| 76 | lines = [] |
| 77 | for r in raw_reqs: |
| 78 | if ";" not in r: |
| 79 | # Unconditional dep — always include. |
| 80 | lines.append(r.strip()) |
| 81 | continue |
| 82 | |
| 83 | req_part, _, marker_part = r.partition(";") |
| 84 | marker_part = marker_part.strip() |
| 85 | |
| 86 | extra_matches = EXTRA_RE.findall(marker_part) |
| 87 | if not extra_matches: |
| 88 | # Non-extra marker (python_version, etc.) — keep as-is. |
| 89 | lines.append(r.strip()) |
| 90 | continue |
| 91 | |
| 92 | if not ACTIVE_EXTRAS.intersection(extra_matches): |
| 93 | continue # Skip inactive extras (tensorizer, bench, …). |
| 94 | |
| 95 | # Strip the extra== conditions but keep any remaining markers |
| 96 | # (e.g. python_version). |
| 97 | cleaned = EXTRA_RE.sub("", marker_part) |
| 98 | cleaned = re.sub(r"\band\b\s*\band\b", "and", cleaned) |
| 99 | cleaned = re.sub(r"^\s*and\s+|\s+and\s*$", "", cleaned).strip() |
| 100 | |
| 101 | if cleaned: |
| 102 | lines.append(f"{req_part.strip()} ; {cleaned}") |
| 103 | else: |
| 104 | lines.append(req_part.strip()) |
| 105 | |
| 106 | with open(out_path, "w") as f: |
| 107 | for line in lines: |
| 108 | f.write(line + "\n") |
| 109 | |
| 110 | print(f"Wrote {len(lines)} constraints to {out_path}") |
| 111 | PYEOF |
| 112 | |
| 113 | echo ">>> Installed vllm deps (first 20 lines):" |
| 114 | head -20 "${WORK_DIR}/vllm-constraints.txt" |
| 115 | |
| 116 | # Remove Ray's vllm pin — the installed vllm's transitive deps |
| 117 | # (written above) replace it in the resolution. vllm itself cannot |
| 118 | # be resolved from PyPI for in-development versions, so we test |
| 119 | # whether Ray's requirements can coexist with vllm's dependency |
| 120 | # constraints instead. |
| 121 | sed -i '/^vllm/d' "${WORK_DIR}/llm-requirements.txt" |
| 122 | |
| 123 | # Install uv if needed |
| 124 | if ! command -v uv &>/dev/null; then |
| 125 | echo ">>> Installing uv" |
| 126 | pip install uv -q |
| 127 | fi |
| 128 | |
| 129 | # Resolve: given vllm's constraints, can Ray compile a lock file? |
| 130 | # |
| 131 | # vllm's dependency constraints are the fixed side — Ray is flexible and |
| 132 | # can regenerate its lock files. We pass vllm's constraints via -c so |
| 133 | # the resolver treats them as non-negotiable bounds, then check whether |
| 134 | # Ray's own requirements can still be satisfied within those bounds. |
| 135 | echo "" |
| 136 | echo "============================================================" |
| 137 | echo ">>> Resolving: Can Ray generate compatible lock files?" |
| 138 | echo "============================================================" |
| 139 | |
| 140 | EXTRA_INDEX_ARGS=() |
| 141 | if [[ -n "${TORCH_INDEX_URL}" ]]; then |
| 142 | EXTRA_INDEX_ARGS+=(--extra-index-url "${TORCH_INDEX_URL}") |
| 143 | fi |
| 144 | |
| 145 | set +e |
| 146 | uv pip compile \ |
| 147 | "${WORK_DIR}/requirements.txt" \ |
| 148 | "${WORK_DIR}/cloud-requirements.txt" \ |
| 149 | "${WORK_DIR}/base-test-requirements.txt" \ |
| 150 | "${WORK_DIR}/llm-requirements.txt" \ |
| 151 | "${WORK_DIR}/llm-test-requirements.txt" \ |
| 152 | -c "${WORK_DIR}/vllm-constraints.txt" \ |
| 153 | --python-version 3.12 \ |
| 154 | --python-platform x86_64-manylinux_2_31 \ |
| 155 | "${EXTRA_INDEX_ARGS[@]}" \ |
| 156 | --index-strategy unsafe-best-match \ |
| 157 | --unsafe-package setuptools \ |
| 158 | --unsafe-package ray \ |
| 159 | --no-header \ |
| 160 | -o "${WORK_DIR}/resolved.txt" \ |
| 161 | 2>&1 |
| 162 | EXIT_CODE=$? |
| 163 | set -e |
| 164 | |
| 165 | echo "" |
| 166 | echo "==========================================" |
| 167 | if [ $EXIT_CODE -eq 0 ]; then |
| 168 | echo "SUCCESS: Ray can generate lock files compatible with this vllm." |
| 169 | echo "" |
| 170 | echo "Key resolved versions:" |
| 171 | grep -E '^(protobuf|torch|numpy|transformers)==' \ |
| 172 | "${WORK_DIR}/resolved.txt" | sort || true |
| 173 | echo "==========================================" |
| 174 | exit 0 |
| 175 | fi |
| 176 | |
| 177 | echo "FAILURE: Ray cannot generate lock files compatible with this vllm." |
| 178 | echo "This means a fundamental dependency conflict exists that Ray" |
| 179 | echo "cannot resolve by regenerating its lock files." |
| 180 | echo "See: https://github.com/vllm-project/vllm/issues/33599" |
| 181 | echo "==========================================" |
| 182 | |
| 183 | # Buildkite annotation |
| 184 | if [ -f /usr/bin/buildkite-agent ]; then |
| 185 | buildkite-agent annotate --style 'warning' --context 'ray-compat' << EOF |
| 186 | ### :warning: Ray Dependency Compatibility Warning |
| 187 | This PR introduces dependencies that **cannot** be resolved with Ray's requirements. |
| 188 | Ray would not be able to regenerate its lock files to accommodate this vllm version. |
| 189 | |
| 190 | Please check the **Ray Dependency Compatibility Check** step logs for details. |
| 191 | See [issue #33599](https://github.com/vllm-project/vllm/issues/33599) for context. |
| 192 | EOF |
| 193 | fi |
| 194 | |
| 195 | # Notify Slack if webhook is configured and PR/branch are valid. |
| 196 | if [ -n "$RAY_COMPAT_SLACK_WEBHOOK_URL" ]; then |
| 197 | PR="${BUILDKITE_PULL_REQUEST:-}" |
| 198 | BRANCH="${BUILDKITE_BRANCH:-}" |
| 199 | |
| 200 | # Skip notification if PR is invalid or branch is empty |
| 201 | if [[ "$PR" = "false" || -z "$PR" || -z "$BRANCH" ]]; then |
| 202 | echo ">>> Skipping Slack notification (invalid PR or empty branch: PR=$PR, branch=$BRANCH)" |
| 203 | else |
| 204 | echo ">>> Sending Slack notification" |
| 205 | # Single quotes are intentional: the f-string expressions are Python, not shell. |
| 206 | # shellcheck disable=SC2016 |
| 207 | PAYLOAD=$(python3 -c ' |
| 208 | import json, os, sys |
| 209 | pr = os.getenv("BUILDKITE_PULL_REQUEST", "N/A") |
| 210 | branch = os.getenv("BUILDKITE_BRANCH", "unknown") |
| 211 | url = os.getenv("BUILDKITE_BUILD_URL", "#") |
| 212 | data = { |
| 213 | "text": ":warning: Ray Dependency Compatibility Check Failed", |
| 214 | "blocks": [{ |
| 215 | "type": "section", |
| 216 | "text": { |
| 217 | "type": "mrkdwn", |
| 218 | "text": ( |
| 219 | "*:warning: Ray Dependency Compatibility Check Failed*\n" |
| 220 | f"PR #{pr} on branch `{branch}` introduces dependencies " |
| 221 | f"that cannot be resolved with Ray'\''s requirements.\n" |
| 222 | f"<{url}|View Build>" |
| 223 | ), |
| 224 | }, |
| 225 | }], |
| 226 | } |
| 227 | print(json.dumps(data)) |
| 228 | ') |
| 229 | |
| 230 | HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST "$RAY_COMPAT_SLACK_WEBHOOK_URL" \ |
| 231 | -H 'Content-type: application/json' \ |
| 232 | -d "$PAYLOAD") |
| 233 | echo " Slack webhook response: $HTTP_CODE" |
| 234 | fi |
| 235 | else |
| 236 | echo ">>> Skipping Slack notification (RAY_COMPAT_SLACK_WEBHOOK_URL not set)" |
| 237 | fi |
| 238 | |
| 239 | exit 1 |