@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4b612664fd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 06:00 UTC

[CI] Renovation of nightly wheel build & generation (take 2) (#29838)

Signed-off-by: Shengqi Chen <harry-chen@outlook.com>

Shengqi Chen committed Dec 2, 2025 at 14:17 UTC 4b612664fdfb4e87af6684403872d83ac04fa496
9 files changed +606 -148
.buildkite/release-pipeline.yaml
+1 -15
@@ -8,7 +8,7 @@ steps:
8 commands:
9 # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
10 # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
11 - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg VLLM_MAIN_CUDA_VERSION=12.9 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
11 + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
12 - "mkdir artifacts"
13 - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
14 - "bash .buildkite/scripts/upload-wheels.sh"
@@ -30,19 +30,6 @@ steps:
30 DOCKER_BUILDKIT: "1"
31
32 # x86 + CUDA builds
33 - - label: "Build wheel - CUDA 12.8"
34 - depends_on: ~
35 - id: build-wheel-cuda-12-8
36 - agents:
37 - queue: cpu_queue_postmerge
38 - commands:
39 - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.8.1 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
40 - - "mkdir artifacts"
41 - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
42 - - "bash .buildkite/scripts/upload-wheels.sh"
43 - env:
44 - DOCKER_BUILDKIT: "1"
45 -
33 - label: "Build wheel - CUDA 12.9"
34 depends_on: ~
35 id: build-wheel-cuda-12-9
@@ -109,7 +96,6 @@ steps:
96 - label: "Annotate release workflow"
97 depends_on:
98 - create-multi-arch-manifest
112 - - build-wheel-cuda-12-8
99 id: annotate-release-workflow
100 agents:
101 queue: cpu_queue_postmerge
.buildkite/scripts/generate-nightly-index.py new
+369
@@ -0,0 +1,369 @@
1 +#!/usr/bin/env python3
2 +# SPDX-License-Identifier: Apache-2.0
3 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4 +
5 +# do not complain about line length (for docstring)
6 +# ruff: noqa: E501
7 +
8 +import argparse
9 +import json
10 +import re
11 +import sys
12 +from dataclasses import asdict, dataclass
13 +from pathlib import Path
14 +from typing import Any
15 +from urllib.parse import quote
16 +
17 +if not sys.version_info >= (3, 12):
18 + raise RuntimeError("This script requires Python 3.12 or higher.")
19 +
20 +INDEX_HTML_TEMPLATE = """<!DOCTYPE html>
21 +<html>
22 + <meta name="pypi:repository-version" content="1.0">
23 + <body>
24 +{items}
25 + </body>
26 +</html>
27 +"""
28 +
29 +
30 +@dataclass
31 +class WheelFileInfo:
32 + package_name: str
33 + version: str
34 + build_tag: str | None
35 + python_tag: str
36 + abi_tag: str
37 + platform_tag: str
38 + variant: str | None
39 + filename: str
40 +
41 +
42 +def parse_from_filename(file: str) -> WheelFileInfo:
43 + """
44 + Parse wheel file name to extract metadata.
45 +
46 + The format of wheel names:
47 + {package_name}-{version}(-{build_tag})?-{python_tag}-{abi_tag}-{platform_tag}.whl
48 + All versions could contain a variant like '+cu129' or '.cpu' or `.rocm` (or not).
49 + Example:
50 + vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
51 + vllm-0.10.2rc2+cu129-cp38-abi3-manylinux2014_aarch64.whl
52 + vllm-0.11.1rc8.dev14+gaa384b3c0-cp38-abi3-manylinux2014_aarch64.whl
53 + vllm-0.11.1rc8.dev14+gaa384b3c0.cu130-cp38-abi3-manylinux1_x86_64.whl
54 + """
55 + wheel_file_re = re.compile(
56 + r"^(?P<package_name>.+)-(?P<version>[^-]+?)(-(?P<build_tag>[^-]+))?-(?P<python_tag>[^-]+)-(?P<abi_tag>[^-]+)-(?P<platform_tag>[^-]+)\.whl$"
57 + )
58 + match = wheel_file_re.match(file)
59 + if not match:
60 + raise ValueError(f"Invalid wheel file name: {file}")
61 +
62 + package_name = match.group("package_name")
63 + version = match.group("version")
64 + build_tag = match.group("build_tag")
65 + python_tag = match.group("python_tag")
66 + abi_tag = match.group("abi_tag")
67 + platform_tag = match.group("platform_tag")
68 +
69 + # extract variant from version
70 + variant = None
71 + if "dev" in version:
72 + ver_after_dev = version.split("dev")[-1]
73 + if "." in ver_after_dev:
74 + variant = ver_after_dev.split(".")[-1]
75 + version = version.removesuffix("." + variant)
76 + else:
77 + if "+" in version:
78 + version, variant = version.split("+")
79 +
80 + return WheelFileInfo(
81 + package_name=package_name,
82 + version=version,
83 + build_tag=build_tag,
84 + python_tag=python_tag,
85 + abi_tag=abi_tag,
86 + platform_tag=platform_tag,
87 + variant=variant,
88 + filename=file,
89 + )
90 +
91 +
92 +def generate_project_list(subdir_names: list[str]) -> str:
93 + """
94 + Generate project list HTML content linking to each project & variant sub-directory.
95 + """
96 + href_tags = []
97 + for name in sorted(subdir_names):
98 + name = name.strip("/").strip(".")
99 + href_tags.append(f' <a href="{name}/">{name}/</a><br/>')
100 + return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags))
101 +
102 +
103 +def generate_package_index_and_metadata(
104 + wheel_files: list[WheelFileInfo], wheel_base_dir: Path, index_base_dir: Path
105 +) -> tuple[str, str]:
106 + """
107 + Generate package index HTML content for a specific package, linking to actual wheel files.
108 + """
109 + href_tags = []
110 + metadata = []
111 + for file in sorted(wheel_files, key=lambda x: x.filename):
112 + relative_path = (
113 + wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename
114 + )
115 + # handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B'
116 + # NOTE: this is AWS S3 specific behavior!
117 + file_path_quoted = quote(relative_path.as_posix(), safe=":%/")
118 + href_tags.append(f' <a href="{file_path_quoted}">{file.filename}</a><br/>')
119 + file_meta = asdict(file)
120 + file_meta["path"] = file_path_quoted
121 + metadata.append(file_meta)
122 + index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags))
123 + metadata_str = json.dumps(metadata, indent=2)
124 + return index_str, metadata_str
125 +
126 +
127 +def generate_index_and_metadata(
128 + whl_files: list[str],
129 + wheel_base_dir: Path,
130 + index_base_dir: Path,
131 + default_variant: str | None = None,
132 + alias_to_default: str | None = None,
133 +):
134 + """
135 + Generate index for all wheel files.
136 +
137 + Args:
138 + whl_files (list[str]): List of wheel files (must be directly under `wheel_base_dir`).
139 + wheel_base_dir (Path): Base directory for wheel files.
140 + index_base_dir (Path): Base directory to store index files.
141 + default_variant (str | None): The default variant name, if any.
142 + alias_to_default (str | None): Alias variant name for the default variant, if any.
143 +
144 + First, parse all wheel files to extract metadata.
145 + We need to collect all wheel files for each variant, and generate an index for it (in a sub-directory).
146 + The index for the default variant (if any) is generated in the root index directory.
147 +
148 + If `default_variant` is provided, all wheels must have variant suffixes, and the default variant index
149 + is purely a copy of the corresponding variant index, with only the links adjusted.
150 + Otherwise, all wheels without variant suffixes are treated as the default variant.
151 +
152 + If `alias_to_default` is provided, an additional alias sub-directory is created, it has the same content
153 + as the default variant index, but the links are adjusted accordingly.
154 +
155 + Index directory structure:
156 + index_base_dir/ (hosted at wheels.vllm.ai/{nightly,$commit,$version}/)
157 + index.html # project list, linking to "vllm/" and other packages, and all variant sub-directories
158 + vllm/
159 + index.html # package index, pointing to actual files in wheel_base_dir (relative path)
160 + metadata.json # machine-readable metadata for all wheels in this package
161 + cpu/ # cpu variant sub-directory
162 + index.html
163 + vllm/
164 + index.html
165 + metadata.json
166 + cu129/ # cu129 is actually the alias to default variant
167 + index.html
168 + vllm/
169 + index.html
170 + metadata.json
171 + cu130/ # cu130 variant sub-directory
172 + index.html
173 + vllm/
174 + index.html
175 + metadata.json
176 + ...
177 +
178 + metadata.json stores a dump of all wheel files' metadata in a machine-readable format:
179 + [
180 + {
181 + "package_name": "vllm",
182 + "version": "0.10.2rc2",
183 + "build_tag": null,
184 + "python_tag": "cp38",
185 + "abi_tag": "abi3",
186 + "platform_tag": "manylinux2014_aarch64",
187 + "variant": "cu129",
188 + "filename": "vllm-0.10.2rc2+cu129-cp38-abi3-manylinux2014_aarch64.whl",
189 + "path": "../vllm-0.10.2rc2%2Bcu129-cp38-abi3-manylinux2014_aarch64.whl" # to be concatenated with the directory URL and URL-encoded
190 + },
191 + ...
192 + ]
193 + """
194 +
195 + parsed_files = [parse_from_filename(f) for f in whl_files]
196 +
197 + if not parsed_files:
198 + print("No wheel files found, skipping index generation.")
199 + return
200 +
201 + # Group by variant
202 + variant_to_files: dict[str, list[WheelFileInfo]] = {}
203 + for file in parsed_files:
204 + variant = file.variant or "default"
205 + if variant not in variant_to_files:
206 + variant_to_files[variant] = []
207 + variant_to_files[variant].append(file)
208 +
209 + print(f"Found variants: {list(variant_to_files.keys())}")
210 +
211 + # sanity check for default variant
212 + if default_variant:
213 + if "default" in variant_to_files:
214 + raise ValueError(
215 + "All wheel files must have variant suffixes when `default_variant` is specified."
216 + )
217 + if default_variant not in variant_to_files:
218 + raise ValueError(
219 + f"Default variant '{default_variant}' not found among wheel files."
220 + )
221 +
222 + if alias_to_default:
223 + if "default" not in variant_to_files:
224 + # e.g. only some wheels are uploaded to S3 currently
225 + print(
226 + "[WARN] Alias to default variant specified, but no default variant found."
227 + )
228 + elif alias_to_default in variant_to_files:
229 + raise ValueError(
230 + f"Alias variant name '{alias_to_default}' already exists among wheel files."
231 + )
232 + else:
233 + variant_to_files[alias_to_default] = variant_to_files["default"].copy()
234 + print(f"Alias variant '{alias_to_default}' created for default variant.")
235 +
236 + # Generate index for each variant
237 + subdir_names = set()
238 + for variant, files in variant_to_files.items():
239 + if variant == "default":
240 + variant_dir = index_base_dir
241 + else:
242 + variant_dir = index_base_dir / variant
243 + subdir_names.add(variant)
244 +
245 + variant_dir.mkdir(parents=True, exist_ok=True)
246 +
247 + # gather all package names in this variant
248 + packages = set(f.package_name for f in files)
249 + if variant == "default":
250 + # these packages should also appear in the "project list"
251 + # generate after all variants are processed
252 + subdir_names = subdir_names.union(packages)
253 + else:
254 + # generate project list for this variant directly
255 + project_list_str = generate_project_list(sorted(packages))
256 + with open(variant_dir / "index.html", "w") as f:
257 + f.write(project_list_str)
258 +
259 + for package in packages:
260 + # filter files belonging to this package only
261 + package_files = [f for f in files if f.package_name == package]
262 + package_dir = variant_dir / package
263 + package_dir.mkdir(parents=True, exist_ok=True)
264 + index_str, metadata_str = generate_package_index_and_metadata(
265 + package_files, wheel_base_dir, package_dir
266 + )
267 + with open(package_dir / "index.html", "w") as f:
268 + f.write(index_str)
269 + with open(package_dir / "metadata.json", "w") as f:
270 + f.write(metadata_str)
271 +
272 + # Generate top-level project list index
273 + project_list_str = generate_project_list(sorted(subdir_names))
274 + with open(index_base_dir / "index.html", "w") as f:
275 + f.write(project_list_str)
276 +
277 +
278 +if __name__ == "__main__":
279 + """
280 + Arguments:
281 + --version <version> : version string for the current build (e.g., commit hash)
282 + --current-objects <path_to_json> : path to JSON file containing current S3 objects listing in this version directory
283 + --output-dir <output_directory> : directory to store generated index files
284 + --alias-to-default <alias_variant_name> : (optional) alias variant name for the default variant
285 + """
286 +
287 + parser = argparse.ArgumentParser(
288 + description="Process nightly build wheel files to generate indices."
289 + )
290 + parser.add_argument(
291 + "--version",
292 + type=str,
293 + required=True,
294 + help="Version string for the current build (e.g., commit hash)",
295 + )
296 + parser.add_argument(
297 + "--current-objects",
298 + type=str,
299 + required=True,
300 + help="Path to JSON file containing current S3 objects listing in this version directory",
301 + )
302 + parser.add_argument(
303 + "--output-dir",
304 + type=str,
305 + required=True,
306 + help="Directory to store generated index files",
307 + )
308 + parser.add_argument(
309 + "--alias-to-default",
310 + type=str,
311 + default=None,
312 + help="Alias variant name for the default variant",
313 + )
314 +
315 + args = parser.parse_args()
316 +
317 + version = args.version
318 + if "/" in version or "\\" in version:
319 + raise ValueError("Version string must not contain slashes.")
320 + current_objects_path = Path(args.current_objects)
321 + output_dir = Path(args.output_dir)
322 + if not output_dir.exists():
323 + output_dir.mkdir(parents=True, exist_ok=True)
324 +
325 + # Read current objects JSON
326 + with open(current_objects_path) as f:
327 + current_objects: dict[str, list[dict[str, Any]]] = json.load(f)
328 +
329 + # current_objects looks like from list_objects_v2 S3 API:
330 + """
331 + "Contents": [
332 + {
333 + "Key": "e2f56c309d2a28899c68975a7e104502d56deb8f/vllm-0.11.2.dev363+ge2f56c309-cp38-abi3-manylinux1_x86_64.whl",
334 + "LastModified": "2025-11-28T14:00:32+00:00",
335 + "ETag": "\"37a38339c7cdb61ca737021b968075df-52\"",
336 + "ChecksumAlgorithm": [
337 + "CRC64NVME"
338 + ],
339 + "ChecksumType": "FULL_OBJECT",
340 + "Size": 435649349,
341 + "StorageClass": "STANDARD"
342 + },
343 + ...
344 + ]
345 + """
346 +
347 + # Extract wheel file keys
348 + wheel_files = []
349 + for item in current_objects.get("Contents", []):
350 + key: str = item["Key"]
351 + if key.endswith(".whl"):
352 + wheel_files.append(key.split("/")[-1]) # only the filename is used
353 +
354 + print(f"Found {len(wheel_files)} wheel files for version {version}: {wheel_files}")
355 +
356 + # Generate index and metadata, assuming wheels and indices are stored as:
357 + # s3://vllm-wheels/{version}/<wheel files>
358 + # s3://vllm-wheels/<anything>/<index files>
359 + wheel_base_dir = Path(output_dir).parent / version
360 + index_base_dir = Path(output_dir)
361 +
362 + generate_index_and_metadata(
363 + whl_files=wheel_files,
364 + wheel_base_dir=wheel_base_dir,
365 + index_base_dir=index_base_dir,
366 + default_variant=None,
367 + alias_to_default=args.alias_to_default,
368 + )
369 + print(f"Successfully generated index and metadata in {output_dir}")
.buildkite/scripts/upload-wheels.sh
+69 -52
@@ -2,6 +2,28 @@
2
3 set -ex
4
5 +# ======== part 0: setup ========
6 +
7 +BUCKET="vllm-wheels"
8 +INDICES_OUTPUT_DIR="indices"
9 +DEFAULT_VARIANT_ALIAS="cu129" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py
10 +PYTHON=${PYTHON_PROG:=python3} # try to read from env var, otherwise use python3
11 +SUBPATH=$BUILDKITE_COMMIT
12 +S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
13 +
14 +# detect if python3.10+ is available
15 +has_new_python=$($PYTHON -c "print(1 if __import__('sys').version_info >= (3,12) else 0)")
16 +if [[ "$has_new_python" -eq 0 ]]; then
17 + # use new python from docker
18 + docker pull python:3-slim
19 + PYTHON="docker run --rm -v $(pwd):/app -w /app python:3-slim python3"
20 +fi
21 +
22 +echo "Using python interpreter: $PYTHON"
23 +echo "Python version: $($PYTHON --version)"
24 +
25 +# ========= part 1: collect, rename & upload the wheel ==========
26 +
27 # Assume wheels are in artifacts/dist/*.whl
28 wheel_files=(artifacts/dist/*.whl)
29
@@ -10,74 +32,69 @@ if [[ ${#wheel_files[@]} -ne 1 ]]; then
32 echo "Error: Expected exactly one wheel file in artifacts/dist/, but found ${#wheel_files[@]}"
33 exit 1
34 fi
13 -
14 -# Get the single wheel file
35 wheel="${wheel_files[0]}"
36
17 -# Detect architecture and rename 'linux' to appropriate manylinux version
18 -arch=$(uname -m)
19 -if [[ $arch == "x86_64" ]]; then
20 - manylinux_version="manylinux1"
21 -elif [[ $arch == "aarch64" ]]; then
22 - manylinux_version="manylinux2014"
23 -else
24 - echo "Warning: Unknown architecture $arch, using manylinux1 as default"
25 - manylinux_version="manylinux1"
26 -fi
37 +# current build image uses ubuntu 20.04, which corresponds to manylinux_2_31
38 +# refer to https://github.com/mayeut/pep600_compliance?tab=readme-ov-file#acceptable-distros-to-build-wheels
39 +manylinux_version="manylinux_2_31"
40
41 # Rename 'linux' to the appropriate manylinux version in the wheel filename
42 +if [[ "$wheel" != *"linux"* ]]; then
43 + echo "Error: Wheel filename does not contain 'linux': $wheel"
44 + exit 1
45 +fi
46 new_wheel="${wheel/linux/$manylinux_version}"
47 mv -- "$wheel" "$new_wheel"
48 wheel="$new_wheel"
49 +echo "Renamed wheel to: $wheel"
50
51 # Extract the version from the wheel
52 version=$(unzip -p "$wheel" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2)
35 -echo "Version: $version"
36 -
37 -normal_wheel="$wheel" # Save the original wheel filename
38 -
39 -# If the version contains "dev", rename it to v1.0.0.dev for consistency
40 -if [[ $version == *dev* ]]; then
41 - suffix="${version##*.}"
42 - if [[ $suffix == cu* ]]; then
43 - new_version="1.0.0.dev+${suffix}"
44 - else
45 - new_version="1.0.0.dev"
46 - fi
47 - new_wheel="${wheel/$version/$new_version}"
48 - # use cp to keep both files in the artifacts directory
49 - cp -- "$wheel" "$new_wheel"
50 - wheel="$new_wheel"
51 - version="$new_version"
52 -fi
53 +echo "Version in wheel: $version"
54 +pure_version="${version%%+*}"
55 +echo "Pure version (without variant): $pure_version"
56
54 -# Upload the wheel to S3
55 -python3 .buildkite/generate_index.py --wheel "$normal_wheel"
57 +# copy wheel to its own bucket
58 +aws s3 cp "$wheel" "$S3_COMMIT_PREFIX"
59
57 -# generate index for this commit
58 -aws s3 cp "$wheel" "s3://vllm-wheels/$BUILDKITE_COMMIT/"
59 -aws s3 cp "$normal_wheel" "s3://vllm-wheels/$BUILDKITE_COMMIT/"
60 +# ========= part 2: generate and upload indices ==========
61 +# generate indices for all existing wheels in the commit directory
62 +# this script might be run multiple times if there are multiple variants being built
63 +# so we need to guarantee there is little chance for "TOCTOU" issues
64 +# i.e., one process is generating indices while another is uploading a new wheel
65 +# so we need to ensure no time-consuming operations happen below
66
61 -if [[ $normal_wheel == *"cu129"* ]]; then
62 - # only upload index.html for cu129 wheels (default wheels) as it
63 - # is available on both x86 and arm64
64 - aws s3 cp index.html "s3://vllm-wheels/$BUILDKITE_COMMIT/vllm/index.html"
65 - aws s3 cp "s3://vllm-wheels/nightly/index.html" "s3://vllm-wheels/$BUILDKITE_COMMIT/index.html"
67 +# list all wheels in the commit directory
68 +echo "Existing wheels on S3:"
69 +aws s3 ls "$S3_COMMIT_PREFIX"
70 +obj_json="objects.json"
71 +aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$SUBPATH/" --delimiter / --output json > "$obj_json"
72 +mkdir -p "$INDICES_OUTPUT_DIR"
73 +
74 +# call script to generate indicies for all existing wheels
75 +# this indices have relative paths that could work as long as it is next to the wheel directory in s3
76 +# i.e., the wheels are always in s3://vllm-wheels/<commit>/
77 +# and indices can be placed in /<commit>/, or /nightly/, or /<version>/
78 +if [[ ! -z "$DEFAULT_VARIANT_ALIAS" ]]; then
79 + alias_arg="--alias-to-default $DEFAULT_VARIANT_ALIAS"
80 else
67 - echo "Skipping index files for non-cu129 wheels"
81 + alias_arg=""
82 fi
83
70 -# generate index for nightly
71 -aws s3 cp "$wheel" "s3://vllm-wheels/nightly/"
72 -aws s3 cp "$normal_wheel" "s3://vllm-wheels/nightly/"
84 +$PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" $alias_arg
85
74 -if [[ $normal_wheel == *"cu129"* ]]; then
75 - # only upload index.html for cu129 wheels (default wheels) as it
76 - # is available on both x86 and arm64
77 - aws s3 cp index.html "s3://vllm-wheels/nightly/vllm/index.html"
78 -else
79 - echo "Skipping index files for non-cu129 wheels"
86 +# copy indices to /<commit>/ unconditionally
87 +echo "Uploading indices to $S3_COMMIT_PREFIX"
88 +aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
89 +
90 +# copy to /nightly/ only if it is on the main branch and not a PR
91 +if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]]; then
92 + echo "Uploading indices to overwrite /nightly/"
93 + aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/nightly/"
94 fi
95
82 -aws s3 cp "$wheel" "s3://vllm-wheels/$version/"
83 -aws s3 cp index.html "s3://vllm-wheels/$version/vllm/index.html"
96 +# copy to /<pure_version>/ only if it does not have "dev" in the version
97 +if [[ "$version" != *"dev"* ]]; then
98 + echo "Uploading indices to overwrite /$pure_version/"
99 + aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/$pure_version/"
100 +fi
.buildkite/test-pipeline.yaml
+1 -2
@@ -215,7 +215,6 @@ steps:
215 timeout_in_minutes: 10
216 gpu: h100
217 num_gpus: 8
218 - optional: true
218 working_dir: "/vllm-workspace/tests"
219 source_file_dependencies:
220 - examples/offline_inference/torchrun_dp_example.py
@@ -1370,4 +1369,4 @@ steps:
1369 num_gpus: 2
1370 working_dir: "/vllm-workspace"
1371 commands:
1373 - - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
1372 + - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
\ No newline at end of file
docs/getting_started/installation/cpu.md
+14 -1
@@ -46,10 +46,23 @@ vLLM is a Python library that supports the following CPU variants. Select your C
46
47 ### Pre-built wheels
48
49 -Currently, there are no pre-built CPU wheels.
49 +Please refer to the instructions for [pre-built wheels on GPU](./gpu.md#pre-built-wheels).
50 +
51 +When specifying the index URL, please make sure to use the `cpu` variant subdirectory.
52 +For example, the nightly build index is: `https://wheels.vllm.ai/nightly/cpu/`.
53
54 ### Build wheel from source
55
56 +#### Set up using Python-only build (without compilation) {#python-only-build}
57 +
58 +Please refer to the instructions for [Python-only build on GPU](./gpu.md#python-only-build), and replace the build commands with:
59 +
60 +```bash
61 +VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cpu VLLM_TARGET_DEVICE=cpu uv pip install --editable .
62 +```
63 +
64 +#### Full build (with compilation) {#full-build}
65 +
66 === "Intel/AMD x86"
67
68 --8<-- "docs/getting_started/installation/cpu.x86.inc.md:build-wheel-from-source"
docs/getting_started/installation/gpu.cuda.inc.md
+33 -40
@@ -26,43 +26,50 @@ uv pip install vllm --torch-backend=auto
26
27 ??? console "pip"
28 ```bash
29 - # Install vLLM with CUDA 12.8.
30 - pip install vllm --extra-index-url https://download.pytorch.org/whl/cu128
29 + # Install vLLM with CUDA 12.9.
30 + pip install vllm --extra-index-url https://download.pytorch.org/whl/cu129
31 ```
32
33 -We recommend leveraging `uv` to [automatically select the appropriate PyTorch index at runtime](https://docs.astral.sh/uv/guides/integration/pytorch/#automatic-backend-selection) by inspecting the installed CUDA driver version via `--torch-backend=auto` (or `UV_TORCH_BACKEND=auto`). To select a specific backend (e.g., `cu126`), set `--torch-backend=cu126` (or `UV_TORCH_BACKEND=cu126`). If this doesn't work, try running `uv self update` to update `uv` first.
33 +We recommend leveraging `uv` to [automatically select the appropriate PyTorch index at runtime](https://docs.astral.sh/uv/guides/integration/pytorch/#automatic-backend-selection) by inspecting the installed CUDA driver version via `--torch-backend=auto` (or `UV_TORCH_BACKEND=auto`). To select a specific backend (e.g., `cu128`), set `--torch-backend=cu128` (or `UV_TORCH_BACKEND=cu128`). If this doesn't work, try running `uv self update` to update `uv` first.
34
35 !!! note
36 NVIDIA Blackwell GPUs (B200, GB200) require a minimum of CUDA 12.8, so make sure you are installing PyTorch wheels with at least that version. PyTorch itself offers a [dedicated interface](https://pytorch.org/get-started/locally/) to determine the appropriate pip command to run for a given target configuration.
37
38 -As of now, vLLM's binaries are compiled with CUDA 12.8 and public PyTorch release versions by default. We also provide vLLM binaries compiled with CUDA 12.6, 11.8, and public PyTorch release versions:
38 +As of now, vLLM's binaries are compiled with CUDA 12.9 and public PyTorch release versions by default. We also provide vLLM binaries compiled with CUDA 12.8, 13.0, and public PyTorch release versions:
39
40 ```bash
41 -# Install vLLM with a specific CUDA version (e.g., 11.8 or 12.6).
41 +# Install vLLM with a specific CUDA version (e.g., 13.0).
42 export VLLM_VERSION=$(curl -s https://api.github.com/repos/vllm-project/vllm/releases/latest | jq -r .tag_name | sed 's/^v//')
43 -export CUDA_VERSION=118 # or 126
44 -uv pip install https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cu${CUDA_VERSION}-cp38-abi3-manylinux1_x86_64.whl --extra-index-url https://download.pytorch.org/whl/cu${CUDA_VERSION}
43 +export CUDA_VERSION=130 # or other
44 +uv pip install https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cu${CUDA_VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl --extra-index-url https://download.pytorch.org/whl/cu${CUDA_VERSION}
45 ```
46
47 #### Install the latest code
48
49 -LLM inference is a fast-evolving field, and the latest code may contain bug fixes, performance improvements, and new features that are not released yet. To allow users to try the latest code without waiting for the next release, vLLM provides wheels for Linux running on an x86 platform with CUDA 12 for every commit since `v0.5.3`.
49 +LLM inference is a fast-evolving field, and the latest code may contain bug fixes, performance improvements, and new features that are not released yet. To allow users to try the latest code without waiting for the next release, vLLM provides wheels for every commit since `v0.5.3` on <https://wheels.vllm.ai/nightly>. There are multiple indices that could be used:
50 +
51 +* `https://wheels.vllm.ai/nightly`: the default variant (CUDA with version specified in `VLLM_MAIN_CUDA_VERSION`) built with the last commit on the `main` branch. Currently it is CUDA 12.9.
52 +* `https://wheels.vllm.ai/nightly/<variant>`: all other variants. Now this includes `cu130`, and `cpu`. The default variant (`cu129`) also has a subdirectory to keep consistency.
53 +
54 +To install from nightly index, run:
55
56 ```bash
57 uv pip install -U vllm \
58 --torch-backend=auto \
54 - --extra-index-url https://wheels.vllm.ai/nightly
59 + --extra-index-url https://wheels.vllm.ai/nightly # add variant subdirectory here if needed
60 ```
61
57 -??? console "pip"
62 +!!! warning "`pip` caveat"
63 +
64 + Using `pip` to install from nightly indices is _not supported_, because `pip` combines packages from `--extra-index-url` and the default index, choosing only the latest version, which makes it difficult to install a development version prior to the released version. In contrast, `uv` gives the extra index [higher priority than the default index](https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes).
65 +
66 + If you insist on using `pip`, you have to specify the full URL of the wheel file (which can be obtained from the web page).
67 +
68 ```bash
59 - pip install -U vllm \
60 - --pre \
61 - --extra-index-url https://wheels.vllm.ai/nightly
69 + pip install -U https://wheels.vllm.ai/nightly/vllm-0.11.2.dev399%2Bg3c7461c18-cp38-abi3-manylinux_2_31_x86_64.whl # current nightly build (the filename will change!)
70 + pip install -U https://wheels.vllm.ai/${VLLM_COMMIT}/vllm-0.11.2.dev399%2Bg3c7461c18-cp38-abi3-manylinux_2_31_x86_64.whl # from specific commit
71 ```
72
64 - `--pre` is required for `pip` to consider pre-released versions.
65 -
73 ##### Install specific revisions
74
75 If you want to access the wheels for previous commits (e.g. to bisect the behavior change, performance regression), you can specify the commit hash in the URL:
@@ -71,33 +78,13 @@ If you want to access the wheels for previous commits (e.g. to bisect the behavi
78 export VLLM_COMMIT=72d9c316d3f6ede485146fe5aabd4e61dbc59069 # use full commit hash from the main branch
79 uv pip install vllm \
80 --torch-backend=auto \
74 - --extra-index-url https://wheels.vllm.ai/${VLLM_COMMIT}
81 + --extra-index-url https://wheels.vllm.ai/${VLLM_COMMIT} # add variant subdirectory here if needed
82 ```
83
77 -The `uv` approach works for vLLM `v0.6.6` and later and offers an easy-to-remember command. A unique feature of `uv` is that packages in `--extra-index-url` have [higher priority than the default index](https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes). If the latest public release is `v0.6.6.post1`, `uv`'s behavior allows installing a commit before `v0.6.6.post1` by specifying the `--extra-index-url`. In contrast, `pip` combines packages from `--extra-index-url` and the default index, choosing only the latest version, which makes it difficult to install a development version prior to the released version.
78 -
79 -??? note "pip"
80 - If you want to access the wheels for previous commits (e.g. to bisect the behavior change,
81 - performance regression), due to the limitation of `pip`, you have to specify the full URL of the
82 - wheel file by embedding the commit hash in the URL:
83 -
84 - ```bash
85 - export VLLM_COMMIT=33f460b17a54acb3b6cc0b03f4a17876cff5eafd # use full commit hash from the main branch
86 - pip install https://wheels.vllm.ai/${VLLM_COMMIT}/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl
87 - ```
88 -
89 - Note that the wheels are built with Python 3.8 ABI (see [PEP
90 - 425](https://peps.python.org/pep-0425/) for more details about ABI), so **they are compatible
91 - with Python 3.8 and later**. The version string in the wheel file name (`1.0.0.dev`) is just a
92 - placeholder to have a unified URL for the wheels, the actual versions of wheels are contained in
93 - the wheel metadata (the wheels listed in the extra index url have correct versions). Although we
94 - don't support Python 3.8 any more (because PyTorch 2.5 dropped support for Python 3.8), the
95 - wheels are still built with Python 3.8 ABI to keep the same wheel name as before.
96 -
84 # --8<-- [end:pre-built-wheels]
85 # --8<-- [start:build-wheel-from-source]
86
100 -#### Set up using Python-only build (without compilation)
87 +#### Set up using Python-only build (without compilation) {#python-only-build}
88
89 If you only need to change Python code, you can build and install vLLM without compilation. Using `uv pip`'s [`--editable` flag](https://docs.astral.sh/uv/pip/packages/#editable-packages), changes you make to the code will be reflected when you run vLLM:
90
@@ -121,18 +108,24 @@ This command will do the following:
108 In case you see an error about wheel not found when running the above command, it might be because the commit you based on in the main branch was just merged and the wheel is being built. In this case, you can wait for around an hour to try again, or manually assign the previous commit in the installation using the `VLLM_PRECOMPILED_WHEEL_LOCATION` environment variable.
109
110 ```bash
124 -export VLLM_COMMIT=72d9c316d3f6ede485146fe5aabd4e61dbc59069 # use full commit hash from the main branch
125 -export VLLM_PRECOMPILED_WHEEL_LOCATION=https://wheels.vllm.ai/${VLLM_COMMIT}/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl
111 +export VLLM_PRECOMPILED_WHEEL_COMMIT=$(git rev-parse HEAD~1) # or earlier commit on main
112 +export VLLM_USE_PRECOMPILED=1
113 uv pip install --editable .
114 ```
115
116 +There are more environment variables to control the behavior of Python-only build:
117 +
118 +* `VLLM_PRECOMPILED_WHEEL_LOCATION`: specify the exact wheel URL or local file path of a pre-compiled wheel to use. All other logic to find the wheel will be skipped.
119 +* `VLLM_PRECOMPILED_WHEEL_COMMIT`: override the commit hash to download the pre-compiled wheel. It can be `nightly` to use the last **already built** commit on the main branch.
120 +* `VLLM_PRECOMPILED_WHEEL_VARIANT`: specify the variant subdirectory to use on the nightly index, e.g., `cu129`, `cpu`. If not specified, the CUDA variant with `VLLM_MAIN_CUDA_VERSION` will be tried, then fallback to the default variant on the remote index.
121 +
122 You can find more information about vLLM's wheels in [Install the latest code](#install-the-latest-code).
123
124 !!! note
125 There is a possibility that your source code may have a different commit ID compared to the latest vLLM wheel, which could potentially lead to unknown errors.
126 It is recommended to use the same commit ID for the source code as the vLLM wheel you have installed. Please refer to [Install the latest code](#install-the-latest-code) for instructions on how to install a specified wheel.
127
135 -#### Full build (with compilation)
128 +#### Full build (with compilation) {#full-build}
129
130 If you want to modify C++ or CUDA code, you'll need to build vLLM from source. This can take several minutes:
131
docs/getting_started/installation/gpu.md
+1 -1
@@ -52,7 +52,7 @@ vLLM is a Python library that supports the following GPU variants. Select your G
52
53 --8<-- "docs/getting_started/installation/gpu.xpu.inc.md:set-up-using-python"
54
55 -### Pre-built wheels
55 +### Pre-built wheels {#pre-built-wheels}
56
57 === "NVIDIA CUDA"
58
setup.py
+115 -33
@@ -311,7 +311,7 @@ class precompiled_build_ext(build_ext):
311 """Disables extension building when using precompiled binaries."""
312
313 def run(self) -> None:
314 - assert _is_cuda(), "VLLM_USE_PRECOMPILED is only supported for CUDA builds"
314 + return
315
316 def build_extensions(self) -> None:
317 print("Skipping build_ext: using precompiled extensions.")
@@ -322,14 +322,121 @@ class precompiled_wheel_utils:
322 """Extracts libraries and other files from an existing wheel."""
323
324 @staticmethod
325 - def extract_precompiled_and_patch_package(wheel_url_or_path: str) -> dict:
325 + def fetch_metadata_for_variant(
326 + commit: str, variant: str | None
327 + ) -> tuple[list[dict], str]:
328 + """
329 + Fetches metadata for a specific variant of the precompiled wheel.
330 + """
331 + variant_dir = f"{variant}/" if variant is not None else ""
332 + repo_url = f"https://wheels.vllm.ai/{commit}/{variant_dir}vllm/"
333 + meta_url = repo_url + "metadata.json"
334 + print(f"Trying to fetch nightly build metadata from {meta_url}")
335 + from urllib.request import urlopen
336 +
337 + with urlopen(meta_url) as resp:
338 + # urlopen raises HTTPError on unexpected status code
339 + wheels = json.loads(resp.read().decode("utf-8"))
340 + return wheels, repo_url
341 +
342 + @staticmethod
343 + def determine_wheel_url() -> tuple[str, str | None]:
344 + """
345 + Try to determine the precompiled wheel URL or path to use.
346 + The order of preference is:
347 + 1. user-specified wheel location (can be either local or remote, via
348 + VLLM_PRECOMPILED_WHEEL_LOCATION)
349 + 2. user-specified variant from nightly repo (current main commit via
350 + VLLM_PRECOMPILED_WHEEL_VARIANT)
351 + 3. the variant corresponding to VLLM_MAIN_CUDA_VERSION from nightly repo
352 + 4. the default variant from nightly repo (current main commit)
353 + """
354 + wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION", None)
355 + if wheel_location is not None:
356 + print(f"Using user-specified precompiled wheel location: {wheel_location}")
357 + return wheel_location, None
358 + else:
359 + import platform
360 +
361 + arch = platform.machine()
362 + # try to fetch the wheel metadata from the nightly wheel repo
363 + main_variant = "cu" + envs.VLLM_MAIN_CUDA_VERSION.replace(".", "")
364 + variant = os.getenv("VLLM_PRECOMPILED_WHEEL_VARIANT", main_variant)
365 + commit = os.getenv(
366 + "VLLM_PRECOMPILED_WHEEL_COMMIT",
367 + precompiled_wheel_utils.get_base_commit_in_main_branch(),
368 + )
369 + print(f"Using precompiled wheel commit {commit} with variant {variant}")
370 + try_default = False
371 + wheels, repo_url, download_filename = None, None, None
372 + try:
373 + wheels, repo_url = precompiled_wheel_utils.fetch_metadata_for_variant(
374 + commit, variant
375 + )
376 + except Exception as e:
377 + logger.warning(
378 + "Failed to fetch precompiled wheel metadata for variant %s: %s",
379 + variant,
380 + e,
381 + )
382 + try_default = True # try outside handler to keep the stacktrace simple
383 + if try_default:
384 + print("Trying the default variant from remote")
385 + wheels, repo_url = precompiled_wheel_utils.fetch_metadata_for_variant(
386 + commit, None
387 + )
388 + # if this also fails, then we have nothing more to try / cache
389 + assert wheels is not None and repo_url is not None, (
390 + "Failed to fetch precompiled wheel metadata"
391 + )
392 + # The metadata.json has the following format:
393 + # see .buildkite/scripts/generate-nightly-index.py for details
394 + """[{
395 + "package_name": "vllm",
396 + "version": "0.11.2.dev278+gdbc3d9991",
397 + "build_tag": null,
398 + "python_tag": "cp38",
399 + "abi_tag": "abi3",
400 + "platform_tag": "manylinux1_x86_64",
401 + "variant": null,
402 + "filename": "vllm-0.11.2.dev278+gdbc3d9991-cp38-abi3-manylinux1_x86_64.whl",
403 + "path": "../vllm-0.11.2.dev278%2Bgdbc3d9991-cp38-abi3-manylinux1_x86_64.whl"
404 + },
405 + ...]"""
406 + from urllib.parse import urljoin
407 +
408 + for wheel in wheels:
409 + # TODO: maybe check more compatibility later? (python_tag, abi_tag, etc)
410 + if wheel.get("package_name") == "vllm" and arch in wheel.get(
411 + "platform_tag", ""
412 + ):
413 + print(f"Found precompiled wheel metadata: {wheel}")
414 + if "path" not in wheel:
415 + raise ValueError(f"Wheel metadata missing path: {wheel}")
416 + wheel_url = urljoin(repo_url, wheel["path"])
417 + download_filename = wheel.get("filename")
418 + print(f"Using precompiled wheel URL: {wheel_url}")
419 + break
420 + else:
421 + raise ValueError(
422 + f"No precompiled vllm wheel found for architecture {arch} "
423 + f"from repo {repo_url}. All available wheels: {wheels}"
424 + )
425 +
426 + return wheel_url, download_filename
427 +
428 + @staticmethod
429 + def extract_precompiled_and_patch_package(
430 + wheel_url_or_path: str, download_filename: str | None
431 + ) -> dict:
432 import tempfile
433 import zipfile
434
435 temp_dir = None
436 try:
437 if not os.path.isfile(wheel_url_or_path):
332 - wheel_filename = wheel_url_or_path.split("/")[-1]
438 + # use provided filename first, then derive from URL
439 + wheel_filename = download_filename or wheel_url_or_path.split("/")[-1]
440 temp_dir = tempfile.mkdtemp(prefix="vllm-wheels")
441 wheel_path = os.path.join(temp_dir, wheel_filename)
442 print(f"Downloading wheel from {wheel_url_or_path} to {wheel_path}")
@@ -648,38 +755,13 @@ package_data = {
755 ]
756 }
757
758 +
759 # If using precompiled, extract and patch package_data (in advance of setup)
760 if envs.VLLM_USE_PRECOMPILED:
653 - assert _is_cuda(), "VLLM_USE_PRECOMPILED is only supported for CUDA builds"
654 - wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION", None)
655 - if wheel_location is not None:
656 - wheel_url = wheel_location
657 - else:
658 - import platform
659 -
660 - arch = platform.machine()
661 - if arch == "x86_64":
662 - wheel_tag = "manylinux1_x86_64"
663 - elif arch == "aarch64":
664 - wheel_tag = "manylinux2014_aarch64"
665 - else:
666 - raise ValueError(f"Unsupported architecture: {arch}")
667 - base_commit = precompiled_wheel_utils.get_base_commit_in_main_branch()
668 - wheel_url = f"https://wheels.vllm.ai/{base_commit}/vllm-1.0.0.dev-cp38-abi3-{wheel_tag}.whl"
669 - nightly_wheel_url = (
670 - f"https://wheels.vllm.ai/nightly/vllm-1.0.0.dev-cp38-abi3-{wheel_tag}.whl"
671 - )
672 - from urllib.request import urlopen
673 -
674 - try:
675 - with urlopen(wheel_url) as resp:
676 - if resp.status != 200:
677 - wheel_url = nightly_wheel_url
678 - except Exception as e:
679 - print(f"[warn] Falling back to nightly wheel: {e}")
680 - wheel_url = nightly_wheel_url
681 -
682 - patch = precompiled_wheel_utils.extract_precompiled_and_patch_package(wheel_url)
761 + wheel_url, download_filename = precompiled_wheel_utils.determine_wheel_url()
762 + patch = precompiled_wheel_utils.extract_precompiled_and_patch_package(
763 + wheel_url, download_filename
764 + )
765 for pkg, files in patch.items():
766 package_data.setdefault(pkg, []).extend(files)
767
vllm/envs.py
+3 -4
@@ -74,7 +74,7 @@ if TYPE_CHECKING:
74 VLLM_MEDIA_CONNECTOR: str = "http"
75 VLLM_MM_INPUT_CACHE_GIB: int = 4
76 VLLM_TARGET_DEVICE: str = "cuda"
77 - VLLM_MAIN_CUDA_VERSION: str = "12.8"
77 + VLLM_MAIN_CUDA_VERSION: str = "12.9"
78 MAX_JOBS: str | None = None
79 NVCC_THREADS: str | None = None
80 VLLM_USE_PRECOMPILED: bool = False
@@ -445,10 +445,9 @@ environment_variables: dict[str, Callable[[], Any]] = {
445 # Target device of vLLM, supporting [cuda (by default),
446 # rocm, cpu]
447 "VLLM_TARGET_DEVICE": lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda").lower(),
448 - # Main CUDA version of vLLM, supporting [12.6, 12.8, 12.9],
449 - # 12.8 is the default. This follows PyTorch but can be overridden.
448 + # Main CUDA version of vLLM. This follows PyTorch but can be overridden.
449 "VLLM_MAIN_CUDA_VERSION": lambda: os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower()
451 - or "12.8",
450 + or "12.9",
451 # Maximum number of compilation jobs to run in parallel.
452 # By default this is the number of CPUs
453 "MAX_JOBS": lambda: os.getenv("MAX_JOBS", None),