@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f2ace1d57d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 3, 04:15 UTC

[Frontend][RFC] Rust front-end integration (#40848)

Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>

Nick Hill committed May 20, 2026 at 21:24 UTC f2ace1d57d28df8d4c5e973dd62d87f47d628cb3
35 files changed +893 -91
.buildkite/image_build/image_build.sh
+7
@@ -223,6 +223,13 @@ echo "CACHE_FROM_MAIN: ${CACHE_FROM_MAIN}"
223
224 check_and_skip_if_image_exists
225
226 +# The rust frontend lives in a git submodule under rust/. Buildkite's default
227 +# checkout does not recurse submodules, and the Dockerfile only sees what's in
228 +# the build context, so initialize the submodule here before invoking bake.
229 +echo "--- :git: Initializing git submodules"
230 +git submodule sync --recursive
231 +git submodule update --init --recursive
232 +
233 echo "--- :docker: Setting up Docker buildx bake"
234 echo "Target: ${TARGET}"
235 echo "vLLM bake file: ${VLLM_BAKE_FILE_PATH}"
.buildkite/image_build/image_build_cpu.sh
+6
@@ -21,6 +21,12 @@ else
21 exit 0
22 fi
23
24 +# The rust frontend lives in a git submodule under rust/. Buildkite's default
25 +# checkout does not recurse submodules, and the Dockerfile only sees what's in
26 +# the build context, so initialize the submodule here before building.
27 +git submodule sync --recursive
28 +git submodule update --init --recursive
29 +
30 # build
31 docker build --file docker/Dockerfile.cpu \
32 --build-arg max_jobs=16 \
.buildkite/image_build/image_build_cpu_arm64.sh
+6
@@ -21,6 +21,12 @@ else
21 exit 0
22 fi
23
24 +# The rust frontend lives in a git submodule under rust/. Buildkite's default
25 +# checkout does not recurse submodules, and the Dockerfile only sees what's in
26 +# the build context, so initialize the submodule here before building.
27 +git submodule sync --recursive
28 +git submodule update --init --recursive
29 +
30 # build
31 docker build --file docker/Dockerfile.cpu \
32 --build-arg max_jobs=16 \
.buildkite/image_build/image_build_hpu.sh
+1 -1
@@ -11,7 +11,7 @@ REPO=$2
11 BUILDKITE_COMMIT=$3
12
13 # authenticate with AWS ECR
14 -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
14 +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
15
16 # skip build if image already exists
17 if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then
.buildkite/image_build/image_build_xpu.sh
+2 -2
@@ -11,8 +11,8 @@ REPO=$2
11 BUILDKITE_COMMIT=$3
12
13 # authenticate with AWS ECR
14 -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
15 -aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com
14 +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
15 +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
16
17 # skip build if image already exists
18 if ! docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu &> /dev/null; then
.buildkite/test_areas/rust_frontend.yaml new
+107
@@ -0,0 +1,107 @@
1 +group: Rust Frontend
2 +depends_on:
3 + - image-build
4 +steps:
5 +- label: Rust Frontend OpenAI Coverage
6 + timeout_in_minutes: 90
7 + device: h200_18gb
8 + working_dir: "/vllm-workspace/tests"
9 + source_file_dependencies:
10 + - rust/
11 + - vllm/benchmarks/
12 + - vllm/entrypoints/openai/
13 + - vllm/entrypoints/serve/
14 + - vllm/v1/sample/
15 + - tests/utils.py
16 + - tests/benchmarks/test_serve_cli.py
17 + - tests/entrypoints/openai/chat_completion/test_chat_completion.py
18 + # - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
19 + # - tests/entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py
20 + # - tests/entrypoints/openai/completion/test_prompt_validation.py
21 + - tests/entrypoints/openai/completion/test_shutdown.py
22 + # - tests/entrypoints/openai/test_return_token_ids.py
23 + # - tests/entrypoints/openai/test_uds.py
24 + - tests/v1/sample/test_logprobs_e2e.py
25 + commands:
26 + - export VLLM_USE_RUST_FRONTEND=1
27 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
28 + - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
29 + - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py
30 + # - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"
31 + # - pytest -v -s entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py
32 + # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
33 + - pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
34 + # - pytest -v -s entrypoints/openai/test_return_token_ids.py
35 + # - pytest -v -s entrypoints/openai/test_uds.py
36 + - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
37 +
38 +- label: Rust Frontend Serve/Admin Coverage
39 + timeout_in_minutes: 60
40 + device: h200_18gb
41 + working_dir: "/vllm-workspace/tests"
42 + source_file_dependencies:
43 + - rust/
44 + - vllm/entrypoints/openai/
45 + - vllm/entrypoints/serve/
46 + - vllm/v1/engine/
47 + - tests/utils.py
48 + # - tests/entrypoints/rpc/test_collective_rpc.py
49 + - tests/entrypoints/serve/disagg/test_serving_tokens.py
50 + - tests/entrypoints/serve/instrumentator/test_basic.py
51 + - tests/entrypoints/serve/instrumentator/test_metrics.py
52 + # - tests/entrypoints/serve/instrumentator/test_sleep.py
53 + commands:
54 + - export VLLM_USE_RUST_FRONTEND=1
55 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
56 + # - pytest -v -s entrypoints/rpc/test_collective_rpc.py
57 + - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
58 + - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
59 + - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
60 + # - pytest -v -s entrypoints/serve/instrumentator/test_sleep.py
61 +
62 +- label: Rust Frontend Core Correctness
63 + timeout_in_minutes: 30
64 + device: h200_18gb
65 + working_dir: "/vllm-workspace/tests"
66 + source_file_dependencies:
67 + - rust/
68 + - vllm/entrypoints/openai/
69 + - tests/utils.py
70 + - tests/entrypoints/openai/correctness/test_lmeval.py
71 + commands:
72 + - export VLLM_USE_RUST_FRONTEND=1
73 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
74 + - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
75 +
76 +- label: Rust Frontend Tool Use
77 + timeout_in_minutes: 60
78 + working_dir: "/vllm-workspace/tests"
79 + source_file_dependencies:
80 + - rust/
81 + - vllm/entrypoints/openai/
82 + - vllm/tool_parsers/
83 + - tests/utils.py
84 + - tests/tool_use/
85 + commands:
86 + - export VLLM_USE_RUST_FRONTEND=1
87 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
88 + - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
89 +
90 +- label: Rust Frontend Distributed
91 + timeout_in_minutes: 30
92 + num_devices: 4
93 + working_dir: "/vllm-workspace/tests"
94 + source_file_dependencies:
95 + - rust/
96 + - vllm/distributed/
97 + - vllm/engine/
98 + - vllm/executor/
99 + - vllm/v1/engine/
100 + - vllm/v1/worker/
101 + - tests/utils.py
102 + - tests/v1/distributed/test_internal_lb_dp.py
103 + commands:
104 + - export VLLM_USE_RUST_FRONTEND=1
105 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
106 + - export NCCL_CUMEM_HOST_ENABLE=0
107 + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
.dockerignore
+2
@@ -2,6 +2,7 @@
2 /build
3 dist
4 vllm/*.so
5 +vllm/vllm-rs
6
7 # Byte-compiled / optimized / DLL files
8 __pycache__/
@@ -31,3 +32,4 @@ share/python-wheels/
32 .installed.cfg
33 *.egg
34 MANIFEST
35 +rust/target/
.gitignore
+3
@@ -26,6 +26,9 @@ __pycache__/
26 # C extensions
27 *.so
28
29 +# Rust binaries
30 +vllm/vllm-rs
31 +
32 # Distribution / packaging
33 .Python
34 build/
.gitmodules new
+3
@@ -0,0 +1,3 @@
1 +[submodule "rust"]
2 + path = rust
3 + url = https://github.com/Inferact/vllm-frontend-rs.git
build_rust.sh new
+43
@@ -0,0 +1,43 @@
1 +#!/bin/bash
2 +# Build the vllm-rs Rust frontend binary and install it into the vllm package.
3 +# Usage: ./build_rust.sh [--debug]
4 +#
5 +# By default builds in release mode. Pass --debug for faster compile times
6 +# during development.
7 +
8 +set -euo pipefail
9 +
10 +REPO_ROOT="$(cd "$(dirname "$0")" && pwd)"
11 +RUST_DIR="$REPO_ROOT/rust"
12 +TARGET_PATH="$REPO_ROOT/vllm/vllm-rs"
13 +
14 +# Read the required toolchain from rust-toolchain.toml.
15 +TOOLCHAIN=$(grep '^channel' "$RUST_DIR/rust-toolchain.toml" | sed 's/.*= *"\(.*\)"/\1/')
16 +
17 +# Ensure rustup and the required toolchain are available.
18 +if ! command -v rustup &>/dev/null; then
19 + echo "rustup not found, installing..."
20 + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none
21 + source "$HOME/.cargo/env"
22 +fi
23 +
24 +if ! rustup run "$TOOLCHAIN" rustc --version &>/dev/null; then
25 + echo "Installing Rust toolchain: $TOOLCHAIN"
26 + rustup toolchain install "$TOOLCHAIN"
27 +fi
28 +
29 +if [[ "${1:-}" == "--debug" ]]; then
30 + PROFILE_ARGS=()
31 + PROFILE_DIR="debug"
32 +else
33 + PROFILE_ARGS=(--release)
34 + PROFILE_DIR="release"
35 +fi
36 +
37 +cargo +"$TOOLCHAIN" build "${PROFILE_ARGS[@]}" \
38 + --manifest-path "$RUST_DIR/Cargo.toml" \
39 + --bin vllm-rs \
40 + --features native-tls-vendored
41 +
42 +cp "$RUST_DIR/target/$PROFILE_DIR/vllm-rs" "$TARGET_PATH"
43 +echo "Installed vllm-rs to $TARGET_PATH"
docker/Dockerfile
+62
@@ -231,6 +231,63 @@ ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0+PTX'
231 ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
232 #################### BUILD BASE IMAGE ####################
233
234 +#################### RUST BUILD IMAGE ####################
235 +# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the main wheel
236 +# build stage doesn't need the rust toolchain, protoc, or the rust source.
237 +# This stage runs in parallel with csrc-build/extensions-build.
238 +FROM ${BUILD_BASE_IMAGE} AS rust-build
239 +ARG BUILD_OS
240 +
241 +ENV DEBIAN_FRONTEND=noninteractive
242 +
243 +# Install protoc (used by tonic-build/prost-build) and a basic C toolchain
244 +# (some rust crates compile C in their build.rs scripts).
245 +RUN if [ "${BUILD_OS}" = "manylinux" ]; then \
246 + dnf install -y --setopt=install_weak_deps=False \
247 + ca-certificates curl git gcc gcc-c++ make \
248 + protobuf-compiler protobuf-devel \
249 + && dnf clean all && rm -rf /var/cache/dnf; \
250 + else \
251 + apt-get update -y \
252 + && apt-get install -y --no-install-recommends \
253 + ca-certificates curl git build-essential \
254 + protobuf-compiler libprotobuf-dev \
255 + && rm -rf /var/lib/apt/lists/*; \
256 + fi
257 +
258 +# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
259 +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
260 + sh -s -- -y --profile minimal --default-toolchain none
261 +ENV PATH="/root/.cargo/bin:${PATH}"
262 +
263 +WORKDIR /workspace
264 +
265 +# Copy only the rust workspace — the binary is the sole artifact we need.
266 +COPY rust rust
267 +
268 +# Fail loudly if the rust submodule was not initialized on the host before
269 +# `docker build`. Without this check, cargo would emit a confusing error.
270 +RUN if [ ! -f rust/Cargo.toml ]; then \
271 + echo "ERROR: rust/ submodule is not initialized."; \
272 + echo "Run 'git submodule update --init --recursive' on the host before building."; \
273 + exit 1; \
274 + fi
275 +
276 +# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
277 +# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
278 +ENV CARGO_BUILD_JOBS=4
279 +
280 +# Build the release binary. Cache cargo registry/git and target/, but copy the
281 +# binary out of the target/ cache mount so it persists into the image layer
282 +# for later COPY --from=rust-build.
283 +RUN --mount=type=cache,target=/root/.cargo/registry \
284 + --mount=type=cache,target=/root/.cargo/git \
285 + --mount=type=cache,target=/workspace/rust/target \
286 + cd rust \
287 + && cargo build --release --bin vllm-rs --features native-tls-vendored \
288 + && cp target/release/vllm-rs /workspace/vllm-rs
289 +#################### RUST BUILD IMAGE ####################
290 +
291 #################### CSRC BUILD IMAGE ####################
292 FROM base AS csrc-build
293 ARG TARGETPLATFORM
@@ -435,6 +492,10 @@ WORKDIR /workspace
492 COPY --from=csrc-build /workspace/dist /precompiled-wheels
493 COPY . .
494
495 +# Drop the pre-built rust frontend binary into the source tree. setup.py
496 +# detects it and ships it as-is, skipping the local cargo build.
497 +COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
498 +
499 ARG GIT_REPO_CHECK=0
500 RUN --mount=type=bind,source=.git,target=.git \
501 if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
@@ -455,6 +516,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
516 RUN --mount=type=cache,target=/root/.cache/uv \
517 --mount=type=bind,source=.git,target=.git \
518 if [ "${vllm_target_device}" = "cuda" ]; then \
519 + export VLLM_USE_PRECOMPILED=1; \
520 export VLLM_PRECOMPILED_WHEEL_LOCATION=$(ls /precompiled-wheels/*.whl); \
521 fi && \
522 python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
docker/Dockerfile.cpu
+50
@@ -36,6 +36,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
36 && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \
37 && curl -LsSf https://astral.sh/uv/install.sh | sh
38
39 +# Compiler and linker environment
40 ENV CC=/usr/bin/gcc-12 CXX=/usr/bin/g++-12
41 ENV CCACHE_DIR=/root/.cache/ccache
42 ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
@@ -80,6 +81,51 @@ FROM base-${TARGETARCH} AS base
81
82 RUN echo 'ulimit -c 0' >> ~/.bashrc
83
84 +######################### RUST BUILD IMAGE #########################
85 +# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build
86 +# stage doesn't need the rust toolchain or protoc. This stage runs in parallel
87 +# with the main vllm-build stage.
88 +FROM ubuntu:22.04 AS rust-build
89 +ENV DEBIAN_FRONTEND=noninteractive
90 +
91 +RUN apt-get update -y \
92 + && apt-get install -y --no-install-recommends \
93 + ca-certificates curl git build-essential \
94 + protobuf-compiler libprotobuf-dev \
95 + && rm -rf /var/lib/apt/lists/*
96 +
97 +# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
98 +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
99 + sh -s -- -y --profile minimal --default-toolchain none
100 +ENV PATH="/root/.cargo/bin:${PATH}"
101 +
102 +WORKDIR /workspace
103 +
104 +# Copy only the rust workspace — the binary is the sole artifact we need.
105 +COPY rust rust
106 +
107 +# Fail loudly if the rust submodule was not initialized on the host before
108 +# `docker build`. Without this check, cargo would emit a confusing error.
109 +RUN if [ ! -f rust/Cargo.toml ]; then \
110 + echo "ERROR: rust/ submodule is not initialized."; \
111 + echo "Run 'git submodule update --init --recursive' on the host before building."; \
112 + exit 1; \
113 + fi
114 +
115 +# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
116 +# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
117 +ENV CARGO_BUILD_JOBS=4
118 +
119 +# Build the release binary. Cache cargo registry/git and target/, but copy the
120 +# binary out of the target/ cache mount so it persists into the image layer
121 +# for later COPY --from=rust-build.
122 +RUN --mount=type=cache,target=/root/.cargo/registry \
123 + --mount=type=cache,target=/root/.cargo/git \
124 + --mount=type=cache,target=/workspace/rust/target \
125 + cd rust \
126 + && cargo build --release --bin vllm-rs --features native-tls-vendored \
127 + && cp target/release/vllm-rs /workspace/vllm-rs
128 +
129 ######################### BUILD IMAGE #########################
130 FROM base AS vllm-build
131
@@ -114,6 +160,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
160
161 COPY . .
162
163 +# Drop the pre-built rust frontend binary into the source tree. setup.py
164 +# detects it and ships it as-is, skipping the local cargo build.
165 +COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
166 +
167 RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
168
169 RUN --mount=type=cache,target=/root/.cache/uv \
docker/Dockerfile.nightly_torch
+45
@@ -94,6 +94,47 @@ RUN cat torch_build_versions.txt
94
95 #################### BASE BUILD IMAGE ####################
96
97 +#################### RUST BUILD IMAGE ####################
98 +# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build
99 +# stage doesn't need the rust toolchain or protoc.
100 +FROM ubuntu:22.04 AS rust-build
101 +ENV DEBIAN_FRONTEND=noninteractive
102 +
103 +RUN apt-get update -y \
104 + && apt-get install -y --no-install-recommends \
105 + ca-certificates curl git build-essential \
106 + protobuf-compiler libprotobuf-dev \
107 + && rm -rf /var/lib/apt/lists/*
108 +
109 +# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
110 +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
111 + sh -s -- -y --profile minimal --default-toolchain none
112 +ENV PATH="/root/.cargo/bin:${PATH}"
113 +
114 +WORKDIR /workspace
115 +
116 +COPY rust rust
117 +
118 +# Fail loudly if the rust submodule was not initialized on the host before
119 +# `docker build`.
120 +RUN if [ ! -f rust/Cargo.toml ]; then \
121 + echo "ERROR: rust/ submodule is not initialized."; \
122 + echo "Run 'git submodule update --init --recursive' on the host before building."; \
123 + exit 1; \
124 + fi
125 +
126 +# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
127 +# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
128 +ENV CARGO_BUILD_JOBS=4
129 +
130 +RUN --mount=type=cache,target=/root/.cargo/registry \
131 + --mount=type=cache,target=/root/.cargo/git \
132 + --mount=type=cache,target=/workspace/rust/target \
133 + cd rust \
134 + && cargo build --release --bin vllm-rs --features native-tls-vendored \
135 + && cp target/release/vllm-rs /workspace/vllm-rs
136 +#################### RUST BUILD IMAGE ####################
137 +
138 #################### WHEEL BUILD IMAGE ####################
139 FROM base AS build
140 ARG TARGETPLATFORM
@@ -104,6 +145,10 @@ ENV UV_HTTP_TIMEOUT=500
145
146 COPY . .
147
148 +# Drop the pre-built rust frontend binary into the source tree. setup.py
149 +# detects it and ships it as-is, skipping the local cargo build.
150 +COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
151 +
152 RUN python3 use_existing_torch.py
153
154 RUN --mount=type=cache,target=/root/.cache/uv \
docker/Dockerfile.rocm
+51
@@ -106,14 +106,61 @@ ONBUILD RUN git clone ${VLLM_REPO} \
106 && cd vllm \
107 && git fetch -v --prune -- origin ${VLLM_BRANCH} \
108 && git checkout FETCH_HEAD \
109 + && git submodule update --init --recursive \
110 && if [ ${VLLM_REPO} != "https://github.com/vllm-project/vllm.git" ] ; then \
111 git remote add upstream "https://github.com/vllm-project/vllm.git" \
112 && git fetch upstream ; fi
113 FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
114
115 +# -----------------------
116 +# Rust build stage
117 +# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
118 +# don't need the rust toolchain or protoc. Runs in parallel with the main wheel
119 +# build for faster end-to-end builds.
120 +FROM fetch_vllm AS rust-build
121 +ARG COMMON_WORKDIR
122 +
123 +# Fail loudly if the rust submodule was not initialized on the host before
124 +# `docker build`. The rust frontend source is brought in via the fetch_vllm
125 +# stage, so an uninitialized submodule would otherwise produce a confusing
126 +# cargo failure.
127 +RUN if [ ! -f ${COMMON_WORKDIR}/vllm/rust/Cargo.toml ]; then \
128 + echo "ERROR: rust/ submodule is not initialized."; \
129 + echo "Run 'git submodule update --init --recursive' on the host before building."; \
130 + exit 1; \
131 + fi
132 +
133 +# protoc is used by tonic-build/prost-build.
134 +RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
135 + protobuf-compiler libprotobuf-dev \
136 + && rm -rf /var/lib/apt/lists/*
137 +
138 +# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
139 +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
140 + sh -s -- -y --profile minimal --default-toolchain none
141 +ENV PATH="/root/.cargo/bin:${PATH}"
142 +
143 +# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit
144 +# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
145 +ENV CARGO_BUILD_JOBS=4
146 +
147 +# Build the release binary. Cache cargo registry/git, and copy the binary out
148 +# so it persists into the image layer for later COPY --from=rust-build.
149 +RUN --mount=type=cache,target=/root/.cargo/registry \
150 + --mount=type=cache,target=/root/.cargo/git \
151 + cd ${COMMON_WORKDIR}/vllm/rust \
152 + && cargo build --release --bin vllm-rs --features native-tls-vendored \
153 + && cp target/release/vllm-rs /tmp/vllm-rs
154 +
155 # -----------------------
156 # vLLM build stages
157 FROM fetch_vllm AS build_vllm
158 +ARG COMMON_WORKDIR
159 +
160 +# Drop the pre-built rust frontend binary into the source tree. setup.py
161 +# detects it and ships it as-is, skipping the local cargo build.
162 +COPY --from=rust-build /tmp/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs
163 +
164 # Build vLLM (setup.py auto-detects sccache in PATH)
165 RUN cd vllm \
166 && python3 -m pip install -r requirements/rocm.txt \
@@ -293,6 +340,10 @@ FROM fetch_vllm AS build_vllm_wheel_release
340
341 ARG COMMON_WORKDIR
342
343 +# Drop the pre-built rust frontend binary into the source tree. setup.py
344 +# detects it and ships it as-is, skipping the local cargo build.
345 +COPY --from=rust-build /tmp/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs
346 +
347 # Create /install directory for custom wheels
348 RUN mkdir -p /install
349
docker/Dockerfile.xpu
+45
@@ -1,3 +1,43 @@
1 +######################### RUST BUILD IMAGE #########################
2 +# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the main image
3 +# doesn't need the rust toolchain or protoc. Runs in parallel with vllm-base.
4 +FROM ubuntu:22.04 AS rust-build
5 +ENV DEBIAN_FRONTEND=noninteractive
6 +
7 +RUN apt-get update -y \
8 + && apt-get install -y --no-install-recommends \
9 + ca-certificates curl git build-essential \
10 + protobuf-compiler libprotobuf-dev \
11 + && rm -rf /var/lib/apt/lists/*
12 +
13 +# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
14 +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
15 + sh -s -- -y --profile minimal --default-toolchain none
16 +ENV PATH="/root/.cargo/bin:${PATH}"
17 +
18 +WORKDIR /workspace
19 +
20 +COPY rust rust
21 +
22 +# Fail loudly if the rust submodule was not initialized on the host before
23 +# `docker build`.
24 +RUN if [ ! -f rust/Cargo.toml ]; then \
25 + echo "ERROR: rust/ submodule is not initialized."; \
26 + echo "Run 'git submodule update --init --recursive' on the host before building."; \
27 + exit 1; \
28 + fi
29 +
30 +# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
31 +# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
32 +ENV CARGO_BUILD_JOBS=4
33 +
34 +RUN --mount=type=cache,target=/root/.cargo/registry \
35 + --mount=type=cache,target=/root/.cargo/git \
36 + --mount=type=cache,target=/workspace/rust/target \
37 + cd rust \
38 + && cargo build --release --bin vllm-rs --features native-tls-vendored \
39 + && cp target/release/vllm-rs /workspace/vllm-rs
40 +
41 FROM intel/deep-learning-essentials:2025.3.2-0-devel-ubuntu24.04 AS vllm-base
42
43 WORKDIR /workspace/
@@ -99,6 +139,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
139 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
140
141 COPY . .
142 +
143 +# Drop the pre-built rust frontend binary into the source tree. setup.py
144 +# detects it and ships it as-is, skipping the local cargo build.
145 +COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
146 +
147 ARG GIT_REPO_CHECK=0
148 RUN --mount=type=bind,source=.git,target=.git \
149 if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
docs/contributing/README.md
+7
@@ -43,6 +43,13 @@ If you are only developing vLLM's Python code, install vLLM using:
43 VLLM_USE_PRECOMPILED=1 uv pip install -e .
44 ```
45
46 +To rebuild only the Rust frontend binary:
47 +
48 +```bash
49 +./build_rust.sh # release build
50 +./build_rust.sh --debug # faster build for development
51 +```
52 +
53 If you are developing vLLM's Python and CUDA/C++ code, install Pytorch first:
54
55 ```bash
docs/contributing/ci/nightly_builds.md
+4 -4
@@ -136,10 +136,10 @@ When installing vLLM with `VLLM_USE_PRECOMPILED=1`, the `setup.py` script:
136 3. **Selects compatible wheel** based on:
137 - Package name (`vllm`)
138 - Platform tag (architecture match)
139 -4. **Downloads and extracts** precompiled binaries from the wheel:
140 - - C++ extension modules (`.so` files)
141 - - Flash Attention Python modules
142 - - Triton kernel Python files
139 +4. **Downloads and extracts** precompiled artifacts from the wheel:
140 + - Native extension modules (`.so` files)
141 + - The `vllm-rs` Rust frontend binary
142 + - Flash Attention Python modules and Triton/FlashMLA Python files
143 5. **Patches package_data** to include extracted files in the installation
144
145 !!! note "What is the base commit?"
docs/getting_started/installation/gpu.cuda.inc.md
+11 -1
@@ -101,12 +101,22 @@ This command will do the following:
101 1. Look for the current branch in your vLLM clone.
102 1. Identify the corresponding base commit in the main branch.
103 1. Download the pre-built wheel of the base commit.
104 -1. Use its compiled libraries in the installation.
104 +1. Use its compiled libraries and `vllm-rs` binary in the installation.
105
106 !!! note
107 1. If you change C++ or kernel code, you cannot use Python-only build; otherwise you will see an import error about library not found or undefined symbol.
108 2. If you rebase your dev branch, it is recommended to uninstall vllm and re-run the above command to make sure your libraries are up to date.
109
110 +!!! tip "Rebuilding the Rust frontend"
111 +If you need to recompile the `vllm-rs` Rust frontend binary, you can rebuild and install it without re-running the full pip install:
112 +
113 + ```bash
114 + ./build_rust.sh # release build
115 + ./build_rust.sh --debug # faster build for development
116 + ```
117 +
118 + This will install the required Rust toolchain if needed, build the binary, and place it in `vllm/vllm-rs`.
119 +
120 In case you see an error about wheel not found when running the above command, it might be because the commit you based on in the `main` branch was just merged and its precompiled wheel is not available yet. You can wait around an hour and retry, or set `VLLM_PRECOMPILED_WHEEL_COMMIT=nightly` to automatically select the most recent already-built commit on `main`.
121
122 ```bash
pyproject.toml
+1
@@ -6,6 +6,7 @@ requires = [
6 "packaging>=24.2",
7 "setuptools>=77.0.3,<81.0.0",
8 "setuptools-scm>=8.0",
9 + "setuptools-rust>=1.9.0",
10 "torch == 2.11.0",
11 "wheel",
12 "jinja2",
requirements/build/cpu.txt
+1
@@ -4,6 +4,7 @@ ninja
4 packaging>=24.2
5 setuptools==77.0.3 # this version can reuse CMake build dir
6 setuptools-scm>=8
7 +setuptools-rust>=1.9.0
8 torch==2.11.0+cpu; platform_machine == "x86_64" or platform_machine == "s390x" or platform_machine == "aarch64"
9 torch==2.11.0; platform_system == "Darwin" or platform_machine == "ppc64le" or platform_machine == "riscv64"
10 wheel
requirements/build/cuda.txt
+1
@@ -4,6 +4,7 @@ ninja
4 packaging>=24.2
5 setuptools>=77.0.3,<81.0.0
6 setuptools-scm>=8
7 +setuptools-rust>=1.9.0
8 torch==2.11.0
9 wheel
10 jinja2>=3.1.6
requirements/rocm.txt
+1
@@ -15,6 +15,7 @@ tensorizer==2.10.1
15 packaging>=24.2
16 setuptools>=77.0.3,<80.0.0
17 setuptools-scm>=8
18 +setuptools-rust>=1.9.0
19 runai-model-streamer[s3,gcs,azure]==0.15.7
20 conch-triton-kernels==1.2.1
21 timm>=1.0.17
rust new
+1
@@ -0,0 +1 @@
1 +Subproject commit ad6771ac093420632253ca43bfc81007beb2d6ce
rust-toolchain.toml new
+2
@@ -0,0 +1,2 @@
1 +[toolchain]
2 +channel = "1.95"
setup.py
+115 -43
@@ -18,6 +18,8 @@ import torch
18 from packaging.version import Version, parse
19 from setuptools import Extension, setup
20 from setuptools.command.build_ext import build_ext
21 +from setuptools_rust import Binding, RustExtension
22 +from setuptools_rust.build import build_rust
23 from setuptools_scm import get_version
24 from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME
25
@@ -33,11 +35,24 @@ def load_module_from_path(module_name, path):
35 ROOT_DIR = Path(__file__).parent
36 logger = logging.getLogger(__name__)
37
38 +PRECOMPILED_RUST_FRONTEND_PATH = ROOT_DIR / "vllm" / "vllm-rs"
39 +
40 # cannot import envs directly because it depends on vllm,
41 # which is not installed yet
42 envs = load_module_from_path("envs", os.path.join(ROOT_DIR, "vllm", "envs.py"))
43
44 VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
45 +USE_PRECOMPILED_EXTENSIONS = envs.VLLM_USE_PRECOMPILED
46 +# VLLM_USE_PRECOMPILED implies precompiled rust frontend too.
47 +USE_PRECOMPILED_RUST_FRONTEND = (
48 + envs.VLLM_USE_PRECOMPILED or envs.VLLM_USE_PRECOMPILED_RUST
49 +)
50 +
51 +
52 +def should_require_rust_frontend() -> bool:
53 + value = os.getenv("VLLM_REQUIRE_RUST_FRONTEND", "")
54 + return value.lower() not in ("", "0", "false", "no")
55 +
56
57 if sys.platform.startswith("darwin") and VLLM_TARGET_DEVICE != "cpu":
58 logger.warning("VLLM_TARGET_DEVICE automatically set to `cpu` due to macOS")
@@ -405,6 +420,24 @@ class precompiled_build_ext(build_ext):
420 return
421
422
423 +class precompiled_build_rust(build_rust):
424 + """Skips local Rust builds when the precompiled wheel already ships vllm-rs."""
425 +
426 + def run(self) -> None:
427 + if PRECOMPILED_RUST_FRONTEND_PATH.exists():
428 + logger.info(
429 + "Skipping local Rust build: using precompiled %s",
430 + PRECOMPILED_RUST_FRONTEND_PATH,
431 + )
432 + return
433 +
434 + logger.warning(
435 + "Precompiled wheel did not provide %s; falling back to local Rust build.",
436 + PRECOMPILED_RUST_FRONTEND_PATH,
437 + )
438 + super().run()
439 +
440 +
441 class precompiled_wheel_utils:
442 """Extracts libraries and other files from an existing wheel."""
443
@@ -653,7 +686,11 @@ class precompiled_wheel_utils:
686
687 @staticmethod
688 def extract_precompiled_and_patch_package(
656 - wheel_url_or_path: str, download_filename: str | None
689 + wheel_url_or_path: str,
690 + download_filename: str | None,
691 + *,
692 + extract_extensions: bool,
693 + extract_rust_frontend: bool,
694 ) -> dict:
695 import tempfile
696 import zipfile
@@ -676,20 +713,26 @@ class precompiled_wheel_utils:
713 package_data_patch = {}
714
715 with zipfile.ZipFile(wheel_path) as wheel:
679 - files_to_copy = [
680 - "vllm/_C.abi3.so",
681 - "vllm/_C_stable_libtorch.abi3.so",
682 - "vllm/_moe_C.abi3.so",
683 - "vllm/_flashmla_C.abi3.so",
684 - "vllm/_flashmla_extension_C.abi3.so",
685 - "vllm/_sparse_flashmla_C.abi3.so",
686 - "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
687 - "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
688 - "vllm/cumem_allocator.abi3.so",
689 - "vllm/spinloop.abi3.so",
690 - # ROCm-specific libraries
691 - "vllm/_rocm_C.abi3.so",
692 - ]
716 + exact_members = set()
717 + if extract_extensions:
718 + exact_members.update(
719 + {
720 + "vllm/_C.abi3.so",
721 + "vllm/_C_stable_libtorch.abi3.so",
722 + "vllm/_moe_C.abi3.so",
723 + "vllm/_flashmla_C.abi3.so",
724 + "vllm/_flashmla_extension_C.abi3.so",
725 + "vllm/_sparse_flashmla_C.abi3.so",
726 + "vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
727 + "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
728 + "vllm/cumem_allocator.abi3.so",
729 + "vllm/spinloop.abi3.so",
730 + # ROCm-specific libraries
731 + "vllm/_rocm_C.abi3.so",
732 + }
733 + )
734 + if extract_rust_frontend:
735 + exact_members.add("vllm/vllm-rs")
736
737 flash_attn_regex = re.compile(
738 r"vllm/vllm_flash_attn/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
@@ -708,27 +751,25 @@ class precompiled_wheel_utils:
751 )
752 # DeepGEMM: extract all files (.py, .so, .cuh, .h, .hpp, etc.)
753 deep_gemm_regex = re.compile(r"vllm/third_party/deep_gemm/.*")
711 - file_members = list(
712 - filter(lambda x: x.filename in files_to_copy, wheel.filelist)
713 - )
714 - file_members += list(
715 - filter(
716 - lambda x: flash_attn_regex.match(x.filename)
717 - and x.filename not in flash_attn_files_to_skip,
718 - wheel.filelist,
719 - )
720 - )
721 - file_members += list(
722 - filter(
723 - lambda x: triton_kernels_regex.match(x.filename), wheel.filelist
724 - )
725 - )
726 - file_members += list(
727 - filter(lambda x: flashmla_regex.match(x.filename), wheel.filelist)
728 - )
729 - file_members += list(
730 - filter(lambda x: deep_gemm_regex.match(x.filename), wheel.filelist)
731 - )
754 + file_members = []
755 + for member in wheel.filelist:
756 + if member.filename in exact_members:
757 + file_members.append(member)
758 + continue
759 +
760 + if not extract_extensions:
761 + continue
762 +
763 + if (
764 + (
765 + flash_attn_regex.match(member.filename)
766 + and member.filename not in flash_attn_files_to_skip
767 + )
768 + or triton_kernels_regex.match(member.filename)
769 + or flashmla_regex.match(member.filename)
770 + or deep_gemm_regex.match(member.filename)
771 + ):
772 + file_members.append(member)
773
774 for file in file_members:
775 print(f"[extract] {file.filename}")
@@ -739,6 +780,9 @@ class precompiled_wheel_utils:
780 open(target_path, "wb") as dst,
781 ):
782 shutil.copyfileobj(src, dst)
783 + mode = file.external_attr >> 16
784 + if mode:
785 + os.chmod(target_path, mode)
786
787 pkg = os.path.dirname(file.filename).replace("/", ".")
788 package_data_patch.setdefault(pkg, []).append(
@@ -911,7 +955,7 @@ def get_vllm_version() -> str:
955 if envs.VLLM_TARGET_DEVICE == "empty":
956 version += f"{sep}empty"
957 elif _is_cuda():
914 - if envs.VLLM_USE_PRECOMPILED and not envs.VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX:
958 + if USE_PRECOMPILED_EXTENSIONS and not envs.VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX:
959 version += f"{sep}precompiled"
960 else:
961 cuda_version = str(get_nvcc_cuda_version())
@@ -1006,7 +1050,7 @@ if _is_hip():
1050
1051 if _is_cuda():
1052 ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
1009 - if envs.VLLM_USE_PRECOMPILED or (
1053 + if USE_PRECOMPILED_EXTENSIONS or (
1054 CUDA_HOME and get_nvcc_cuda_version() >= Version("12.3")
1055 ):
1056 # FA3 requires CUDA 12.3 or later
@@ -1016,7 +1060,7 @@ if _is_cuda():
1060 ext_modules.append(
1061 CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa4_cutedsl_C", optional=True)
1062 )
1019 - if envs.VLLM_USE_PRECOMPILED or (
1063 + if USE_PRECOMPILED_EXTENSIONS or (
1064 CUDA_HOME and get_nvcc_cuda_version() >= Version("12.9")
1065 ):
1066 # FlashMLA requires CUDA 12.9 or later
@@ -1065,15 +1109,25 @@ package_data = {
1109 }
1110
1111
1068 -# If using precompiled, extract and patch package_data (in advance of setup)
1069 -if envs.VLLM_USE_PRECOMPILED:
1112 +# If using precompiled artifacts, extract and patch package_data in advance.
1113 +if USE_PRECOMPILED_RUST_FRONTEND:
1114 wheel_url, download_filename = precompiled_wheel_utils.determine_wheel_url()
1115 patch = precompiled_wheel_utils.extract_precompiled_and_patch_package(
1072 - wheel_url, download_filename
1116 + wheel_url,
1117 + download_filename,
1118 + extract_extensions=USE_PRECOMPILED_EXTENSIONS,
1119 + extract_rust_frontend=True,
1120 )
1121 for pkg, files in patch.items():
1122 package_data.setdefault(pkg, []).extend(files)
1123
1124 +# If the rust frontend binary is already present in the source tree (e.g.,
1125 +# pre-built in a separate Docker build stage), ship it as-is.
1126 +if PRECOMPILED_RUST_FRONTEND_PATH.exists():
1127 + vllm_files = package_data.setdefault("vllm", [])
1128 + if "vllm-rs" not in vllm_files:
1129 + vllm_files.append("vllm-rs")
1130 +
1131 if _no_device():
1132 ext_modules = []
1133
@@ -1082,14 +1136,32 @@ if not ext_modules:
1136 else:
1137 cmdclass = {
1138 "build_ext": precompiled_build_ext
1085 - if envs.VLLM_USE_PRECOMPILED
1139 + if USE_PRECOMPILED_EXTENSIONS
1140 else cmake_build_ext,
1141 }
1142 +if USE_PRECOMPILED_RUST_FRONTEND or PRECOMPILED_RUST_FRONTEND_PATH.exists():
1143 + cmdclass["build_rust"] = precompiled_build_rust
1144 +
1145 +# Rust frontend binary, built via setuptools-rust and installed into the
1146 +# package directory alongside the Python modules.
1147 +# TODO: we may use `RustBin` to directly install it into `bin` directory, but this
1148 +# requires extra work on using precompiled binaries.
1149 +rust_extensions = [
1150 + RustExtension(
1151 + target="vllm.vllm-rs",
1152 + path="rust/src/cmd/Cargo.toml",
1153 + args=["--bin", "vllm-rs"],
1154 + features=["native-tls-vendored"],
1155 + binding=Binding.Exec,
1156 + optional=not should_require_rust_frontend(),
1157 + ),
1158 +]
1159
1160 setup(
1161 # static metadata should rather go in pyproject.toml
1162 version=get_vllm_version(),
1163 ext_modules=ext_modules,
1164 + rust_extensions=rust_extensions,
1165 install_requires=get_requirements(),
1166 extras_require={
1167 # AMD Zen CPU optimizations via zentorch
tests/entrypoints/openai/completion/test_shutdown.py
+3 -9
@@ -299,22 +299,16 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
299 start_time = time.time()
300 proc.send_signal(signal.SIGTERM)
301
302 - # abort timeout (0) should stop the server promptly. On ROCm, process
303 - # exit can spend extra time in HIP/RCCL/native extension teardown after
304 - # the server and engine have already shut down.
305 - max_exit_time = 4.0 if _IS_ROCM else 2.1
306 -
302 + # abort timeout (0) should stop the server promptly.
303 try:
308 - proc.wait(timeout=max_exit_time)
304 + proc.wait(timeout=4.0)
305 except subprocess.TimeoutExpired:
306 proc.kill()
307 proc.wait(timeout=5)
308 pytest.fail("Process did not exit after SIGTERM with abort timeout")
309
310 exit_time = time.time() - start_time
315 - assert exit_time < max_exit_time, (
316 - f"Default shutdown took too long: {exit_time:.1f}s"
317 - )
311 + assert exit_time < 4.1, f"Default shutdown took too long: {exit_time:.1f}s"
312 assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
313
314 await _assert_children_cleaned_up(child_pids)
tests/test_envs.py
+13
@@ -103,6 +103,19 @@ def test_is_envs_cache_enabled() -> None:
103 assert not envs._is_envs_cache_enabled()
104
105
106 +def test_precompiled_install_flags_are_orthogonal() -> None:
107 + with patch.dict(
108 + os.environ,
109 + {
110 + "VLLM_PRECOMPILED_WHEEL_LOCATION": "/tmp/vllm.whl",
111 + "VLLM_USE_PRECOMPILED_RUST": "1",
112 + },
113 + clear=False,
114 + ):
115 + assert environment_variables["VLLM_USE_PRECOMPILED"]() is False
116 + assert environment_variables["VLLM_USE_PRECOMPILED_RUST"]() is True
117 +
118 +
119 class TestEnvWithChoices:
120 """Test cases for env_with_choices function."""
121
vllm/entrypoints/cli/launch.py
+9
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import argparse
5 +import signal
6
7 import uvloop
8
@@ -110,6 +111,14 @@ def cmd_init() -> list[CLISubcommand]:
111
112 async def run_launch_fastapi(args: argparse.Namespace) -> None:
113 """Run the online serving layer with FastAPI (no GPU inference)."""
114 +
115 + # Interrupt initialization if SIGTERM arrives before uvicorn installs
116 + # its own signal handlers. Once uvicorn is running it replaces this.
117 + def _interrupt_init(*_) -> None:
118 + raise KeyboardInterrupt("terminated")
119 +
120 + signal.signal(signal.SIGTERM, _interrupt_init)
121 +
122 # 1. Socket binding
123 listen_address, sock = setup_server(args)
124
vllm/entrypoints/cli/serve.py
+48 -19
@@ -21,7 +21,11 @@ from vllm.v1.engine.utils import CoreEngineProcManager, launch_core_engines
21 from vllm.v1.executor import Executor
22 from vllm.v1.executor.multiproc_executor import MultiprocExecutor
23 from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
24 -from vllm.v1.utils import APIServerProcessManager, wait_for_completion_or_failure
24 +from vllm.v1.utils import (
25 + APIServerProcessManager,
26 + RustFrontendProcessManager,
27 + wait_for_completion_or_failure,
28 +)
29
30 logger = init_logger(__name__)
31
@@ -81,11 +85,12 @@ class ServeSubcommand(CLISubcommand):
85 )
86
87 # Default api_server_count if not explicitly set.
88 + # - Rust frontend: Use 1 (not applicable as it's multithreaded)
89 # - External LB: Leave as 1 (external LB handles distribution)
90 # - Hybrid LB: Use local DP size (internal LB for local ranks only)
91 # - Internal LB: Use full DP size
92 if args.api_server_count is None:
88 - if is_external_lb:
93 + if is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
94 args.api_server_count = 1
95 elif is_hybrid_lb:
96 args.api_server_count = args.data_parallel_size_local or 1
@@ -102,6 +107,11 @@ class ServeSubcommand(CLISubcommand):
107 "Defaulting api_server_count to data_parallel_size (%d).",
108 args.api_server_count,
109 )
110 + elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1:
111 + logger.warning(
112 + "Ignoring --api-server-count=%d when using rust front-end process"
113 + )
114 + args.api_server_count = 1
115
116 # Elastic EP currently only supports running with at most one API server.
117 if getattr(args, "enable_elastic_ep", False) and args.api_server_count > 1:
@@ -114,7 +124,7 @@ class ServeSubcommand(CLISubcommand):
124
125 if args.api_server_count < 1:
126 run_headless(args)
117 - elif args.api_server_count > 1:
127 + elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH:
128 run_multi_api_server(args)
129 else:
130 # Single API server (this process).
@@ -230,9 +240,15 @@ def run_headless(args: argparse.Namespace):
240
241 def run_multi_api_server(args: argparse.Namespace):
242 assert not args.headless
243 + rust_frontend_path = envs.VLLM_RUST_FRONTEND_PATH
244 num_api_servers: int = args.api_server_count
245 assert num_api_servers > 0
246
247 + if rust_frontend_path and num_api_servers > 1:
248 + raise ValueError(
249 + "VLLM_RUST_FRONTEND_PATH does not support api_server_count > 1"
250 + )
251 +
252 if num_api_servers > 1:
253 setup_multiprocess_prometheus()
254
@@ -270,7 +286,9 @@ def run_multi_api_server(args: argparse.Namespace):
286 dp_rank = parallel_config.data_parallel_rank
287 assert parallel_config.local_engines_only or dp_rank == 0
288
273 - api_server_manager: APIServerProcessManager | None = None
289 + api_server_manager: APIServerProcessManager | RustFrontendProcessManager | None = (
290 + None
291 + )
292
293 from vllm.v1.engine.utils import get_engine_zmq_addresses
294
@@ -279,23 +297,34 @@ def run_multi_api_server(args: argparse.Namespace):
297 with launch_core_engines(
298 vllm_config, executor_class, log_stats, addresses, num_api_servers
299 ) as (local_engine_manager, coordinator, addresses, tensor_queue):
282 - # Construct common args for the APIServerProcessManager up-front.
283 - stats_update_address = None
284 - if coordinator:
285 - stats_update_address = coordinator.get_stats_publish_address()
286 -
287 - # Start API servers.
288 - api_server_manager = APIServerProcessManager(
289 - listen_address=listen_address,
290 - sock=sock,
291 - args=args,
292 - num_servers=num_api_servers,
293 - input_addresses=addresses.inputs,
294 - output_addresses=addresses.outputs,
295 - stats_update_address=stats_update_address,
296 - tensor_queue=tensor_queue,
300 + stats_update_address = (
301 + coordinator.get_stats_publish_address() if coordinator else None
302 )
303
304 + if rust_frontend_path:
305 + # Start rust front-end process.
306 + api_server_manager = RustFrontendProcessManager(
307 + binary_path=rust_frontend_path,
308 + sock=sock,
309 + args=args,
310 + input_address=addresses.inputs[0],
311 + output_address=addresses.outputs[0],
312 + engine_count=parallel_config.data_parallel_size,
313 + stats_update_address=stats_update_address,
314 + )
315 + else:
316 + # Start API server(s).
317 + api_server_manager = APIServerProcessManager(
318 + listen_address=listen_address,
319 + sock=sock,
320 + args=args,
321 + num_servers=num_api_servers,
322 + input_addresses=addresses.inputs,
323 + output_addresses=addresses.outputs,
324 + stats_update_address=stats_update_address,
325 + tensor_queue=tensor_queue,
326 + )
327 +
328 # Wait for API servers.
329 try:
330 wait_for_completion_or_failure(
vllm/entrypoints/openai/api_server.py
+8 -8
@@ -533,8 +533,7 @@ def validate_api_server_args(args):
533
534 @instrument(span_name="API server setup")
535 def setup_server(args):
536 - """Validate API server args, set up signal handler, create socket
537 - ready to serve."""
536 + """Validate API server args and create the server socket."""
537
538 log_version_and_model(logger, VLLM_VERSION, args.model)
539 log_non_default_args(args)
@@ -560,12 +559,6 @@ def setup_server(args):
559 # many concurrent requests active
560 set_ulimit()
561
563 - def signal_handler(*_) -> None:
564 - # Interrupt server on sigterm while initializing
565 - raise KeyboardInterrupt("terminated")
566 -
567 - signal.signal(signal.SIGTERM, signal_handler)
568 -
562 if args.uds:
563 listen_address = f"unix:{args.uds}"
564 else:
@@ -675,6 +668,13 @@ async def run_server(args, **uvicorn_kwargs) -> None:
668 # Add process-specific prefix to stdout and stderr.
669 decorate_logs("APIServer")
670
671 + # Interrupt initialization if SIGTERM arrives before uvicorn installs its
672 + # own signal handlers. Once uvicorn is running it replaces this.
673 + def _interrupt_init(*_) -> None:
674 + raise KeyboardInterrupt("terminated")
675 +
676 + signal.signal(signal.SIGTERM, _interrupt_init)
677 +
678 listen_address, sock = setup_server(args)
679 await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
680
vllm/entrypoints/utils.py
+39 -1
@@ -9,6 +9,7 @@ from argparse import Namespace
9 from http import HTTPStatus
10 from logging import Logger
11 from string import Template
12 +from typing import Any
13
14 import regex as re
15 from fastapi import Request
@@ -210,7 +211,7 @@ def get_max_tokens(
211 )
212
213
213 -def log_non_default_args(args: Namespace | EngineArgs):
214 +def get_non_default_args(args: Namespace | EngineArgs) -> dict[str, Any]:
215 from vllm.entrypoints.openai.cli_args import make_arg_parser
216
217 non_default_args = {}
@@ -237,6 +238,43 @@ def log_non_default_args(args: Namespace | EngineArgs):
238 "Unsupported argument type. Must be Namespace or EngineArgs instance."
239 )
240
241 + return non_default_args
242 +
243 +
244 +def _jsonify_arg_value(value: Any) -> Any:
245 + if value is None or isinstance(value, bool | int | float | str):
246 + return value
247 + if dataclasses.is_dataclass(value) and not isinstance(value, type):
248 + return {
249 + key: _jsonify_arg_value(val)
250 + for key, val in dataclasses.asdict(value).items()
251 + }
252 + if isinstance(value, dict):
253 + return {str(key): _jsonify_arg_value(val) for key, val in value.items()}
254 + if isinstance(value, tuple | list):
255 + return [_jsonify_arg_value(item) for item in value]
256 + if (model_dump := getattr(value, "model_dump", None)) is not None:
257 + return _jsonify_arg_value(model_dump(mode="json"))
258 + if (to_dict := getattr(value, "dict", None)) is not None:
259 + return _jsonify_arg_value(to_dict())
260 + return repr(value)
261 +
262 +
263 +def jsonify_non_default_args(
264 + args: Namespace | EngineArgs,
265 + *,
266 + exclude: set[str] | None = None,
267 +) -> dict[str, Any]:
268 + non_default_args = get_non_default_args(args)
269 + if exclude is not None:
270 + for key in exclude:
271 + non_default_args.pop(key, None)
272 +
273 + return {key: _jsonify_arg_value(value) for key, value in non_default_args.items()}
274 +
275 +
276 +def log_non_default_args(args: Namespace | EngineArgs):
277 + non_default_args = get_non_default_args(args)
278 logger.info("non-default args: %s", non_default_args)
279
280
vllm/envs.py
+51 -1
@@ -86,6 +86,7 @@ if TYPE_CHECKING:
86 MAX_JOBS: str | None = None
87 NVCC_THREADS: str | None = None
88 VLLM_USE_PRECOMPILED: bool = False
89 + VLLM_USE_PRECOMPILED_RUST: bool = False
90 VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX: bool = False
91 VLLM_DOCKER_BUILD_CONTEXT: bool = False
92 VLLM_KEEP_ALIVE_ON_ENGINE_DEATH: bool = False
@@ -134,6 +135,8 @@ if TYPE_CHECKING:
135 Q_SCALE_CONSTANT: int = 200
136 K_SCALE_CONSTANT: int = 200
137 V_SCALE_CONSTANT: int = 100
138 + VLLM_USE_RUST_FRONTEND: bool = False
139 + VLLM_RUST_FRONTEND_PATH: str | None = "auto"
140 VLLM_SERVER_DEV_MODE: bool = False
141 VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
142 VLLM_MLA_DISABLE: bool = False
@@ -514,6 +517,40 @@ def get_env_or_set_default(
517
518 logger = logging.getLogger(__name__)
519
520 +
521 +def _resolve_rust_frontend_path() -> str | None:
522 + """Resolve the Rust frontend binary path.
523 +
524 + Returns None if VLLM_USE_RUST_FRONTEND is not enabled.
525 + When enabled, resolves VLLM_RUST_FRONTEND_PATH ("auto" by default)
526 + to the actual binary path.
527 + """
528 + use_rust = bool(int(os.environ.get("VLLM_USE_RUST_FRONTEND", "0")))
529 + raw = os.environ.get("VLLM_RUST_FRONTEND_PATH", "auto")
530 +
531 + if not use_rust:
532 + if os.environ.get("VLLM_RUST_FRONTEND_PATH") is not None:
533 + logger.warning(
534 + "VLLM_RUST_FRONTEND_PATH is set but VLLM_USE_RUST_FRONTEND "
535 + "is not enabled. The Rust frontend will not be used. "
536 + "Set VLLM_USE_RUST_FRONTEND=1 to enable it."
537 + )
538 + return None
539 +
540 + if raw.lower() in ("auto", "1", "true"):
541 + pkg_dir = os.path.dirname(os.path.abspath(__file__))
542 + candidate = os.path.join(pkg_dir, "vllm-rs")
543 + if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
544 + return candidate
545 +
546 + raise FileNotFoundError(
547 + "VLLM_RUST_FRONTEND_PATH=auto but the vllm-rs binary was "
548 + f"not found at {candidate}. "
549 + "Build with setuptools-rust or set the path explicitly."
550 + )
551 + return raw
552 +
553 +
554 environment_variables: dict[str, Callable[[], Any]] = {
555 # ================== Installation Time Env Vars ==================
556 # Target device of vLLM, supporting [cuda (by default),
@@ -551,11 +588,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
588 # By default this is 1.
589 # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
590 "NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None),
554 - # If set, vllm will use precompiled binaries (*.so)
591 + # If set, vllm will use precompiled native binaries (*.so and vllm-rs).
592 "VLLM_USE_PRECOMPILED": lambda: (
593 os.environ.get("VLLM_USE_PRECOMPILED", "").strip().lower() in ("1", "true")
594 or bool(os.environ.get("VLLM_PRECOMPILED_WHEEL_LOCATION"))
595 ),
596 + # If set, vllm will use the precompiled Rust frontend binary (vllm-rs).
597 + "VLLM_USE_PRECOMPILED_RUST": lambda: (
598 + os.environ.get("VLLM_USE_PRECOMPILED_RUST", "").strip().lower() in ("1", "true")
599 + ),
600 # If set, skip adding +precompiled suffix to version string
601 "VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX": lambda: bool(
602 int(os.environ.get("VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX", "0"))
@@ -1154,6 +1195,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
1195 # If set to "0", disable LayerName opaque type for layer_name
1196 # parameters in custom ops. Defaults to enabled on torch >= 2.11.
1197 "VLLM_USE_LAYERNAME": lambda: bool(int(os.getenv("VLLM_USE_LAYERNAME", "1"))),
1198 + # If set, use the Rust frontend binary instead of the Python API server
1199 + # process(es).
1200 + "VLLM_USE_RUST_FRONTEND": lambda: bool(
1201 + int(os.getenv("VLLM_USE_RUST_FRONTEND", "0"))
1202 + ),
1203 + # Path to the Rust frontend binary. Defaults to "auto" which discovers
1204 + # the binary installed with the vllm package. Only used when
1205 + # VLLM_USE_RUST_FRONTEND=1.
1206 + "VLLM_RUST_FRONTEND_PATH": lambda: _resolve_rust_frontend_path(),
1207 # If set, vllm will run in development mode, which will enable
1208 # some additional endpoints for developing and debugging,
1209 # e.g. `/reset_prefix_cache`
vllm/v1/engine/__init__.py
+1
@@ -75,6 +75,7 @@ class EngineCoreReadyResponse:
75 max_model_len: int
76 num_gpu_blocks: int
77 dp_stats_address: str | None
78 + dtype: str | None = None
79
80
81 class EngineCoreRequest(
vllm/v1/engine/core.py
+1
@@ -1437,6 +1437,7 @@ class EngineCoreProc(EngineCore):
1437 max_model_len=self.vllm_config.model_config.max_model_len,
1438 num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0,
1439 dp_stats_address=self.frontend_stats_publish_address,
1440 + dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
1441 )
1442 ready_payload = msgspec.msgpack.encode(ready_response)
1443 for input_socket in input_sockets:
vllm/v1/utils.py
+143 -2
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import argparse
4 import contextlib
5 +import json
6 import multiprocessing
7 import threading
8 import time
@@ -233,6 +234,146 @@ class APIServerProcessManager:
234 shutdown(self.processes, timeout=timeout)
235
236
237 +class RustFrontendProcessManager:
238 + """Manages a single Rust frontend subprocess.
239 +
240 + Launches the Rust vllm-rs binary in 'frontend' mode, passing the
241 + listening socket fd and ZMQ transport addresses. Provides the same
242 + interface as APIServerProcessManager for process monitoring.
243 + """
244 +
245 + def __init__(
246 + self,
247 + binary_path: str,
248 + sock: Any,
249 + args: argparse.Namespace,
250 + input_address: str,
251 + output_address: str,
252 + engine_count: int,
253 + stats_update_address: str | None = None,
254 + ):
255 + import os
256 + import subprocess
257 +
258 + fd = sock.fileno()
259 + os.set_inheritable(fd, True)
260 +
261 + cmd = [
262 + binary_path,
263 + "frontend",
264 + "--listen-fd",
265 + str(fd),
266 + "--input-address",
267 + input_address,
268 + "--output-address",
269 + output_address,
270 + "--engine-count",
271 + str(engine_count),
272 + ]
273 + if stats_update_address is not None:
274 + cmd.extend(["--coordinator-address", stats_update_address])
275 + from vllm.entrypoints.utils import jsonify_non_default_args
276 +
277 + args_json = json.dumps(
278 + jsonify_non_default_args(args, exclude={"api_server_count"}),
279 + sort_keys=True,
280 + )
281 + cmd.extend(["--args-json", args_json])
282 +
283 + logger.info("Launching Rust frontend: %s", " ".join(cmd))
284 + self._proc = subprocess.Popen(cmd, pass_fds=(fd,))
285 +
286 + # Create a process wrapper with a sentinel fd for monitoring
287 + self.processes: list[_SubprocessWrapper] = [
288 + _SubprocessWrapper(self._proc, "RustFrontend")
289 + ]
290 +
291 + self._finalizer = weakref.finalize(self, _shutdown_subprocesses, self.processes)
292 +
293 + def shutdown(self, timeout: float | None = None) -> None:
294 + if self._finalizer.detach() is not None:
295 + _shutdown_subprocesses(self.processes, timeout=timeout)
296 +
297 +
298 +class _SubprocessWrapper:
299 + """Wraps subprocess.Popen to provide the BaseProcess-like interface
300 + needed by wait_for_completion_or_failure."""
301 +
302 + def __init__(self, proc, name: str):
303 + self._proc = proc
304 + self.name = name
305 + self.pid = proc.pid
306 + self._sentinel_conn: connection.Connection | None = None
307 + self._sentinel_send: connection.Connection | None = None
308 +
309 + # Use a Pipe-based sentinel so subprocess monitoring works uniformly
310 + # across platforms with multiprocessing.connection.wait().
311 + recv, send = connection.Pipe(duplex=False)
312 + self._sentinel_conn = recv
313 + self._sentinel_send = send
314 +
315 + def monitor_subprocess() -> None:
316 + try:
317 + proc.wait()
318 + finally:
319 + with contextlib.suppress(Exception):
320 + send.close()
321 +
322 + threading.Thread(
323 + target=monitor_subprocess, daemon=True, name=f"{name}Monitor"
324 + ).start()
325 +
326 + @property
327 + def sentinel(self):
328 + return self._sentinel_conn
329 +
330 + @property
331 + def exitcode(self) -> int | None:
332 + return self._proc.returncode if self._proc.poll() is not None else None
333 +
334 + def is_alive(self) -> bool:
335 + return self._proc.poll() is None
336 +
337 + def terminate(self):
338 + self._proc.terminate()
339 +
340 + def join(self, timeout=None):
341 + with contextlib.suppress(Exception):
342 + self._proc.wait(timeout=timeout)
343 +
344 + def __del__(self):
345 + with contextlib.suppress(Exception):
346 + if self._sentinel_conn is not None:
347 + self._sentinel_conn.close()
348 + if self._sentinel_send is not None:
349 + self._sentinel_send.close()
350 +
351 +
352 +def _shutdown_subprocesses(
353 + procs: list[_SubprocessWrapper], timeout: float | None = None
354 +) -> None:
355 + """Shutdown subprocess wrappers (mirrors the shutdown() function)."""
356 + if timeout is None:
357 + timeout = 0.0
358 + timeout = max(timeout, 5.0)
359 +
360 + for proc in procs:
361 + if proc.is_alive():
362 + proc.terminate()
363 +
364 + deadline = time.monotonic() + timeout
365 + for proc in procs:
366 + remaining = deadline - time.monotonic()
367 + if remaining <= 0:
368 + break
369 + if proc.is_alive():
370 + proc.join(remaining)
371 +
372 + for proc in procs:
373 + if proc.is_alive() and (pid := proc.pid) is not None:
374 + kill_process_tree(pid)
375 +
376 +
377 def run_api_server_worker_proc(
378 listen_address, sock, args, client_config=None, **uvicorn_kwargs
379 ) -> None:
@@ -253,7 +394,7 @@ def run_api_server_worker_proc(
394
395
396 def wait_for_completion_or_failure(
256 - api_server_manager: APIServerProcessManager,
397 + api_server_manager: "APIServerProcessManager | RustFrontendProcessManager",
398 engine_manager: Union["CoreEngineProcManager", "CoreEngineActorManager"]
399 | None = None,
400 coordinator: "DPCoordinator | None" = None,
@@ -274,7 +415,7 @@ def wait_for_completion_or_failure(
415 logger.info("Waiting for API servers to complete ...")
416 # Create a mapping of sentinels to their corresponding processes
417 # for efficient lookup
277 - sentinel_to_proc: dict[Any, BaseProcess] = {
418 + sentinel_to_proc: dict[Any, BaseProcess | _SubprocessWrapper | None] = {
419 proc.sentinel: proc for proc in api_server_manager.processes
420 }
421