@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 865781e627

[Rust Frontend] Add standalone Rust renderer (#50289)

Sage committed Aug 6, 2026 at 14:23 UTC 865781e62769fc469e5b1859773bb17bc36e5b3c
15 files changed +720 -40
rust/README.md
+39 -1
@@ -15,6 +15,7 @@ The component is organized as a Cargo workspace with several crates, layered bot
15 │ vllm-cmd / vllm-rs │ CLI entrypoint:
16 │ │ Python vLLM frontend subprocess
17 │ │ Rust managed-engine serve mode
18 +│ │ Engine-free render mode
19 ├─────────────────────────────────┤
20 │ vllm-server │ OpenAI-compatible HTTP API (axum)
21 ├─────────────────────────────────┤
@@ -74,9 +75,46 @@ To build the `vllm-rs` in isolation:
75 ./build_rust.sh
76 ```
77
78 +### Engine-free renderer
79 +
80 +`vllm-rs render` serves text-only request preprocessing without starting or
81 +connecting to a Python inference engine:
82 +
83 +```bash
84 +cargo run --manifest-path rust/Cargo.toml -p vllm-cmd --release -- \
85 + render Qwen/Qwen3-32B \
86 + --host 127.0.0.1 --max-model-len 32768
87 +```
88 +
89 +It exposes `/v1/chat/completions/render` and `/v1/completions/render`. Only
90 +tokenizer and model configuration files are loaded; model weights, PyTorch,
91 +and vLLM kernels are not required.
92 +
93 +The render endpoints return the public token-in `GenerateRequest` consumed by
94 +the Rust `/inference/v1/generate` endpoint. A chat render response, or one item
95 +from a completion render response, can be submitted to that endpoint without
96 +changing its fields.
97 +
98 +The render and inference paths use the same `vllm-chat` and `vllm-text`
99 +request-preparation logic; render mode stops before engine submission.
100 +Tool-call and reasoning parsers use model-based auto-detection by default. Use
101 +`--tool-call-parser` and `--reasoning-parser` to override either selection;
102 +unified parsers require the same selection for both options.
103 +
104 +```bash
105 +curl http://127.0.0.1:8000/v1/chat/completions/render \
106 + -H "Content-Type: application/json" \
107 + -d '{
108 + "model": "Qwen/Qwen3-32B",
109 + "messages": [{"role": "user", "content": "Hello"}],
110 + "max_completion_tokens": 16
111 + }'
112 +```
113 +
114 ### Example Request
115
79 -After either startup path, you can use any OpenAI-compatible client:
116 +After either full-frontend startup path, you can use any OpenAI-compatible
117 +client against the inference endpoints:
118
119 ```bash
120 curl http://127.0.0.1:8000/v1/chat/completions \
rust/src/chat/src/lib.rs
+31 -22
@@ -97,6 +97,10 @@ pub struct ChatRequestProcessor {
97 /// Effective model dtype reported by the engine.
98 /// Absent for text-only frontends without an engine handshake.
99 model_dtype: Option<ModelDtype>,
100 + /// Tool-call parser selection used when preparing generation requests.
101 + tool_call_parser: ParserSelection,
102 + /// Reasoning parser selection used when preparing generation requests.
103 + reasoning_parser: ParserSelection,
104 }
105
106 impl ChatRequestProcessor {
@@ -106,6 +110,8 @@ impl ChatRequestProcessor {
110 Self {
111 backend,
112 model_dtype: Some(model_dtype),
113 + tool_call_parser: ParserSelection::Auto,
114 + reasoning_parser: ParserSelection::Auto,
115 }
116 }
117
@@ -114,9 +120,22 @@ impl ChatRequestProcessor {
120 Self {
121 backend,
122 model_dtype: None,
123 + tool_call_parser: ParserSelection::Auto,
124 + reasoning_parser: ParserSelection::Auto,
125 }
126 }
127
128 + /// Configure the parser selections used to prepare generation requests.
129 + pub fn with_parser_selections(
130 + mut self,
131 + tool_call_parser: ParserSelection,
132 + reasoning_parser: ParserSelection,
133 + ) -> Self {
134 + self.tool_call_parser = tool_call_parser;
135 + self.reasoning_parser = reasoning_parser;
136 + self
137 + }
138 +
139 async fn finalize_rendered_prompt(
140 &self,
141 request: &ChatRequest,
@@ -196,10 +215,15 @@ impl ChatRequestProcessor {
215 pub async fn prepare(
216 &self,
217 mut request: ChatRequest,
199 - options: NewChatOutputProcessorOptions<'_>,
218 ) -> Result<(TextRequest, DynChatOutputProcessor)> {
219 request.validate()?;
202 - let output_processor = self.backend.new_chat_output_processor(&mut request, options)?;
220 + let output_processor = self.backend.new_chat_output_processor(
221 + &mut request,
222 + NewChatOutputProcessorOptions {
223 + tool_call_parser: &self.tool_call_parser,
224 + reasoning_parser: &self.reasoning_parser,
225 + },
226 + )?;
227 let text_request = self.prepare_text_request(request).await?;
228 Ok((text_request, output_processor))
229 }
@@ -213,10 +237,6 @@ impl ChatRequestProcessor {
237 pub struct ChatLlm {
238 text: TextLlm,
239 processor: ChatRequestProcessor,
216 - /// Tool-call parser selection.
217 - tool_call_parser: ParserSelection,
218 - /// Reasoning parser selection.
219 - reasoning_parser: ParserSelection,
240 }
241
242 impl ChatLlm {
@@ -228,8 +248,6 @@ impl ChatLlm {
248 Self {
249 text,
250 processor: ChatRequestProcessor::new(backend, model_dtype),
231 - tool_call_parser: ParserSelection::Auto,
232 - reasoning_parser: ParserSelection::Auto,
251 }
252 }
253
@@ -242,13 +260,13 @@ impl ChatLlm {
260
261 /// Set tool-call parser selection.
262 pub fn with_tool_call_parser(mut self, selection: ParserSelection) -> Self {
245 - self.tool_call_parser = selection;
263 + self.processor.tool_call_parser = selection;
264 self
265 }
266
267 /// Set reasoning parser selection.
268 pub fn with_reasoning_parser(mut self, selection: ParserSelection) -> Self {
251 - self.reasoning_parser = selection;
269 + self.processor.reasoning_parser = selection;
270 self
271 }
272
@@ -306,7 +324,7 @@ impl ChatLlm {
324
325 /// Effective tool-call parser name for this model, if parsing is enabled.
326 pub fn tool_call_parser_name(&self) -> Option<&str> {
309 - match &self.tool_call_parser {
327 + match &self.processor.tool_call_parser {
328 ParserSelection::Auto => {
329 ToolParserFactory::global().resolve_name_for_model(self.model_id())
330 }
@@ -317,7 +335,7 @@ impl ChatLlm {
335
336 /// Effective reasoning parser name for this model, if parsing is enabled.
337 pub fn reasoning_parser_name(&self) -> Option<&str> {
320 - match &self.reasoning_parser {
338 + match &self.processor.reasoning_parser {
339 ParserSelection::Auto => {
340 ReasoningParserFactory::global().resolve_name_for_model(self.model_id())
341 }
@@ -328,16 +346,7 @@ impl ChatLlm {
346
347 /// Render, tokenize, and submit one chat request.
348 pub async fn chat(&self, request: ChatRequest) -> Result<ChatEventStream> {
331 - let (text_request, output_processor) = self
332 - .processor
333 - .prepare(
334 - request,
335 - NewChatOutputProcessorOptions {
336 - tool_call_parser: &self.tool_call_parser,
337 - reasoning_parser: &self.reasoning_parser,
338 - },
339 - )
340 - .await?;
349 + let (text_request, output_processor) = self.processor.prepare(request).await?;
350 let request_id = text_request.request_id.clone();
351 let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
352
rust/src/cmd/src/cli.rs
+65 -1
@@ -29,7 +29,8 @@ use vllm_managed_engine::ManagedEngineConfig;
29 use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
30 use vllm_server::{
31 ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
32 - DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig,
32 + DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RenderConfig, RendererSelection,
33 + TlsConfig,
34 };
35
36 use crate::cli::unsupported::UnsupportedArgs;
@@ -90,6 +91,8 @@ pub enum Command {
91 /// Run vLLM benchmarks.
92 #[command(subcommand)]
93 Bench(BenchCommand),
94 + /// Run engine-free request rendering and preprocessing.
95 + Render(RenderArgs),
96 }
97
98 /// Supported benchmark commands.
@@ -99,6 +102,67 @@ pub enum BenchCommand {
102 Serve(vllm_bench::BenchServeArgs),
103 }
104
105 +/// Arguments for the engine-free text renderer.
106 +#[derive(Debug, Args, PartialEq, Eq)]
107 +pub struct RenderArgs {
108 + /// Model identifier or local model directory containing tokenizer files.
109 + model: String,
110 + /// HTTP bind host.
111 + #[arg(long, default_value = "127.0.0.1")]
112 + host: String,
113 + /// HTTP bind port.
114 + #[arg(long, default_value_t = 8000)]
115 + port: u16,
116 + /// Public model names accepted by the API.
117 + #[arg(long, num_args = 0..)]
118 + served_model_name: Vec<String>,
119 + /// Select the tool-call parser. Use `auto` to infer from the model or
120 + /// `none` to disable parsing.
121 + #[arg(long, default_value_t)]
122 + tool_call_parser: ParserSelection,
123 + /// Select the reasoning parser. Use `auto` to infer from the model or
124 + /// `none` to disable parsing.
125 + #[arg(long, default_value_t)]
126 + reasoning_parser: ParserSelection,
127 + /// Select the native chat renderer implementation.
128 + #[arg(long = "tokenizer-mode", default_value_t)]
129 + renderer: RendererSelection,
130 + /// Override the model chat template with a file path or inline template.
131 + #[arg(long)]
132 + chat_template: Option<String>,
133 + /// Default JSON keyword arguments merged into every chat-template render.
134 + #[arg(long, value_parser = parse_json::<HashMap<String, Value>>, value_name = "JSON")]
135 + default_chat_template_kwargs: Option<HashMap<String, Value>>,
136 + /// How message content is exposed to the chat template.
137 + #[arg(long, default_value_t)]
138 + chat_template_content_format: ChatTemplateContentFormatOption,
139 + /// Maximum model context length used for request validation.
140 + #[arg(long)]
141 + max_model_len: u32,
142 + /// Maximum accepted logprobs count; -1 disables the cap.
143 + #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
144 + max_logprobs: Option<i32>,
145 +}
146 +
147 +impl RenderArgs {
148 + pub(super) fn into_config(self) -> RenderConfig {
149 + RenderConfig {
150 + model: self.model,
151 + served_model_name: self.served_model_name,
152 + host: self.host,
153 + port: self.port,
154 + tool_call_parser: self.tool_call_parser,
155 + reasoning_parser: self.reasoning_parser,
156 + renderer: self.renderer,
157 + chat_template: self.chat_template,
158 + default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(),
159 + chat_template_content_format: self.chat_template_content_format,
160 + max_model_len: self.max_model_len,
161 + max_logprobs: self.max_logprobs,
162 + }
163 + }
164 +}
165 +
166 /// A JSON-encoded list of strings, matching Python's `json.loads` CLI type for
167 /// the CORS list arguments (e.g. `--allowed-origins '["*"]'`). Parsing the whole
168 /// value as one item keeps clap from treating the field as a repeated flag.
rust/src/cmd/src/cli/tests.rs
+35
@@ -27,6 +27,41 @@ fn bench_serve_args_parse_without_managed_engine_repartition() {
27 assert!(args.request_rate.is_infinite());
28 }
29
30 +#[test]
31 +fn render_args_build_supported_config() {
32 + let cli = Cli::try_parse_from([
33 + "vllm-rs",
34 + "render",
35 + "Qwen/Qwen2.5-0.5B-Instruct",
36 + "--port",
37 + "8080",
38 + "--max-model-len",
39 + "32768",
40 + "--served-model-name",
41 + "qwen",
42 + "--tokenizer-mode",
43 + "deepseek_v32",
44 + "--max-logprobs",
45 + "-1",
46 + ])
47 + .unwrap();
48 +
49 + let Command::Render(args) = cli.command else {
50 + panic!("expected render args");
51 + };
52 + let config = args.into_config();
53 +
54 + assert_eq!(config.model, "Qwen/Qwen2.5-0.5B-Instruct");
55 + assert_eq!(config.host, "127.0.0.1");
56 + assert_eq!(config.port, 8080);
57 + assert_eq!(config.max_model_len, 32768);
58 + assert_eq!(config.served_model_name, ["qwen"]);
59 + assert_eq!(config.tool_call_parser, ParserSelection::Auto);
60 + assert_eq!(config.reasoning_parser, ParserSelection::Auto);
61 + assert_eq!(config.renderer, RendererSelection::DeepSeekV32);
62 + assert_eq!(config.max_logprobs, Some(-1));
63 +}
64 +
65 #[test]
66 fn serve_args_forward_python_flags_with_separator() {
67 let cli = Cli::try_parse_from([
rust/src/cmd/src/main.rs
+3
@@ -211,5 +211,8 @@ async fn async_main(cli: Cli) -> Result<()> {
211 )),
212 }
213 }
214 + Command::Render(args) => {
215 + vllm_server::serve_render(args.into_config(), shutdown_signal()).await
216 + }
217 }
218 }
rust/src/server/src/lib.rs
+2
@@ -9,6 +9,7 @@ mod grpc;
9 mod listener;
10 mod lora;
11 mod middleware;
12 +mod render;
13 mod routes;
14 mod runtime;
15 mod server_info;
@@ -35,6 +36,7 @@ use hyper::server::conn::http1;
36 use hyper_util::rt::{TokioIo, TokioTimer};
37 use hyper_util::server::graceful::GracefulShutdown;
38 use hyper_util::service::TowerToHyperService;
39 +pub use render::{RenderConfig, serve_render};
40 use tokio::net::TcpListener;
41 use tokio::time::{Instant, sleep_until};
42 use tokio_util::sync::CancellationToken;
rust/src/server/src/render.rs new
+146
@@ -0,0 +1,146 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use std::collections::HashMap;
5 +use std::sync::Arc;
6 +
7 +use anyhow::{Context as _, Result, bail};
8 +use serde_json::Value;
9 +use tokio_util::sync::CancellationToken;
10 +use tracing::info;
11 +use vllm_chat::{
12 + ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions,
13 + ParserSelection, RendererSelection, load_model_backends,
14 +};
15 +use vllm_text::TextRequestProcessor;
16 +
17 +use crate::{HttpListenerMode, listener::Listener};
18 +
19 +/// Configuration for the engine-free text rendering server.
20 +#[derive(Debug)]
21 +pub struct RenderConfig {
22 + pub model: String,
23 + pub served_model_name: Vec<String>,
24 + pub host: String,
25 + pub port: u16,
26 + pub tool_call_parser: ParserSelection,
27 + pub reasoning_parser: ParserSelection,
28 + pub renderer: RendererSelection,
29 + pub chat_template: Option<String>,
30 + pub default_chat_template_kwargs: HashMap<String, Value>,
31 + pub chat_template_content_format: ChatTemplateContentFormatOption,
32 + pub max_model_len: u32,
33 + pub max_logprobs: Option<i32>,
34 +}
35 +
36 +impl RenderConfig {
37 + /// Validate configuration before initializing renderer/tokenizer backends
38 + /// or binding a listener.
39 + pub fn validate(&self) -> Result<()> {
40 + vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
41 + if self.max_logprobs.is_some_and(|value| value < -1) {
42 + bail!("max_logprobs must be non-negative or -1");
43 + }
44 + Ok(())
45 + }
46 +}
47 +
48 +pub(crate) struct RenderState {
49 + pub(crate) model: String,
50 + pub(crate) served_model_names: Vec<String>,
51 + pub(crate) text: TextRequestProcessor,
52 + pub(crate) chat: ChatRequestProcessor,
53 +}
54 +
55 +async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
56 + let loaded = load_model_backends(
57 + &config.model,
58 + LoadModelBackendsOptions {
59 + renderer: config.renderer,
60 + language_model_only: true,
61 + chat_template: config.chat_template.clone(),
62 + chat_template_content_format: config.chat_template_content_format,
63 + default_chat_template_kwargs: config.default_chat_template_kwargs.clone(),
64 + limit_mm_per_prompt: Default::default(),
65 + },
66 + )
67 + .await
68 + .context("failed to load renderer/tokenizer backends")?;
69 + let served_model_names =
70 + crate::effective_served_model_names(&config.model, &config.served_model_name);
71 + let text = TextRequestProcessor::new(loaded.text_backend, config.max_model_len)
72 + .with_max_logprobs(config.max_logprobs);
73 + let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
74 + config.tool_call_parser.clone(),
75 + config.reasoning_parser.clone(),
76 + );
77 + Ok(Arc::new(RenderState {
78 + model: config.model.clone(),
79 + served_model_names,
80 + text,
81 + chat,
82 + }))
83 +}
84 +
85 +/// Run the text-only preprocessing server without starting or connecting
86 +/// to an inference engine.
87 +pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> {
88 + config.validate().context("invalid render server configuration")?;
89 + let state = tokio::select! {
90 + result = build_state(&config) => result?,
91 + _ = shutdown.cancelled() => return Ok(()),
92 + };
93 + let listener_mode = HttpListenerMode::BindTcp {
94 + host: config.host.clone(),
95 + port: config.port,
96 + };
97 + let listener = Listener::bind(&listener_mode)
98 + .await
99 + .with_context(|| format!("failed to bind {}:{}", config.host, config.port))?;
100 + let address = listener.local_addr_display()?;
101 + info!(
102 + %address,
103 + model = %config.model,
104 + "starting engine-free Rust render server"
105 + );
106 + axum::serve(listener, crate::routes::render::build_router(state))
107 + .with_graceful_shutdown(shutdown.cancelled_owned())
108 + .await
109 + .context("render server failed")
110 +}
111 +
112 +#[cfg(test)]
113 +mod tests {
114 + use super::*;
115 +
116 + #[tokio::test]
117 + async fn serve_render_rejects_unknown_parser_before_startup() {
118 + let shutdown = CancellationToken::new();
119 + shutdown.cancel();
120 + let error = serve_render(
121 + RenderConfig {
122 + model: "test-model".to_string(),
123 + served_model_name: Vec::new(),
124 + host: "127.0.0.1".to_string(),
125 + port: 8000,
126 + tool_call_parser: ParserSelection::Auto,
127 + reasoning_parser: ParserSelection::Explicit("typo".to_string()),
128 + renderer: RendererSelection::Auto,
129 + chat_template: None,
130 + default_chat_template_kwargs: HashMap::new(),
131 + chat_template_content_format: ChatTemplateContentFormatOption::Auto,
132 + max_model_len: 128,
133 + max_logprobs: None,
134 + },
135 + shutdown,
136 + )
137 + .await
138 + .unwrap_err();
139 + let report = format!("{error:#}");
140 +
141 + assert!(
142 + report.contains("reasoning parser `typo` is not registered"),
143 + "{report}"
144 + );
145 + }
146 +}
rust/src/server/src/routes.rs
+1
@@ -12,6 +12,7 @@ mod metrics;
12 pub(crate) mod openai;
13 mod pause;
14 mod profile;
15 +pub(super) mod render;
16 mod server_info;
17 mod sleep;
18 mod tokenize;
rust/src/server/src/routes/inference/generate.rs
+4 -2
@@ -26,10 +26,12 @@ use vllm_llm::{
26 };
27
28 use self::convert::{ResponseOptions, prepare_generate_request};
29 +pub(crate) use self::types::GenerateRequest;
30 use self::types::{
30 - GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
31 - GenerateResponseStreamChoice, GenerateStreamResponse,
31 + GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
32 + GenerateStreamResponse,
33 };
34 +pub(crate) use self::validate::validate_request_compat;
35 use crate::config::ApiServerOptions;
36 use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
37 use crate::routes::openai::utils::logprobs::clamp_logprob;
rust/src/server/src/routes/inference/generate/validate.rs
+1 -1
@@ -6,7 +6,7 @@ use crate::error::{ApiError, bail_invalid_request};
6
7 /// Enforce the minimal compatibility contract for the Rust token generate
8 /// route.
9 -pub(super) fn validate_request_compat(
9 +pub(crate) fn validate_request_compat(
10 request: &GenerateRequest,
11 served_model_names: &[String],
12 ) -> Result<(), ApiError> {
rust/src/server/src/routes/openai/chat_completions.rs
+14 -5
@@ -22,17 +22,18 @@ use tracing::{debug, error, info, trace};
22 use tracing_futures::Instrument as _;
23 use vllm_chat::{
24 AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatEventStream, ChatEventStreamTrait,
25 - CollectedAssistantMessage, FinishReason,
25 + ChatRequest, CollectedAssistantMessage, FinishReason,
26 };
27 use vllm_engine_core_client::protocol::output::StopReason;
28
29 use self::convert::{ResponseOptions, prepare_chat_request};
30 +pub(crate) use self::types::ChatCompletionRequest;
31 use crate::config::ApiServerOptions;
32 use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error};
33 +use crate::lora::LoraModelResolution;
34 use crate::routes::openai::chat_completions::types::{
33 - AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
34 - ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
35 - ChatMessageDelta,
35 + AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionResponse,
36 + ChatCompletionStreamChoice, ChatCompletionStreamResponse, ChatMessageDelta,
37 };
38 use crate::routes::openai::utils::logprobs::{
39 decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps,
@@ -43,7 +44,15 @@ use crate::routes::openai::utils::types::{
44 use crate::routes::openai::utils::usage::ContinuousUsage;
45 use crate::routes::openai::utils::validated_json::ValidatedJson;
46 use crate::state::AppState;
46 -use crate::utils::{resolve_request_context, unix_timestamp};
47 +use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
48 +
49 +pub(crate) fn lower_chat_request(
50 + request: ChatCompletionRequest,
51 + lora_resolution: &LoraModelResolution,
52 + ctx: ResolvedRequestContext,
53 +) -> Result<ChatRequest, ApiError> {
54 + prepare_chat_request(request, lora_resolution, ctx).map(|prepared| prepared.chat_request)
55 +}
56
57 /// Validate one chat completion request and proxy it into the shared
58 /// `vllm-chat` stack.
rust/src/server/src/routes/openai/completions.rs
+17 -4
@@ -21,12 +21,14 @@ use thiserror_ext::AsReport as _;
21 use tracing::{debug, error, info, trace};
22 use tracing_futures::Instrument as _;
23 use vllm_engine_core_client::protocol::output::StopReason;
24 +use vllm_text::tokenizer::Tokenizer;
25 use vllm_text::{
26 DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
26 - TextOutputStreamExt as _,
27 + TextOutputStreamExt as _, TextRequest,
28 };
29
30 use self::convert::{ResponseOptions, prepare_completion_request};
31 +pub(crate) use self::types::CompletionRequest;
32 use super::utils::logprobs::{
33 collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai,
34 prompt_logprobs_to_maps, text_len,
@@ -34,15 +36,26 @@ use super::utils::logprobs::{
36 use super::utils::types::Usage;
37 use crate::config::ApiServerOptions;
38 use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
39 +use crate::lora::LoraModelResolution;
40 use crate::routes::openai::completions::types::{
38 - CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
39 - CompletionStreamChoice, CompletionStreamResponse,
41 + CompletionChoice, CompletionResponse, CompletionSseChunk, CompletionStreamChoice,
42 + CompletionStreamResponse,
43 };
44 use crate::routes::openai::utils::types::LogProbs;
45 use crate::routes::openai::utils::usage::ContinuousUsage;
46 use crate::routes::openai::utils::validated_json::ValidatedJson;
47 use crate::state::AppState;
45 -use crate::utils::{resolve_request_context, unix_timestamp};
48 +use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
49 +
50 +pub(crate) fn lower_completion_request(
51 + request: CompletionRequest,
52 + lora_resolution: &LoraModelResolution,
53 + ctx: ResolvedRequestContext,
54 + tokenizer: &dyn Tokenizer,
55 +) -> Result<TextRequest, ApiError> {
56 + prepare_completion_request(request, lora_resolution, ctx, tokenizer)
57 + .map(|prepared| prepared.text_request)
58 +}
59
60 /// Validate one completions request and proxy it into the shared `vllm-text`
61 /// stack.
rust/src/server/src/routes/openai/mod.rs
+2
@@ -7,5 +7,7 @@ mod models;
7 pub(crate) mod utils;
8
9 pub use chat_completions::chat_completions;
10 +pub(crate) use chat_completions::{ChatCompletionRequest, lower_chat_request};
11 pub use completions::completions;
12 +pub(crate) use completions::{CompletionRequest, lower_completion_request};
13 pub use models::list_models;
rust/src/server/src/routes/render.rs new
+149
@@ -0,0 +1,149 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use std::sync::Arc;
5 +
6 +use axum::Json;
7 +use axum::Router;
8 +use axum::extract::DefaultBodyLimit;
9 +use axum::extract::State;
10 +use axum::http::{HeaderMap, StatusCode};
11 +use axum::routing::{get, post};
12 +use thiserror_ext::AsReport as _;
13 +use vllm_text::TextRequest;
14 +
15 +use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
16 +use crate::error::{ApiError, text_submit_error};
17 +use crate::lora::LoraModelResolution;
18 +use crate::render::RenderState;
19 +use crate::routes::inference::generate::{
20 + GenerateRequest, validate_request_compat as validate_generate_request,
21 +};
22 +use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
23 +use crate::routes::openai::utils::validated_json::ValidatedJson;
24 +use crate::routes::openai::{
25 + ChatCompletionRequest, CompletionRequest, lower_chat_request, lower_completion_request,
26 +};
27 +use crate::utils::{resolve_request_context, unix_timestamp};
28 +
29 +pub(crate) fn build_router(state: Arc<RenderState>) -> Router {
30 + Router::new()
31 + .route("/health", get(health))
32 + .route("/ping", get(health).post(health))
33 + .route("/v1/models", get(list_models))
34 + .route("/v1/chat/completions/render", post(render_chat))
35 + .route("/v1/completions/render", post(render_completion))
36 + .with_state(state)
37 + .layer(DefaultBodyLimit::max(DEFAULT_REQUEST_BODY_LIMIT_BYTES))
38 +}
39 +
40 +async fn health() -> StatusCode {
41 + StatusCode::OK
42 +}
43 +
44 +async fn list_models(State(state): State<Arc<RenderState>>) -> Json<ListModelsResponse> {
45 + let created = unix_timestamp() as i64;
46 + Json(ListModelsResponse {
47 + object: "list".to_string(),
48 + data: state
49 + .served_model_names
50 + .iter()
51 + .map(|model| ModelObject {
52 + id: model.clone(),
53 + object: "model".to_string(),
54 + created,
55 + owned_by: "vllm".to_string(),
56 + root: Some(state.model.clone()),
57 + parent: None,
58 + max_model_len: Some(state.text.max_model_len()),
59 + })
60 + .collect(),
61 + })
62 +}
63 +
64 +fn model_resolution(state: &RenderState) -> LoraModelResolution {
65 + LoraModelResolution {
66 + model_names: state.served_model_names.clone(),
67 + lora_request: None,
68 + }
69 +}
70 +
71 +fn lower_render_request(
72 + state: &RenderState,
73 + text_request: TextRequest,
74 + model: String,
75 + stream: bool,
76 + stream_options: Option<StreamOptions>,
77 +) -> Result<GenerateRequest, ApiError> {
78 + let prepared = state
79 + .text
80 + .prepare(text_request)
81 + .map_err(|error| text_submit_error("failed to prepare render request", error))?;
82 + let token_ids = prepared.generate_request.prompt_token_ids;
83 + let text_request = prepared.text_request;
84 +
85 + let request = GenerateRequest {
86 + request_id: Some(text_request.request_id),
87 + model: Some(model),
88 + token_ids,
89 + sampling_params: text_request.sampling_params,
90 + stream,
91 + stream_options,
92 + cache_salt: text_request.cache_salt,
93 + priority: text_request.priority,
94 + kv_transfer_params: None,
95 + ec_transfer_params: None,
96 + other: Default::default(),
97 + };
98 + validate_generate_request(&request, &state.served_model_names)?;
99 + Ok(request)
100 +}
101 +
102 +async fn render_chat(
103 + State(state): State<Arc<RenderState>>,
104 + headers: HeaderMap,
105 + ValidatedJson(body): ValidatedJson<ChatCompletionRequest>,
106 +) -> Result<Json<GenerateRequest>, ApiError> {
107 + let model = body.model.clone();
108 + let stream = body.stream;
109 + let stream_options = body.stream_options.clone();
110 + let request_context = resolve_request_context(&headers, body.request_id.as_deref());
111 + let chat_request = lower_chat_request(body, &model_resolution(&state), request_context)?;
112 + let (text_request, _) = state
113 + .chat
114 + .prepare(chat_request)
115 + .await
116 + .map_err(|error| ApiError::invalid_request(error.to_report_string(), None))?;
117 + Ok(Json(lower_render_request(
118 + &state,
119 + text_request,
120 + model,
121 + stream,
122 + stream_options,
123 + )?))
124 +}
125 +
126 +async fn render_completion(
127 + State(state): State<Arc<RenderState>>,
128 + headers: HeaderMap,
129 + ValidatedJson(body): ValidatedJson<CompletionRequest>,
130 +) -> Result<Json<Vec<GenerateRequest>>, ApiError> {
131 + let model = body.model.clone();
132 + let stream = body.stream;
133 + let stream_options = body.stream_options.clone();
134 + let request_context = resolve_request_context(&headers, body.request_id.as_deref());
135 + let tokenizer = state.text.tokenizer();
136 + let text_request = lower_completion_request(
137 + body,
138 + &model_resolution(&state),
139 + request_context,
140 + tokenizer.as_ref(),
141 + )?;
142 + Ok(Json(vec![lower_render_request(
143 + &state,
144 + text_request,
145 + model,
146 + stream,
147 + stream_options,
148 + )?]))
149 +}
rust/src/server/src/routes/tests.rs
+211 -4
@@ -23,8 +23,8 @@ use serial_test::serial;
23 use tower::{Service as _, ServiceExt as _};
24 use vllm_chat::{
25 ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest,
26 - ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
27 - NewChatOutputProcessorOptions,
26 + ChatRequestProcessor, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor,
27 + DynChatRenderer, NewChatOutputProcessorOptions, ParserSelection,
28 };
29 use vllm_engine_core_client::mock_engine::default_ready_response;
30 use vllm_engine_core_client::protocol::decode_value;
@@ -46,13 +46,17 @@ use vllm_engine_core_client::{
46 use vllm_llm::Llm;
47 use vllm_metrics::METRICS;
48 use vllm_text::tokenizer::DynTokenizer;
49 -use vllm_text::{Prompt, TextBackend};
49 +use vllm_text::{Prompt, TextBackend, TextRequestProcessor};
50 use vllm_tokenizer::test_utils::TestTokenizer;
51 use zeromq::prelude::{SocketRecv, SocketSend};
52 use zeromq::{DealerSocket, PushSocket, ZmqMessage};
53
54 -use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
54 +use super::{
55 + build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
56 + render::build_router as build_render_router,
57 +};
58 use crate::config::{ApiServerOptions, CorsConfig};
59 +use crate::render::RenderState;
60 use crate::state::AppState;
61
62 fn request_output(
@@ -682,6 +686,24 @@ async fn test_app() -> axum::Router {
686 test_app_with_dev_mode(false).await
687 }
688
689 +fn test_render_app() -> axum::Router {
690 + test_render_app_with_parser_selections(ParserSelection::Auto, ParserSelection::Auto)
691 +}
692 +
693 +fn test_render_app_with_parser_selections(
694 + tool_call_parser: ParserSelection,
695 + reasoning_parser: ParserSelection,
696 +) -> axum::Router {
697 + let backend = Arc::new(FakeChatBackend::new());
698 + build_render_router(Arc::new(RenderState {
699 + model: "backend-model".to_string(),
700 + served_model_names: vec!["render-model".to_string()],
701 + text: TextRequestProcessor::new(backend.clone(), 128),
702 + chat: ChatRequestProcessor::render_only(backend)
703 + .with_parser_selections(tool_call_parser, reasoning_parser),
704 + }))
705 +}
706 +
707 async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
708 let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
709 b"engine-openai",
@@ -1043,6 +1065,191 @@ fn metric_delta(
1065 - metric_value(rendered_before, metric, labels).unwrap_or(0.0)
1066 }
1067
1068 +#[tokio::test]
1069 +async fn render_chat_returns_generate_request_with_header_request_id() {
1070 + let mut app = test_render_app();
1071 + let response = app
1072 + .call(
1073 + Request::builder()
1074 + .method("POST")
1075 + .uri("/v1/chat/completions/render")
1076 + .header("content-type", "application/json")
1077 + .header("X-Request-Id", "header-req")
1078 + .body(Body::from(
1079 + json!({
1080 + "request_id": "body-req",
1081 + "model": "render-model",
1082 + "messages": [{"role": "user", "content": "hello"}],
1083 + "max_completion_tokens": 8,
1084 + "stream": true,
1085 + "stream_options": {"include_usage": true},
1086 + "cache_salt": "render-cache-salt",
1087 + "priority": -3
1088 + })
1089 + .to_string(),
1090 + ))
1091 + .expect("build request"),
1092 + )
1093 + .await
1094 + .expect("call app");
1095 +
1096 + assert_eq!(response.status(), StatusCode::OK);
1097 + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
1098 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1099 +
1100 + assert_eq!(json["request_id"], "chatcmpl-header-req");
1101 + assert_eq!(json["model"], "render-model");
1102 + assert_eq!(json["stream"], true);
1103 + assert_eq!(json["stream_options"]["include_usage"], true);
1104 + assert_eq!(json["cache_salt"], "render-cache-salt");
1105 + assert_eq!(json["priority"], -3);
1106 + assert_eq!(json["sampling_params"]["max_tokens"], 8);
1107 + assert!(!json["token_ids"].as_array().unwrap().is_empty());
1108 + assert!(json.get("prompt_token_ids").is_none());
1109 + assert!(json.get("mm_features").is_none());
1110 +}
1111 +
1112 +#[tokio::test]
1113 +async fn render_chat_uses_processor_owned_parser_selections() {
1114 + let mut app = test_render_app_with_parser_selections(
1115 + ParserSelection::Auto,
1116 + ParserSelection::Explicit("definitely_missing_reasoning_parser".to_string()),
1117 + );
1118 + let response = app
1119 + .call(
1120 + Request::builder()
1121 + .method("POST")
1122 + .uri("/v1/chat/completions/render")
1123 + .header("content-type", "application/json")
1124 + .body(Body::from(
1125 + json!({
1126 + "model": "render-model",
1127 + "messages": [{"role": "user", "content": "hello"}],
1128 + "max_completion_tokens": 8
1129 + })
1130 + .to_string(),
1131 + ))
1132 + .expect("build request"),
1133 + )
1134 + .await
1135 + .expect("call app");
1136 +
1137 + assert_eq!(response.status(), StatusCode::BAD_REQUEST);
1138 + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
1139 + let body = String::from_utf8(body.to_vec()).expect("decode body");
1140 + assert!(
1141 + body.contains("reasoning parser `definitely_missing_reasoning_parser` is not registered")
1142 + );
1143 +}
1144 +
1145 +#[tokio::test]
1146 +async fn render_completion_returns_generate_request_with_body_request_id() {
1147 + let mut app = test_render_app();
1148 + let response = app
1149 + .call(
1150 + Request::builder()
1151 + .method("POST")
1152 + .uri("/v1/completions/render")
1153 + .header("content-type", "application/json")
1154 + .body(Body::from(
1155 + json!({
1156 + "request_id": "body-req",
1157 + "model": "render-model",
1158 + "prompt": "hello",
1159 + "max_tokens": 8
1160 + })
1161 + .to_string(),
1162 + ))
1163 + .expect("build request"),
1164 + )
1165 + .await
1166 + .expect("call app");
1167 +
1168 + assert_eq!(response.status(), StatusCode::OK);
1169 + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
1170 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1171 +
1172 + assert_eq!(json[0]["request_id"], "cmpl-body-req");
1173 + assert_eq!(json[0]["model"], "render-model");
1174 + assert_eq!(json[0]["stream"], false);
1175 + assert_eq!(json[0]["sampling_params"]["max_tokens"], 8);
1176 + assert!(!json[0]["token_ids"].as_array().unwrap().is_empty());
1177 + assert!(json[0].get("stream_options").is_none());
1178 + assert!(json[0].get("prompt_token_ids").is_none());
1179 + assert!(json[0].get("mm_features").is_none());
1180 +}
1181 +
1182 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1183 +#[serial]
1184 +async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
1185 + let mut render_app = test_render_app();
1186 + let render_response = render_app
1187 + .call(
1188 + Request::builder()
1189 + .method("POST")
1190 + .uri("/v1/chat/completions/render")
1191 + .header("content-type", "application/json")
1192 + .body(Body::from(
1193 + json!({
1194 + "request_id": "round-trip",
1195 + "model": "render-model",
1196 + "messages": [{"role": "user", "content": "hello"}],
1197 + "max_completion_tokens": 8,
1198 + "bad_words": ["blocked"],
1199 + "vllm_xargs": {"custom": 1}
1200 + })
1201 + .to_string(),
1202 + ))
1203 + .expect("build render request"),
1204 + )
1205 + .await
1206 + .expect("call render app");
1207 +
1208 + assert_eq!(render_response.status(), StatusCode::OK);
1209 + let render_body = to_bytes(render_response.into_body(), usize::MAX)
1210 + .await
1211 + .expect("read render response");
1212 + let render_json: serde_json::Value =
1213 + serde_json::from_slice(&render_body).expect("decode render response");
1214 + let token_ids: Vec<u32> = serde_json::from_value(render_json["token_ids"].clone())
1215 + .expect("decode rendered token IDs");
1216 +
1217 + assert_eq!(
1218 + render_json["sampling_params"]["bad_words"],
1219 + json!(["blocked"])
1220 + );
1221 + assert_eq!(render_json["sampling_params"]["vllm_xargs"]["custom"], 1);
1222 +
1223 + let (chat, engine_task) = test_models_with_engine_outputs_and_backend_inner(
1224 + b"engine-render-generate-round-trip",
1225 + default_stream_output_specs(),
1226 + Some(token_ids),
1227 + Arc::new(FakeChatBackend::new()),
1228 + )
1229 + .await;
1230 + let mut generate_app = build_router(Arc::new(AppState::new(
1231 + vec!["render-model".to_string()],
1232 + chat,
1233 + )));
1234 + let generate_response = generate_app
1235 + .call(
1236 + Request::builder()
1237 + .method("POST")
1238 + .uri("/inference/v1/generate")
1239 + .header("content-type", "application/json")
1240 + .body(Body::from(render_body))
1241 + .expect("build generate request"),
1242 + )
1243 + .await
1244 + .expect("call generate app");
1245 +
1246 + assert_eq!(generate_response.status(), StatusCode::OK);
1247 + let _ = to_bytes(generate_response.into_body(), usize::MAX)
1248 + .await
1249 + .expect("read generate response");
1250 + engine_task.finish().await;
1251 +}
1252 +
1253 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1254 #[serial]
1255 async fn list_models_returns_configured_model() {