[Frontend] Add content_parts to /inference/v1/generate for raw multim… (#51478)
Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
aoshen02 committed
Aug 11, 2026 at 10:00 UTC
33584901bab6900045b504843a57e172205d3ff1
7 files changed
+150
-7
rust/src/server/src/routes/inference/generate.rs
+21
-4
@@ -45,14 +45,31 @@ use crate::utils::resolve_request_context;
45
pub async fn generate(
46
State(state): State<Arc<AppState>>,
47
headers: HeaderMap,
48
- ValidatedJson(body): ValidatedJson<GenerateRequest>,
48
+ ValidatedJson(mut body): ValidatedJson<GenerateRequest>,
49
) -> Response {
50
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
51
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
52
- let prepared = match prepare_generate_request(body, &lora_resolution, request_context) {
53
- Ok(prepared) => prepared,
54
- Err(error) => return error.into_response(),
52
+
53
+ let mm_features = if let Some(parts) = body.content_parts.take() {
54
+ match state.chat.prepare_media(parts, &mut body.token_ids).await {
55
+ Ok(features) => features,
56
+ Err(e) => {
57
+ return ApiError::invalid_request(
58
+ format!("failed to resolve content_parts: {}", e.as_report()),
59
+ Some("content_parts"),
60
+ )
61
+ .into_response();
62
+ }
63
+ }
64
+ } else {
65
+ None
66
};
67
+
68
+ let prepared =
69
+ match prepare_generate_request(body, &lora_resolution, request_context, mm_features) {
70
+ Ok(prepared) => prepared,
71
+ Err(error) => return error.into_response(),
72
+ };
73
let request_span = tracing::info_span!(
74
"generate",
75
request_id = %prepared.request_id,
rust/src/server/src/routes/inference/generate/convert.rs
+6
-1
@@ -1,6 +1,7 @@
1
// SPDX-License-Identifier: Apache-2.0
2
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+use vllm_engine_core_client::protocol::multimodal::MmFeatures;
5
use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
6
7
use super::types::GenerateRequest;
@@ -37,6 +38,7 @@ pub(super) fn prepare_generate_request(
38
request: GenerateRequest,
39
lora_resolution: &LoraModelResolution,
40
ctx: ResolvedRequestContext,
41
+ mm_features: Option<MmFeatures>,
42
) -> Result<PreparedRequest, ApiError> {
43
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
44
@@ -67,7 +69,7 @@ pub(super) fn prepare_generate_request(
69
let text_request = TextRequest {
70
request_id: ctx.request_id.clone(),
71
prompt: Prompt::TokenIds(request.token_ids),
70
- mm_features: None,
72
+ mm_features,
73
sampling_params,
74
decode_options: TextDecodeOptions::default(),
75
intermediate: false,
@@ -134,6 +136,7 @@ mod tests {
136
request,
137
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
138
ResolvedRequestContext::default(),
139
+ None,
140
)
141
.expect("prepare");
142
@@ -175,6 +178,7 @@ mod tests {
178
request,
179
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
180
ResolvedRequestContext::default(),
181
+ None,
182
)
183
.expect("prepare");
184
@@ -204,6 +208,7 @@ mod tests {
208
request,
209
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
210
ResolvedRequestContext::default(),
211
+ None,
212
)
213
.expect("prepare");
214
rust/src/server/src/routes/inference/generate/types.rs
+3
@@ -3,6 +3,7 @@
3
4
use std::collections::HashMap;
5
6
+use llm_multimodal::MediaContentPart;
7
use serde::{Deserialize, Serialize};
8
use serde_json::{Map, Value};
9
use validator::Validate;
@@ -26,6 +27,8 @@ pub struct GenerateRequest {
27
pub priority: i32,
28
pub kv_transfer_params: Option<HashMap<String, Value>>,
29
pub ec_transfer_params: Option<HashMap<String, Value>>,
30
+ /// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
31
+ pub content_parts: Option<Vec<MediaContentPart>>,
32
#[serde(flatten)]
33
pub other: Map<String, Value>,
34
}
rust/src/server/src/routes/render.rs
+1
@@ -93,6 +93,7 @@ fn lower_render_request(
93
priority: text_request.priority,
94
kv_transfer_params: None,
95
ec_transfer_params: None,
96
+ content_parts: None,
97
other: Default::default(),
98
};
99
validate_generate_request(&request, &state.served_model_names)?;
tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py
+84
@@ -156,3 +156,87 @@ async def test_render_to_generate_roundtrip(client, test_image):
156
assert "red" in text.lower(), (
157
f"Expected model to identify the red image, got: {text!r}"
158
)
159
+
160
+
161
+@pytest.mark.asyncio
162
+async def test_content_parts_generates_tokens(client, test_image):
163
+ """content_parts with raw media should produce output tokens."""
164
+ data_url = encode_image_url(test_image, format="PNG")
165
+
166
+ render_resp = await client.post(
167
+ RENDER_ENDPOINT,
168
+ json={
169
+ "model": MODEL_NAME,
170
+ "messages": [
171
+ {
172
+ "role": "user",
173
+ "content": [
174
+ {"type": "image_url", "image_url": {"url": data_url}},
175
+ {"type": "text", "text": "What color is this?"},
176
+ ],
177
+ }
178
+ ],
179
+ },
180
+ )
181
+ render_resp.raise_for_status()
182
+ token_ids = render_resp.json()["token_ids"]
183
+
184
+ gen_resp = await client.post(
185
+ GEN_ENDPOINT,
186
+ json={
187
+ "token_ids": token_ids,
188
+ "content_parts": [{"type": "image_url", "url": data_url}],
189
+ "sampling_params": {"max_tokens": 10, "temperature": 0.0},
190
+ },
191
+ )
192
+ gen_resp.raise_for_status()
193
+ gen_data = gen_resp.json()
194
+
195
+ assert "choices" in gen_data
196
+ choice = gen_data["choices"][0]
197
+ assert "token_ids" in choice
198
+ assert len(choice["token_ids"]) > 0
199
+
200
+
201
+@pytest.mark.asyncio
202
+async def test_content_parts_streaming(client, test_image):
203
+ """content_parts should work with streaming."""
204
+ import json as json_mod
205
+
206
+ data_url = encode_image_url(test_image, format="PNG")
207
+
208
+ render_resp = await client.post(
209
+ RENDER_ENDPOINT,
210
+ json={
211
+ "model": MODEL_NAME,
212
+ "messages": [
213
+ {
214
+ "role": "user",
215
+ "content": [
216
+ {"type": "image_url", "image_url": {"url": data_url}},
217
+ {"type": "text", "text": "Describe this."},
218
+ ],
219
+ }
220
+ ],
221
+ },
222
+ )
223
+ render_resp.raise_for_status()
224
+ token_ids = render_resp.json()["token_ids"]
225
+
226
+ async with client.stream(
227
+ "POST",
228
+ GEN_ENDPOINT,
229
+ json={
230
+ "token_ids": token_ids,
231
+ "content_parts": [{"type": "image_url", "url": data_url}],
232
+ "sampling_params": {"max_tokens": 8, "temperature": 0.0},
233
+ "stream": True,
234
+ },
235
+ ) as resp:
236
+ resp.raise_for_status()
237
+ chunks = []
238
+ async for line in resp.aiter_lines():
239
+ if line.startswith("data: ") and line != "data: [DONE]":
240
+ chunks.append(json_mod.loads(line[6:]))
241
+
242
+ assert len(chunks) > 0
vllm/entrypoints/scale_out/token_in_token_out/protocol.py
+10
@@ -104,6 +104,16 @@ class GenerateRequest(BaseModel):
104
features: MultiModalFeatures | None = None
105
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
106
107
+ content_parts: list[dict[str, Any]] | None = None
108
+ """Raw multimodal input; server resolves media. Mutually exclusive
109
+ with ``features``."""
110
+
111
+ @model_validator(mode="after")
112
+ def _check_mm_fields_exclusive(self) -> "GenerateRequest":
113
+ if self.content_parts and self.features:
114
+ raise ValueError("content_parts and features are mutually exclusive")
115
+ return self
116
+
117
sampling_params: SamplingParams
118
"""The sampling parameters for the model."""
119
vllm/entrypoints/scale_out/token_in_token_out/serving.py
+25
-2
@@ -14,6 +14,7 @@ import pybase64 as base64
14
from fastapi import Request
15
16
from vllm.engine.protocol import EngineClient
17
+from vllm.entrypoints.chat_utils import AsyncMultiModalItemTracker
18
from vllm.entrypoints.generate.base.serving import (
19
GenerateBaseServing,
20
clamp_prompt_logprobs,
@@ -33,7 +34,7 @@ from vllm.entrypoints.openai.engine.protocol import (
34
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
35
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
36
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
36
-from vllm.inputs import EngineInput, mm_input
37
+from vllm.inputs import EngineInput, TokensPrompt, mm_input
38
from vllm.logger import init_logger
39
from vllm.logprobs import Logprob
40
from vllm.multimodal.inputs import (
@@ -143,7 +144,29 @@ class ServingTokens(GenerateBaseServing):
144
return self.create_error_response(e)
145
146
engine_input: EngineInput
146
- if features := request.features:
147
+ if request.content_parts:
148
+ tracker = AsyncMultiModalItemTracker(self.model_config)
149
+ mm_parser = tracker.create_parser()
150
+ for part in request.content_parts:
151
+ ptype = part.get("type", "")
152
+ url = part.get("url")
153
+ uuid = part.get("uuid")
154
+ if ptype == "image_url":
155
+ mm_parser.parse_image(url, uuid)
156
+ elif ptype == "audio_url":
157
+ mm_parser.parse_audio(url, uuid)
158
+ elif ptype == "video_url":
159
+ mm_parser.parse_video(url, uuid)
160
+ mm_data, mm_uuids = await tracker.resolve_items()
161
+ prompt = TokensPrompt(prompt_token_ids=request.token_ids)
162
+ if mm_data:
163
+ prompt["multi_modal_data"] = mm_data
164
+ if mm_uuids:
165
+ prompt["multi_modal_uuids"] = mm_uuids
166
+ (engine_input,) = await self.online_renderer.renderer.render_cmpl_async(
167
+ [prompt]
168
+ )
169
+ elif features := request.features:
170
# Convert PlaceholderRangeInfo → PlaceholderRange per modality.
171
mm_placeholders: dict[str, list[PlaceholderRange]] = {
172
modality: [