@tduncan / Nvidia-Optimization-Stack-VLLM / commits / ac5f38a0f7

[Refactor] Extract StructuredOutputsParams creation logic from Request.to_sampling_params (#49003)

Signed-off-by: Yifan Zong <yzong@redhat.com>

yzong-rh committed Jul 19, 2026 at 08:18 UTC ac5f38a0f7af29cb5bc15a1e73623922d9832500

5 files changed +95 -98

tests/entrypoints/openai/responses/test_sampling_params.py

+15

@@ -132,6 +132,21 @@ class TestResponsesRequestSamplingParams:

132 assert sampling_params.structured_outputs is not None

133 assert sampling_params.structured_outputs.grammar == "root ::= 'hello'"

134

135 + def test_text_format_json_object_enables_structured_outputs(self):

136 + """text.format json_object enables structured outputs for sampling."""

137 + request = ResponsesRequest(

138 + model="test-model",

139 + input="test input",

140 + text=ResponseTextConfig.model_validate({"format": {"type": "json_object"}}),

141 + )

142 +

143 + sampling_params = request.to_sampling_params(default_max_tokens=1000)

144 +

145 + assert sampling_params.structured_outputs is not None

146 + assert sampling_params.structured_outputs.json_object is True

147 + assert sampling_params.structured_outputs.json is None

148 + assert request.structured_outputs is None

149 +

150 def test_structured_outputs_and_json_schema_conflict(self):

151 """Test that specifying both structured_outputs and json_schema raises."""

152 structured_outputs = StructuredOutputsParams(grammar="root ::= 'hello'")

vllm/entrypoints/openai/chat_completion/protocol.py

+9 -37

@@ -3,7 +3,6 @@

4 # Adapted from

5 # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py

6 -import json

6 import time

7 from typing import Annotated, Any, ClassVar, Literal

@@ -14,7 +13,6 @@ from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnota

13 from pydantic import Field, PrivateAttr, model_serializer, model_validator

15 from vllm.config import ModelConfig

17 -from vllm.config.utils import replace

16 from vllm.entrypoints.chat_utils import (

17 ChatCompletionMessageParam,

18 ChatTemplateContentFormatOption,

@@ -24,13 +22,12 @@ from vllm.entrypoints.openai.engine.protocol import (

22 DeltaMessage,

23 FunctionCall,

24 FunctionDefinition,

27 - LegacyStructuralTagResponseFormat,

25 OpenAIBaseModel,

26 PerRequestTimingMetrics,

27 StreamOptions,

31 - StructuralTagResponseFormat,

28 ToolCall,

29 UsageInfo,

30 + structured_outputs_from_response_format,

31 validate_structural_tag_response_format,

32 validate_structured_outputs_structural_tag,

33 )

@@ -607,6 +604,13 @@ class ChatCompletionRequest(OpenAIBaseModel):

604 include_stop_str_in_output=self.include_stop_str_in_output,

605 )

606

607 + def extract_structured_outputs(self) -> StructuredOutputsParams | None:

608 + """Normalize request constraints into ``StructuredOutputsParams``."""

609 + return structured_outputs_from_response_format(

610 + self.structured_outputs,

611 + self.response_format,

612 + )

613 +

614 def to_sampling_params(

615 self,

616 max_tokens: int,

@@ -651,38 +655,6 @@ class ChatCompletionRequest(OpenAIBaseModel):

655 if prompt_logprobs is None and self.echo:

656 prompt_logprobs = self.top_logprobs

657

654 - response_format = self.response_format

655 - if response_format is not None:

656 - structured_outputs_kwargs = dict[str, Any]()

657 -

658 - # Set structured output params for response format

659 - if response_format.type == "json_object":

660 - structured_outputs_kwargs["json_object"] = True

661 - elif response_format.type == "json_schema":

662 - json_schema = response_format.json_schema

663 - assert json_schema is not None

664 - structured_outputs_kwargs["json"] = json_schema.json_schema

665 - elif response_format.type == "structural_tag":

666 - structural_tag = response_format

667 - assert structural_tag is not None and isinstance(

668 - structural_tag,

669 - (

670 - LegacyStructuralTagResponseFormat,

671 - StructuralTagResponseFormat,

672 - ),

673 - )

674 - s_tag_obj = structural_tag.model_dump(by_alias=True)

675 - structured_outputs_kwargs["structural_tag"] = json.dumps(s_tag_obj)

676 -

677 - # If structured outputs wasn't already enabled,

678 - # we must enable it for these features to work

679 - if len(structured_outputs_kwargs) > 0:

680 - self.structured_outputs = (

681 - StructuredOutputsParams(**structured_outputs_kwargs)

682 - if self.structured_outputs is None

683 - else replace(self.structured_outputs, **structured_outputs_kwargs)

684 - )

685 -

658 extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}

659 if self.kv_transfer_params:

660 # Pass in kv_transfer_params via extra_args

@@ -718,7 +690,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

690 output_kind=RequestOutputKind.DELTA

691 if self.stream

692 else RequestOutputKind.FINAL_ONLY,

721 - structured_outputs=self.structured_outputs,

693 + structured_outputs=self.extract_structured_outputs(),

694 logit_bias=self.logit_bias,

695 bad_words=self.bad_words,

696 thinking_token_budget=self.thinking_token_budget,

vllm/entrypoints/openai/completion/protocol.py

+9 -37

@@ -3,7 +3,6 @@

4 # Adapted from

5 # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py

6 -import json

6 import time

7 from typing import Annotated, Any, Literal

@@ -11,15 +10,13 @@ from pydantic import Field, model_validator

11 import vllm.envs as envs

12 from vllm.config import ModelConfig

14 -from vllm.config.utils import replace

13 from vllm.entrypoints.openai.engine.protocol import (

14 AnyResponseFormat,

17 - LegacyStructuralTagResponseFormat,

15 OpenAIBaseModel,

16 PerRequestTimingMetrics,

17 StreamOptions,

21 - StructuralTagResponseFormat,

18 UsageInfo,

19 + structured_outputs_from_response_format,

20 validate_structural_tag_response_format,

21 validate_structured_outputs_structural_tag,

22 )

@@ -281,6 +278,13 @@ class CompletionRequest(OpenAIBaseModel):

278 include_stop_str_in_output=self.include_stop_str_in_output,

279 )

280

281 + def extract_structured_outputs(self) -> StructuredOutputsParams | None:

282 + """Normalize request constraints into ``StructuredOutputsParams``."""

283 + return structured_outputs_from_response_format(

284 + self.structured_outputs,

285 + self.response_format,

286 + )

287 +

288 def to_sampling_params(

289 self,

290 max_tokens: int,

@@ -330,38 +334,6 @@ class CompletionRequest(OpenAIBaseModel):

334

335 echo_without_generation = self.echo and self.max_tokens == 0

336

333 - response_format = self.response_format

334 - if response_format is not None:

335 - structured_outputs_kwargs = dict[str, Any]()

336 -

337 - # Set structured output params for response format

338 - if response_format.type == "json_object":

339 - structured_outputs_kwargs["json_object"] = True

340 - elif response_format.type == "json_schema":

341 - json_schema = response_format.json_schema

342 - assert json_schema is not None

343 - structured_outputs_kwargs["json"] = json_schema.json_schema

344 - elif response_format.type == "structural_tag":

345 - structural_tag = response_format

346 - assert isinstance(

347 - structural_tag,

348 - (

349 - LegacyStructuralTagResponseFormat,

350 - StructuralTagResponseFormat,

351 - ),

352 - )

353 - s_tag_obj = structural_tag.model_dump(by_alias=True)

354 - structured_outputs_kwargs["structural_tag"] = json.dumps(s_tag_obj)

355 -

356 - # If structured outputs wasn't already enabled,

357 - # we must enable it for these features to work

358 - if len(structured_outputs_kwargs) > 0:

359 - self.structured_outputs = (

360 - StructuredOutputsParams(**structured_outputs_kwargs)

361 - if self.structured_outputs is None

362 - else replace(self.structured_outputs, **structured_outputs_kwargs)

363 - )

364 -

337 extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}

338 if self.kv_transfer_params:

339 # Pass in kv_transfer_params via extra_args

@@ -393,7 +365,7 @@ class CompletionRequest(OpenAIBaseModel):

365 output_kind=RequestOutputKind.DELTA

366 if self.stream

367 else RequestOutputKind.FINAL_ONLY,

396 - structured_outputs=self.structured_outputs,

368 + structured_outputs=self.extract_structured_outputs(),

369 logit_bias=self.logit_bias,

370 allowed_token_ids=self.allowed_token_ids,

371 bad_words=self.bad_words,

vllm/entrypoints/openai/engine/protocol.py

+36 -2

@@ -3,6 +3,7 @@

4 # Adapted from

5 # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py

6 +import json

7 import time

8 from http import HTTPStatus

9 from typing import Any, ClassVar, Literal, TypeAlias

@@ -16,9 +17,11 @@ from pydantic import (

17 model_validator,

18 )

20 +from vllm.config.utils import replace

21 from vllm.entrypoints.chat_utils import make_tool_call_id

22 from vllm.exceptions import VLLMValidationError

23 from vllm.logger import init_logger

24 +from vllm.sampling_params import StructuredOutputsParams

25 from vllm.utils import random_uuid

26 from vllm.utils.import_utils import resolve_obj_by_qualname

@@ -173,6 +176,39 @@ AnyResponseFormat: TypeAlias = (

176 )

177

178

179 +def structured_outputs_from_response_format(

180 + structured_outputs: StructuredOutputsParams | None,

181 + response_format: AnyResponseFormat | None,

182 +) -> StructuredOutputsParams | None:

183 + """Apply ``response_format`` overrides to ``structured_outputs``."""

184 + if response_format is None or response_format.type == "text":

185 + return structured_outputs

186 +

187 + overrides: dict[str, Any]

188 + if response_format.type == "json_object":

189 + overrides = {"json_object": True}

190 + elif response_format.type == "json_schema":

191 + json_schema = response_format.json_schema

192 + assert json_schema is not None

193 + overrides = {"json": json_schema.json_schema}

194 + else:

195 + assert isinstance(

196 + response_format,

197 + (

198 + LegacyStructuralTagResponseFormat,

199 + StructuralTagResponseFormat,

200 + ),

201 + )

202 + overrides = {

203 + "structural_tag": json.dumps(response_format.model_dump(by_alias=True))

204 + }

205 +

206 + if structured_outputs is None:

207 + return StructuredOutputsParams(**overrides)

208 +

209 + return replace(structured_outputs, **overrides)

210 +

211 +

212 def validate_structural_tag_response_format(

213 response_format: AnyStructuralTagResponseFormat | dict[str, Any],

214 ) -> None:

@@ -181,8 +217,6 @@ def validate_structural_tag_response_format(

217 Engine-side validation reports malformed structural tags as generation

218 failures. OpenAI request parsing should classify them as bad requests.

219 """

184 - import json

185 -

220 from pydantic import TypeAdapter, ValidationError

221

222 if isinstance(response_format, dict):

vllm/entrypoints/openai/responses/protocol.py

+26 -22

@@ -354,6 +354,31 @@ class ResponsesRequest(OpenAIBaseModel):

354 "top_k": 0,

355 }

356

357 + def extract_structured_outputs(self) -> StructuredOutputsParams | None:

358 + """Normalize request constraints into ``StructuredOutputsParams``."""

359 + if self.text is None or self.text.format is None:

360 + return self.structured_outputs

361 +

362 + if self.structured_outputs is not None:

363 + raise VLLMValidationError(

364 + "Cannot specify both structured_outputs and text.format",

365 + parameter="structured_outputs",

366 + )

367 +

368 + response_format = self.text.format

369 + if response_format.type == "json_object":

370 + return StructuredOutputsParams(json_object=True)

371 + if (

372 + response_format.type == "json_schema"

373 + and response_format.schema_ is not None

374 + ):

375 + return StructuredOutputsParams(

376 + json=response_format.schema_ # type: ignore[call-arg]

377 + # --follow-imports skip hides the class definition but also hides

378 + # multiple third party conflicts, so best of both evils

379 + )

380 + return None

381 +

382 def to_sampling_params(

383 self,

384 default_max_tokens: int,

@@ -387,27 +412,6 @@ class ResponsesRequest(OpenAIBaseModel):

412 if (frequency_penalty := self.frequency_penalty) is None:

413 frequency_penalty = default_sampling_params.get("frequency_penalty", 0.0)

414

390 - # Structured output

391 - structured_outputs = self.structured_outputs

392 -

393 - # Also check text.format for OpenAI-style json_schema

394 - if self.text is not None and self.text.format is not None:

395 - if structured_outputs is not None:

396 - raise VLLMValidationError(

397 - "Cannot specify both structured_outputs and text.format",

398 - parameter="structured_outputs",

399 - )

400 - response_format = self.text.format

401 - if (

402 - response_format.type == "json_schema"

403 - and response_format.schema_ is not None

404 - ):

405 - structured_outputs = StructuredOutputsParams(

406 - json=response_format.schema_ # type: ignore[call-arg]

407 - # --follow-imports skip hides the class definition but also hides

408 - # multiple third party conflicts, so best of both evils

409 - )

410 -

415 stop = self.stop if self.stop else []

416 if isinstance(stop, str):

417 stop = [stop]

@@ -433,7 +437,7 @@ class ResponsesRequest(OpenAIBaseModel):

437 output_kind=(

438 RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY

439 ),

436 - structured_outputs=structured_outputs,

440 + structured_outputs=self.extract_structured_outputs(),

441 logit_bias=self.logit_bias,

442 extra_args=extra_args,

443 skip_clone=True, # Created fresh per request, safe to skip clone