A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 01:30 UTC
[bugfix] Fix online serving crash when text type response_format is received (#26822)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com> Signed-off-by: j0shuajun <59368606+j0shuajun@users.noreply.github.com> Co-authored-by: j0shuajun <59368606+j0shuajun@users.noreply.github.com>
cjackal committed
Jan 16, 2026 at 13:23 UTC
35bf5d08e8e4345805cebf08b29777f217ddf879
6 files changed
+68
-26
tests/entrypoints/openai/test_chat.py
+19
@@ -671,6 +671,25 @@ async def test_response_format_json_schema(client: openai.AsyncOpenAI):
671
assert loaded == {"result": 2}, loaded
672
673
674
+@pytest.mark.asyncio
675
+async def test_response_format_text(client: openai.AsyncOpenAI):
676
+ for _ in range(2):
677
+ resp = await client.chat.completions.create(
678
+ model=MODEL_NAME,
679
+ messages=[
680
+ {
681
+ "role": "user",
682
+ "content": "what is 1+1?",
683
+ }
684
+ ],
685
+ max_completion_tokens=10,
686
+ response_format={"type": "text"},
687
+ )
688
+
689
+ content = resp.choices[0].message.content
690
+ assert content is not None
691
+
692
+
693
@pytest.mark.asyncio
694
async def test_extra_fields_allowed(client: openai.AsyncOpenAI):
695
resp = await client.chat.completions.create(
vllm/entrypoints/openai/chat_completion/protocol.py
+14
-7
@@ -5,6 +5,7 @@
5
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
7
import time
8
+from dataclasses import replace
9
from typing import Annotated, Any, ClassVar, Literal
10
11
import torch
@@ -417,18 +418,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
418
419
response_format = self.response_format
420
if response_format is not None:
420
- # If structured outputs wasn't already enabled,
421
- # we must enable it for these features to work
422
- if self.structured_outputs is None:
423
- self.structured_outputs = StructuredOutputsParams()
421
+ structured_outputs_kwargs = dict[str, Any]()
422
423
# Set structured output params for response format
424
if response_format.type == "json_object":
427
- self.structured_outputs.json_object = True
425
+ structured_outputs_kwargs["json_object"] = True
426
elif response_format.type == "json_schema":
427
json_schema = response_format.json_schema
428
assert json_schema is not None
431
- self.structured_outputs.json = json_schema.json_schema
429
+ structured_outputs_kwargs["json"] = json_schema.json_schema
430
elif response_format.type == "structural_tag":
431
structural_tag = response_format
432
assert structural_tag is not None and isinstance(
@@ -439,7 +437,16 @@ class ChatCompletionRequest(OpenAIBaseModel):
437
),
438
)
439
s_tag_obj = structural_tag.model_dump(by_alias=True)
442
- self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
440
+ structured_outputs_kwargs["structural_tag"] = json.dumps(s_tag_obj)
441
+
442
+ # If structured outputs wasn't already enabled,
443
+ # we must enable it for these features to work
444
+ if len(structured_outputs_kwargs) > 0:
445
+ self.structured_outputs = (
446
+ StructuredOutputsParams(**structured_outputs_kwargs)
447
+ if self.structured_outputs is None
448
+ else replace(self.structured_outputs, **structured_outputs_kwargs)
449
+ )
450
451
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
452
if self.kv_transfer_params:
vllm/entrypoints/openai/completion/protocol.py
+14
-7
@@ -5,6 +5,7 @@
5
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
7
import time
8
+from dataclasses import replace
9
from typing import Annotated, Any, Literal
10
11
import torch
@@ -247,18 +248,15 @@ class CompletionRequest(OpenAIBaseModel):
248
249
response_format = self.response_format
250
if response_format is not None:
250
- # If structured outputs wasn't already enabled,
251
- # we must enable it for these features to work
252
- if self.structured_outputs is None:
253
- self.structured_outputs = StructuredOutputsParams()
251
+ structured_outputs_kwargs = dict[str, Any]()
252
253
# Set structured output params for response format
254
if response_format.type == "json_object":
257
- self.structured_outputs.json_object = True
255
+ structured_outputs_kwargs["json_object"] = True
256
elif response_format.type == "json_schema":
257
json_schema = response_format.json_schema
258
assert json_schema is not None
261
- self.structured_outputs.json = json_schema.json_schema
259
+ structured_outputs_kwargs["json"] = json_schema.json_schema
260
elif response_format.type == "structural_tag":
261
structural_tag = response_format
262
assert structural_tag is not None and isinstance(
@@ -269,7 +267,16 @@ class CompletionRequest(OpenAIBaseModel):
267
),
268
)
269
s_tag_obj = structural_tag.model_dump(by_alias=True)
272
- self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
270
+ structured_outputs_kwargs["structural_tag"] = json.dumps(s_tag_obj)
271
+
272
+ # If structured outputs wasn't already enabled,
273
+ # we must enable it for these features to work
274
+ if len(structured_outputs_kwargs) > 0:
275
+ self.structured_outputs = (
276
+ StructuredOutputsParams(**structured_outputs_kwargs)
277
+ if self.structured_outputs is None
278
+ else replace(self.structured_outputs, **structured_outputs_kwargs)
279
+ )
280
281
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
282
if self.kv_transfer_params:
vllm/entrypoints/openai/responses/serving.py
+13
-10
@@ -9,7 +9,7 @@ from collections import deque
9
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
10
from contextlib import AsyncExitStack
11
from copy import copy
12
-from dataclasses import dataclass
12
+from dataclasses import dataclass, replace
13
from http import HTTPStatus
14
from typing import Final
15
@@ -467,15 +467,18 @@ class OpenAIServingResponses(OpenAIServing):
467
468
if self.reasoning_parser is not None:
469
reasoning_parser = self.reasoning_parser(tokenizer)
470
- if sampling_params.structured_outputs is None:
471
- sampling_params.structured_outputs = StructuredOutputsParams()
472
- struct_out = sampling_params.structured_outputs
473
- if struct_out.all_non_structural_tag_constraints_none():
474
- sampling_params.structured_outputs.structural_tag = (
475
- reasoning_parser.prepare_structured_tag(
476
- sampling_params.structured_outputs.structural_tag,
477
- self.tool_server,
478
- )
470
+ if (
471
+ isinstance(
472
+ struct_out := sampling_params.structured_outputs,
473
+ StructuredOutputsParams,
474
+ )
475
+ and struct_out.all_non_structural_tag_constraints_none()
476
+ ):
477
+ sampling_params.structured_outputs = replace(
478
+ struct_out,
479
+ structural_tag=reasoning_parser.prepare_structured_tag(
480
+ struct_out.structural_tag, self.tool_server
481
+ ),
482
)
483
generator = self._generate_with_builtin_tools(
484
request_id=request.request_id,
vllm/sampling_params.py
+5
@@ -67,6 +67,11 @@ class StructuredOutputsParams:
67
"You can only use one kind of structured outputs constraint "
68
f"but multiple are specified: {self.__dict__}"
69
)
70
+ if count < 1:
71
+ raise ValueError(
72
+ "You must use one kind of structured outputs constraint "
73
+ f"but none are specified: {self.__dict__}"
74
+ )
75
76
def all_constraints_none(self) -> bool:
77
"""
vllm/tool_parsers/abstract_tool_parser.py
+3
-2
@@ -65,10 +65,11 @@ class ToolParser:
65
# Set structured output params for tool calling
66
if json_schema_from_tool is not None:
67
if isinstance(request, ChatCompletionRequest):
68
- request.structured_outputs = StructuredOutputsParams()
68
# tool_choice: "Forced Function" or "required" will override
69
# structured output json settings to make tool calling work correctly
71
- request.structured_outputs.json = json_schema_from_tool
70
+ request.structured_outputs = StructuredOutputsParams(
71
+ json=json_schema_from_tool
72
+ )
73
request.response_format = None
74
if isinstance(request, ResponsesRequest):
75
request.text = ResponseTextConfig()