A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
fix: response_format for completion (#23212)
Signed-off-by: Nan2018 <qinnanjoshua@gmail.com>
Nan Qin committed
Oct 14, 2025 at 16:23 UTC
9d6964926e78e3a35da49234b9b87153239d33c4
1 file changed
+21
-7
vllm/entrypoints/openai/protocol.py
+21
-7
@@ -1197,6 +1197,10 @@ class CompletionRequest(OpenAIBaseModel):
1197
"Please pass `grammar` to `structured_outputs` instead."
1198
),
1199
)
1200
+ structural_tag: str | None = Field(
1201
+ default=None,
1202
+ description=("If specified, the output will follow the structural tag schema."),
1203
+ )
1204
guided_decoding_backend: str | None = Field(
1205
default=None,
1206
description=(
@@ -1357,10 +1361,27 @@ class CompletionRequest(OpenAIBaseModel):
1361
1362
echo_without_generation = self.echo and self.max_tokens == 0
1363
1364
+ guided_json_object = None
1365
+ if self.response_format is not None:
1366
+ if self.response_format.type == "json_object":
1367
+ guided_json_object = True
1368
+ elif self.response_format.type == "json_schema":
1369
+ json_schema = self.response_format.json_schema
1370
+ assert json_schema is not None
1371
+ self.guided_json = json_schema.json_schema
1372
+ elif self.response_format.type == "structural_tag":
1373
+ structural_tag = self.response_format
1374
+ assert structural_tag is not None and isinstance(
1375
+ structural_tag, StructuralTagResponseFormat
1376
+ )
1377
+ s_tag_obj = structural_tag.model_dump(by_alias=True)
1378
+ self.structural_tag = json.dumps(s_tag_obj)
1379
+
1380
# Forward deprecated guided_* parameters to structured_outputs
1381
if self.structured_outputs is None:
1382
kwargs = dict[str, Any](
1383
json=self.guided_json,
1384
+ json_object=guided_json_object,
1385
regex=self.guided_regex,
1386
choice=self.guided_choice,
1387
grammar=self.guided_grammar,
@@ -1370,13 +1391,6 @@ class CompletionRequest(OpenAIBaseModel):
1391
if len(kwargs) > 0:
1392
self.structured_outputs = StructuredOutputsParams(**kwargs)
1393
1373
- if (
1374
- self.structured_outputs is not None
1375
- and self.response_format is not None
1376
- and self.response_format.type == "json_object"
1377
- ):
1378
- self.structured_outputs.json_object = True
1379
-
1394
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
1395
if self.kv_transfer_params:
1396
# Pass in kv_transfer_params via extra_args