A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Chore] Cleanup guided namespace, move to structured outputs config (#22772)
Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Aaron Pham committed
Sep 18, 2025 at 05:20 UTC
29283e89762a3d572c504e5ea317351696b553a6
51 files changed
+580
-807
.buildkite/scripts/hardware_ci/run-amd-test.sh
-6
@@ -167,12 +167,6 @@ if [[ $commands == *" entrypoints/llm "* ]]; then
167
--ignore=entrypoints/llm/test_prompt_validation.py "}
168
fi
169
170
-#Obsolete currently
171
-##ignore certain Entrypoints/llm tests
172
-#if [[ $commands == *" && pytest -v -s entrypoints/llm/test_guided_generate.py"* ]]; then
173
-# commands=${commands//" && pytest -v -s entrypoints/llm/test_guided_generate.py"/" "}
174
-#fi
175
-
170
# --ignore=entrypoints/openai/test_encoder_decoder.py \
171
# --ignore=entrypoints/openai/test_embedding.py \
172
# --ignore=entrypoints/openai/test_oot_registration.py
.buildkite/test-pipeline.yaml
+1
-2
@@ -108,8 +108,7 @@ steps:
108
- tests/entrypoints/offline_mode
109
commands:
110
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
111
- - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
112
- - pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
111
+ - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
112
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
113
- VLLM_USE_V1=0 pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
114
.github/mergify.yml
+1
-1
@@ -171,7 +171,7 @@ pull_request_rules:
171
- files=examples/online_serving/openai_chat_completion_structured_outputs.py
172
- files=examples/online_serving/openai_chat_completion_structured_outputs_with_reasoning.py
173
- files~=^tests/v1/structured_output/
174
- - files=tests/v1/entrypoints/llm/test_guided_generate.py
174
+ - files=tests/v1/entrypoints/llm/test_struct_output_generate.py
175
- files~=^vllm/v1/structured_output/
176
actions:
177
label:
benchmarks/benchmark_serving_structured_output.py
+8
-8
@@ -696,11 +696,11 @@ def evaluate(ret, args):
696
return re.match(args.regex, actual) is not None
697
698
def _eval_correctness(expected, actual):
699
- if args.structure_type == "guided_json":
699
+ if args.structure_type == "json":
700
return _eval_correctness_json(expected, actual)
701
- elif args.structure_type == "guided_regex":
701
+ elif args.structure_type == "regex":
702
return _eval_correctness_regex(expected, actual)
703
- elif args.structure_type == "guided_choice":
703
+ elif args.structure_type == "choice":
704
return _eval_correctness_choice(expected, actual)
705
else:
706
return None
@@ -780,18 +780,18 @@ def main(args: argparse.Namespace):
780
)
781
782
if args.dataset == "grammar":
783
- args.structure_type = "guided_grammar"
783
+ args.structure_type = "grammar"
784
elif args.dataset == "regex":
785
- args.structure_type = "guided_regex"
785
+ args.structure_type = "regex"
786
elif args.dataset == "choice":
787
- args.structure_type = "guided_choice"
787
+ args.structure_type = "choice"
788
else:
789
- args.structure_type = "guided_json"
789
+ args.structure_type = "json"
790
791
if args.no_structured_output:
792
args.structured_output_ratio = 0
793
if args.save_results:
794
- result_file_name = f"{args.structured_output_ratio}guided"
794
+ result_file_name = f"{args.structured_output_ratio}so"
795
result_file_name += f"_{backend}"
796
result_file_name += f"_{args.request_rate}qps"
797
result_file_name += f"_{args.model.split('/')[-1]}"
docs/api/README.md
+1
-1
@@ -14,7 +14,7 @@ API documentation for vLLM's configuration classes.
14
- [vllm.config.LoRAConfig][]
15
- [vllm.config.MultiModalConfig][]
16
- [vllm.config.PoolerConfig][]
17
-- [vllm.config.DecodingConfig][]
17
+- [vllm.config.StructuredOutputsConfig][]
18
- [vllm.config.ObservabilityConfig][]
19
- [vllm.config.KVTransferConfig][]
20
- [vllm.config.CompilationConfig][]
docs/features/reasoning_outputs.md
+5
-5
@@ -10,12 +10,12 @@ vLLM currently supports the following reasoning models:
10
11
| Model Series | Parser Name | Structured Output Support | Tool Calling |
12
|--------------|-------------|------------------|-------------|
13
-| [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `deepseek_r1` | `guided_json`, `guided_regex` | ❌ |
14
-| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `guided_json`, `guided_regex` | ✅ |
13
+| [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `deepseek_r1` | `json`, `regex` | ❌ |
14
+| [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ |
15
| [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ |
16
-| [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `guided_json`, `guided_regex` | ✅ |
17
-| [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `guided_json`, `guided_regex` | ✅ |
18
-| [GLM-4.5 series](https://huggingface.co/collections/zai-org/glm-45-687c621d34bda8c9e4bf503b) | `glm45` | `guided_json`, `guided_regex` | ✅ |
16
+| [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ |
17
+| [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `json`, `regex` | ✅ |
18
+| [GLM-4.5 series](https://huggingface.co/collections/zai-org/glm-45-687c621d34bda8c9e4bf503b) | `glm45` | `json`, `regex` | ✅ |
19
20
!!! note
21
IBM Granite 3.2 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.
docs/features/structured_outputs.md
+18
-18
@@ -12,23 +12,23 @@ You can generate structured outputs using the OpenAI's [Completions](https://pla
12
13
The following parameters are supported, which must be added as extra parameters:
14
15
-- `guided_choice`: the output will be exactly one of the choices.
16
-- `guided_regex`: the output will follow the regex pattern.
17
-- `guided_json`: the output will follow the JSON schema.
18
-- `guided_grammar`: the output will follow the context free grammar.
15
+- `choice`: the output will be exactly one of the choices.
16
+- `regex`: the output will follow the regex pattern.
17
+- `json`: the output will follow the JSON schema.
18
+- `grammar`: the output will follow the context free grammar.
19
- `structural_tag`: Follow a JSON schema within a set of specified tags within the generated text.
20
21
You can see the complete list of supported parameters on the [OpenAI-Compatible Server](../serving/openai_compatible_server.md) page.
22
23
Structured outputs are supported by default in the OpenAI-Compatible Server. You
24
may choose to specify the backend to use by setting the
25
-`--guided-decoding-backend` flag to `vllm serve`. The default backend is `auto`,
25
+`--structured-outputs-config.backend` flag to `vllm serve`. The default backend is `auto`,
26
which will try to choose an appropriate backend based on the details of the
27
request. You may also choose a specific backend, along with
28
some options. A full set of options is available in the `vllm serve --help`
29
text.
30
31
-Now let´s see an example for each of the cases, starting with the `guided_choice`, as it´s the easiest one:
31
+Now let´s see an example for each of the cases, starting with the `choice`, as it´s the easiest one:
32
33
??? code
34
@@ -45,12 +45,12 @@ Now let´s see an example for each of the cases, starting with the `guided_choic
45
messages=[
46
{"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
47
],
48
- extra_body={"guided_choice": ["positive", "negative"]},
48
+ extra_body={"structured_outputs": {"choice": ["positive", "negative"]}},
49
)
50
print(completion.choices[0].message.content)
51
```
52
53
-The next example shows how to use the `guided_regex`. The idea is to generate an email address, given a simple regex template:
53
+The next example shows how to use the `regex`. The idea is to generate an email address, given a simple regex template:
54
55
??? code
56
@@ -63,18 +63,18 @@ The next example shows how to use the `guided_regex`. The idea is to generate an
63
"content": "Generate an example email address for Alan Turing, who works in Enigma. End in .com and new line. Example result: alan.turing@enigma.com\n",
64
}
65
],
66
- extra_body={"guided_regex": r"\w+@\w+\.com\n", "stop": ["\n"]},
66
+ extra_body={"structured_outputs": {"regex": r"\w+@\w+\.com\n"}, "stop": ["\n"]},
67
)
68
print(completion.choices[0].message.content)
69
```
70
71
One of the most relevant features in structured text generation is the option to generate a valid JSON with pre-defined fields and formats.
72
-For this we can use the `guided_json` parameter in two different ways:
72
+For this we can use the `json` parameter in two different ways:
73
74
- Using directly a [JSON Schema](https://json-schema.org/)
75
- Defining a [Pydantic model](https://docs.pydantic.dev/latest/) and then extracting the JSON Schema from it (which is normally an easier option).
76
77
-The next example shows how to use the `guided_json` parameter with a Pydantic model:
77
+The next example shows how to use the `response_format` parameter with a Pydantic model:
78
79
??? code
80
@@ -119,7 +119,7 @@ The next example shows how to use the `guided_json` parameter with a Pydantic mo
119
JSON schema and how the fields should be populated. This can improve the
120
results notably in most cases.
121
122
-Finally we have the `guided_grammar` option, which is probably the most
122
+Finally we have the `grammar` option, which is probably the most
123
difficult to use, but it´s really powerful. It allows us to define complete
124
languages like SQL queries. It works by using a context free EBNF grammar.
125
As an example, we can use to define a specific format of simplified SQL queries:
@@ -149,7 +149,7 @@ As an example, we can use to define a specific format of simplified SQL queries:
149
"content": "Generate an SQL query to show the 'username' and 'email' from the 'users' table.",
150
}
151
],
152
- extra_body={"guided_grammar": simplified_sql_grammar},
152
+ extra_body={"structured_outputs": {"grammar": simplified_sql_grammar}},
153
)
154
print(completion.choices[0].message.content)
155
```
@@ -292,8 +292,8 @@ An example of using `structural_tag` can be found here: <gh-file:examples/online
292
## Offline Inference
293
294
Offline inference allows for the same types of structured outputs.
295
-To use it, we´ll need to configure the guided decoding using the class `GuidedDecodingParams` inside `SamplingParams`.
296
-The main available options inside `GuidedDecodingParams` are:
295
+To use it, we´ll need to configure the structured outputs using the class `StructuredOutputsParams` inside `SamplingParams`.
296
+The main available options inside `StructuredOutputsParams` are:
297
298
- `json`
299
- `regex`
@@ -309,12 +309,12 @@ shown below:
309
310
```python
311
from vllm import LLM, SamplingParams
312
- from vllm.sampling_params import GuidedDecodingParams
312
+ from vllm.sampling_params import StructuredOutputsParams
313
314
llm = LLM(model="HuggingFaceTB/SmolLM2-1.7B-Instruct")
315
316
- guided_decoding_params = GuidedDecodingParams(choice=["Positive", "Negative"])
317
- sampling_params = SamplingParams(guided_decoding=guided_decoding_params)
316
+ structured_outputs_params = StructuredOutputsParams(choice=["Positive", "Negative"])
317
+ sampling_params = SamplingParams(structured_outputs=structured_outputs_params)
318
outputs = llm.generate(
319
prompts="Classify this sentiment: vLLM is wonderful!",
320
sampling_params=sampling_params,
docs/features/tool_calling.md
+5
-6
@@ -71,7 +71,7 @@ This example demonstrates:
71
* Making a request with `tool_choice="auto"`
72
* Handling the structured response and executing the corresponding function
73
74
-You can also specify a particular function using named function calling by setting `tool_choice={"type": "function", "function": {"name": "get_weather"}}`. Note that this will use the guided decoding backend - so the first time this is used, there will be several seconds of latency (or more) as the FSM is compiled for the first time before it is cached for subsequent requests.
74
+You can also specify a particular function using named function calling by setting `tool_choice={"type": "function", "function": {"name": "get_weather"}}`. Note that this will use the structured outputs backend - so the first time this is used, there will be several seconds of latency (or more) as the FSM is compiled for the first time before it is cached for subsequent requests.
75
76
Remember that it's the caller's responsibility to:
77
@@ -83,19 +83,18 @@ For more advanced usage, including parallel tool calls and different model-speci
83
84
## Named Function Calling
85
86
-vLLM supports named function calling in the chat completion API by default. It does so using Outlines through guided decoding, so this is
87
-enabled by default and will work with any supported model. You are guaranteed a validly-parsable function call - not a
86
+vLLM supports named function calling in the chat completion API by default. This should work with most structured outputs backends supported by vLLM. You are guaranteed a validly-parsable function call - not a
87
high-quality one.
88
90
-vLLM will use guided decoding to ensure the response matches the tool parameter object defined by the JSON schema in the `tools` parameter.
91
-For best results, we recommend ensuring that the expected output format / schema is specified in the prompt to ensure that the model's intended generation is aligned with the schema that it's being forced to generate by the guided decoding backend.
89
+vLLM will use structured outputs to ensure the response matches the tool parameter object defined by the JSON schema in the `tools` parameter.
90
+For best results, we recommend ensuring that the expected output format / schema is specified in the prompt to ensure that the model's intended generation is aligned with the schema that it's being forced to generate by the structured outputs backend.
91
92
To use a named function, you need to define the functions in the `tools` parameter of the chat completion request, and
93
specify the `name` of one of the tools in the `tool_choice` parameter of the chat completion request.
94
95
## Required Function Calling
96
98
-vLLM supports the `tool_choice='required'` option in the chat completion API. Similar to the named function calling, it also uses guided decoding, so this is enabled by default and will work with any supported model. The guided decoding features for `tool_choice='required'` (such as JSON schema with `anyOf`) are currently only supported in the V0 engine with the guided decoding backend `outlines`. However, support for alternative decoding backends are on the [roadmap](../usage/v1_guide.md#features) for the V1 engine.
97
+vLLM supports the `tool_choice='required'` option in the chat completion API. Similar to the named function calling, it also uses structured outputs, so this is enabled by default and will work with any supported model. However, support for alternative decoding backends are on the [roadmap](../usage/v1_guide.md#features) for the V1 engine.
98
99
When tool_choice='required' is set, the model is guaranteed to generate one or more tool calls based on the specified tool list in the `tools` parameter. The number of tool calls depends on the user's query. The output format strictly follows the schema defined in the `tools` parameter.
100
docs/serving/openai_compatible_server.md
+2
-2
@@ -133,7 +133,7 @@ completion = client.chat.completions.create(
133
{"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
134
],
135
extra_body={
136
- "guided_choice": ["positive", "negative"]
136
+ "structured_outputs": {"choice": ["positive", "negative"]}
137
}
138
)
139
```
@@ -374,7 +374,7 @@ The following extra parameters are supported:
374
```python
375
--8<-- "vllm/entrypoints/openai/protocol.py:transcription-extra-params"
376
```
377
-
377
+
378
[](){ #translations-api }
379
380
### Translations API
examples/offline_inference/structured_outputs.py
+29
-25
@@ -1,11 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""
4
-This file demonstrates the example usage of guided decoding
5
-to generate structured outputs using vLLM. It shows how to apply
6
-different guided decoding techniques such as Choice, Regex, JSON schema,
7
-and Grammar to produce structured and formatted results
8
-based on specific prompts.
4
+This file demonstrates the example usage of structured outputs
5
+in vLLM. It shows how to apply different constraints such as choice,
6
+regex, json schema, and grammar to produce structured and formatted
7
+results based on specific prompts.
8
"""
9
10
from enum import Enum
@@ -13,19 +12,23 @@ from enum import Enum
12
from pydantic import BaseModel
13
14
from vllm import LLM, SamplingParams
16
-from vllm.sampling_params import GuidedDecodingParams
15
+from vllm.sampling_params import StructuredOutputsParams
16
17
MAX_TOKENS = 50
18
20
-# Guided decoding by Choice (list of possible options)
21
-guided_decoding_params_choice = GuidedDecodingParams(choice=["Positive", "Negative"])
22
-sampling_params_choice = SamplingParams(guided_decoding=guided_decoding_params_choice)
19
+# Structured outputs by Choice (list of possible options)
20
+structured_outputs_params_choice = StructuredOutputsParams(
21
+ choice=["Positive", "Negative"]
22
+)
23
+sampling_params_choice = SamplingParams(
24
+ structured_outputs=structured_outputs_params_choice
25
+)
26
prompt_choice = "Classify this sentiment: vLLM is wonderful!"
27
25
-# Guided decoding by Regex
26
-guided_decoding_params_regex = GuidedDecodingParams(regex=r"\w+@\w+\.com\n")
28
+# Structured outputs by Regex
29
+structured_outputs_params_regex = StructuredOutputsParams(regex=r"\w+@\w+\.com\n")
30
sampling_params_regex = SamplingParams(
28
- guided_decoding=guided_decoding_params_regex,
31
+ structured_outputs=structured_outputs_params_regex,
32
stop=["\n"],
33
max_tokens=MAX_TOKENS,
34
)
@@ -36,7 +39,7 @@ prompt_regex = (
39
)
40
41
39
-# Guided decoding by JSON using Pydantic schema
42
+# Structured outputs by JSON using Pydantic schema
43
class CarType(str, Enum):
44
sedan = "sedan"
45
suv = "SUV"
@@ -51,17 +54,16 @@ class CarDescription(BaseModel):
54
55
56
json_schema = CarDescription.model_json_schema()
54
-guided_decoding_params_json = GuidedDecodingParams(json=json_schema)
57
+structured_outputs_params_json = StructuredOutputsParams(json=json_schema)
58
sampling_params_json = SamplingParams(
56
- guided_decoding=guided_decoding_params_json,
57
- max_tokens=MAX_TOKENS,
59
+ structured_outputs=structured_outputs_params_json, max_tokens=MAX_TOKENS
60
)
61
prompt_json = (
60
- "Generate a JSON with the brand, model and car_type of"
62
+ "Generate a JSON with the brand, model and car_type of "
63
"the most iconic car from the 90's"
64
)
65
64
-# Guided decoding by Grammar
66
+# Structured outputs by Grammar
67
simplified_sql_grammar = """
68
root ::= select_statement
69
select_statement ::= "SELECT " column " from " table " where " condition
@@ -70,13 +72,15 @@ table ::= "table_1 " | "table_2 "
72
condition ::= column "= " number
73
number ::= "1 " | "2 "
74
"""
73
-guided_decoding_params_grammar = GuidedDecodingParams(grammar=simplified_sql_grammar)
75
+structured_outputs_params_grammar = StructuredOutputsParams(
76
+ grammar=simplified_sql_grammar
77
+)
78
sampling_params_grammar = SamplingParams(
75
- guided_decoding=guided_decoding_params_grammar,
79
+ structured_outputs=structured_outputs_params_grammar,
80
max_tokens=MAX_TOKENS,
81
)
82
prompt_grammar = (
79
- "Generate an SQL query to show the 'username' and 'email'from the 'users' table."
83
+ "Generate an SQL query to show the 'username' and 'email' from the 'users' table."
84
)
85
86
@@ -93,16 +97,16 @@ def main():
97
llm = LLM(model="Qwen/Qwen2.5-3B-Instruct", max_model_len=100)
98
99
choice_output = generate_output(prompt_choice, sampling_params_choice, llm)
96
- format_output("Guided decoding by Choice", choice_output)
100
+ format_output("Structured outputs by Choice", choice_output)
101
102
regex_output = generate_output(prompt_regex, sampling_params_regex, llm)
99
- format_output("Guided decoding by Regex", regex_output)
103
+ format_output("Structured outputs by Regex", regex_output)
104
105
json_output = generate_output(prompt_json, sampling_params_json, llm)
102
- format_output("Guided decoding by JSON", json_output)
106
+ format_output("Structured outputs by JSON", json_output)
107
108
grammar_output = generate_output(prompt_grammar, sampling_params_grammar, llm)
105
- format_output("Guided decoding by Grammar", grammar_output)
109
+ format_output("Structured outputs by Grammar", grammar_output)
110
111
112
if __name__ == "__main__":
examples/online_serving/openai_chat_completion_client_with_tools_required.py
+1
-1
@@ -6,7 +6,7 @@ without any specific flags:
6
7
```bash
8
VLLM_USE_V1=0 vllm serve unsloth/Llama-3.2-1B-Instruct \
9
- --guided-decoding-backend outlines
9
+ --structured-outputs-config.backend outlines
10
```
11
12
This example demonstrates how to generate chat completions
examples/online_serving/structured_outputs/structured_outputs.py
+5
-3
@@ -86,7 +86,7 @@ PARAMS: dict[ConstraintsFormat, dict[str, Any]] = {
86
"content": "Classify this sentiment: vLLM is wonderful!",
87
}
88
],
89
- "extra_body": {"guided_choice": ["positive", "negative"]},
89
+ "extra_body": {"structured_outputs": {"choice": ["positive", "negative"]}},
90
},
91
"regex": {
92
"messages": [
@@ -96,7 +96,7 @@ PARAMS: dict[ConstraintsFormat, dict[str, Any]] = {
96
}
97
],
98
"extra_body": {
99
- "guided_regex": r"[a-z0-9.]{1,20}@\w{6,10}\.com\n",
99
+ "structured_outputs": {"regex": r"[a-z0-9.]{1,20}@\w{6,10}\.com\n"},
100
},
101
},
102
"json": {
@@ -122,7 +122,8 @@ PARAMS: dict[ConstraintsFormat, dict[str, Any]] = {
122
}
123
],
124
"extra_body": {
125
- "guided_grammar": """
125
+ "structured_outputs": {
126
+ "grammar": """
127
root ::= select_statement
128
129
select_statement ::= "SELECT " column " from " table " where " condition
@@ -135,6 +136,7 @@ condition ::= column "= " number
136
137
number ::= "1 " | "2 "
138
""",
139
+ }
140
},
141
},
142
"structural_tag": {
tests/entrypoints/conftest.py
+1
-1
@@ -184,7 +184,7 @@ def sample_enum_json_schema():
184
185
186
@pytest.fixture
187
-def sample_guided_choice():
187
+def sample_structured_outputs_choices():
188
return [
189
"Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript",
190
"Ruby", "Swift", "Kotlin"
tests/entrypoints/llm/test_lazy_outlines.py
deleted
-82
@@ -1,82 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import sys
5
-from contextlib import nullcontext
6
-
7
-from vllm_test_utils import BlameResult, blame
8
-
9
-from vllm import LLM, SamplingParams
10
-from vllm.distributed import cleanup_dist_env_and_memory
11
-from vllm.sampling_params import GuidedDecodingParams
12
-
13
-
14
-def run_normal():
15
- prompts = [
16
- "Hello, my name is",
17
- "The president of the United States is",
18
- "The capital of France is",
19
- "The future of AI is",
20
- ]
21
- sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
22
-
23
- # Create an LLM without guided decoding as a baseline.
24
- llm = LLM(model="distilbert/distilgpt2",
25
- enforce_eager=True,
26
- gpu_memory_utilization=0.3)
27
- outputs = llm.generate(prompts, sampling_params)
28
- for output in outputs:
29
- prompt = output.prompt
30
- generated_text = output.outputs[0].text
31
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
32
-
33
- # Destroy the LLM object and free up the GPU memory.
34
- del llm
35
- cleanup_dist_env_and_memory()
36
-
37
-
38
-def run_xgrammar(sample_regex):
39
- # Create an LLM with guided decoding enabled.
40
- llm = LLM(model="distilbert/distilgpt2",
41
- enforce_eager=True,
42
- guided_decoding_backend="xgrammar",
43
- gpu_memory_utilization=0.3)
44
- prompt = f"Give an example IPv4 address with this regex: {sample_regex}"
45
- guided_decoding = GuidedDecodingParams(regex=sample_regex)
46
- sampling_params = SamplingParams(temperature=0.8,
47
- top_p=0.95,
48
- guided_decoding=guided_decoding)
49
- outputs = llm.generate(
50
- prompts=[prompt] * 2,
51
- sampling_params=sampling_params,
52
- use_tqdm=True,
53
- )
54
-
55
- for output in outputs:
56
- prompt = output.prompt
57
- generated_text = output.outputs[0].text
58
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
59
-
60
-
61
-def test_lazy_outlines(sample_regex):
62
- """If users don't use guided decoding, outlines should not be imported.
63
- """
64
- # make sure outlines is not imported
65
- module_name = "outlines"
66
- # In CI, we only check finally if the module is imported.
67
- # If it is indeed imported, we can rerun the test with `use_blame=True`,
68
- # which will trace every function call to find the first import location,
69
- # and help find the root cause.
70
- # We don't run it in CI by default because it is slow.
71
- use_blame = False
72
- context = blame(
73
- lambda: module_name in sys.modules) if use_blame else nullcontext()
74
- with context as result:
75
- run_normal()
76
- run_xgrammar(sample_regex)
77
- if use_blame:
78
- assert isinstance(result, BlameResult)
79
- print(f"the first import location is:\n{result.trace_stack}")
80
- assert module_name not in sys.modules, (
81
- f"Module {module_name} is imported. To see the first"
82
- f" import location, run the test with `use_blame=True`.")
tests/entrypoints/openai/test_chat.py
+59
-64
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-# imports for guided decoding tests
4
+# imports for structured outputs tests
5
import json
6
from typing import Optional
7
@@ -480,10 +480,11 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
480
481
482
@pytest.mark.asyncio
483
-async def test_guided_choice_chat(client: openai.AsyncOpenAI,
484
- sample_guided_choice, is_v1_server: bool):
483
+async def test_structured_outputs_choice_chat(
484
+ client: openai.AsyncOpenAI, sample_structured_outputs_choices,
485
+ is_v1_server: bool):
486
if not is_v1_server:
486
- pytest.skip("Guided decoding is only supported in v1 engine")
487
+ pytest.skip("Structured outputs is only supported in v1 engine")
488
messages = [{
489
"role": "system",
490
"content": "you are a helpful assistant"
@@ -498,9 +499,10 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
499
messages=messages,
500
max_completion_tokens=10,
501
temperature=0.7,
501
- extra_body=dict(guided_choice=sample_guided_choice))
502
+ extra_body=dict(
503
+ structured_outputs={"choice": sample_structured_outputs_choices}))
504
choice1 = chat_completion.choices[0].message.content
503
- assert choice1 in sample_guided_choice
505
+ assert choice1 in sample_structured_outputs_choices
506
507
messages.append({"role": "assistant", "content": choice1})
508
messages.append({
@@ -512,17 +514,19 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
514
messages=messages,
515
max_completion_tokens=10,
516
temperature=0.7,
515
- extra_body=dict(guided_choice=sample_guided_choice))
517
+ extra_body=dict(
518
+ structured_outputs={"choice": sample_structured_outputs_choices}))
519
choice2 = chat_completion.choices[0].message.content
517
- assert choice2 in sample_guided_choice
520
+ assert choice2 in sample_structured_outputs_choices
521
assert choice1 != choice2
522
523
524
@pytest.mark.asyncio
522
-async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
523
- is_v1_server: bool):
525
+async def test_structured_outputs_json_chat(client: openai.AsyncOpenAI,
526
+ sample_json_schema,
527
+ is_v1_server: bool):
528
if not is_v1_server:
525
- pytest.skip("Guided decoding is only supported in v1 engine")
529
+ pytest.skip("Structured outputs is only supported in v1 engine")
530
531
messages = [{
532
"role": "system",
@@ -538,7 +542,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
542
model=MODEL_NAME,
543
messages=messages,
544
max_completion_tokens=1000,
541
- extra_body=dict(guided_json=sample_json_schema))
545
+ extra_body=dict(structured_outputs={"json": sample_json_schema}))
546
message = chat_completion.choices[0].message
547
assert message.content is not None
548
json1 = json.loads(message.content)
@@ -555,7 +559,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
559
model=MODEL_NAME,
560
messages=messages,
561
max_completion_tokens=1000,
558
- extra_body=dict(guided_json=sample_json_schema))
562
+ extra_body=dict(structured_outputs={"json": sample_json_schema}))
563
message = chat_completion.choices[0].message
564
assert message.content is not None
565
json2 = json.loads(message.content)
@@ -565,10 +569,10 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI, sample_json_schema,
569
570
571
@pytest.mark.asyncio
568
-async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
569
- is_v1_server: bool):
572
+async def test_structured_outputs_regex_chat(client: openai.AsyncOpenAI,
573
+ sample_regex, is_v1_server: bool):
574
if not is_v1_server:
571
- pytest.skip("Guided decoding is only supported in v1 engine")
575
+ pytest.skip("Structured outputs is only supported in v1 engine")
576
577
messages = [{
578
"role": "system",
@@ -583,7 +587,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
587
model=MODEL_NAME,
588
messages=messages,
589
max_completion_tokens=20,
586
- extra_body=dict(guided_regex=sample_regex))
590
+ extra_body=dict(structured_outputs={"regex": sample_regex}))
591
ip1 = chat_completion.choices[0].message.content
592
assert ip1 is not None
593
assert re.fullmatch(sample_regex, ip1) is not None
@@ -594,7 +598,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
598
model=MODEL_NAME,
599
messages=messages,
600
max_completion_tokens=20,
597
- extra_body=dict(guided_regex=sample_regex))
601
+ extra_body=dict(structured_outputs={"regex": sample_regex}))
602
ip2 = chat_completion.choices[0].message.content
603
assert ip2 is not None
604
assert re.fullmatch(sample_regex, ip2) is not None
@@ -602,7 +606,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI, sample_regex,
606
607
608
@pytest.mark.asyncio
605
-async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
609
+async def test_structured_outputs_type_error(client: openai.AsyncOpenAI):
610
messages = [{
611
"role": "system",
612
"content": "you are a helpful assistant"
@@ -614,17 +618,19 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
618
}]
619
620
with pytest.raises(openai.BadRequestError):
617
- _ = await client.chat.completions.create(model=MODEL_NAME,
618
- messages=messages,
619
- extra_body=dict(guided_regex={
620
- 1: "Python",
621
- 2: "C++"
622
- }))
621
+ _ = await client.chat.completions.create(
622
+ model=MODEL_NAME,
623
+ messages=messages,
624
+ extra_body=dict(
625
+ structured_outputs={"regex": {
626
+ 1: "Python",
627
+ 2: "C++"
628
+ }}))
629
630
631
@pytest.mark.asyncio
626
-async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
627
- sample_guided_choice):
632
+async def test_structured_outputs_choice_chat_logprobs(
633
+ client: openai.AsyncOpenAI, sample_structured_outputs_choices):
634
635
messages = [{
636
"role": "system",
@@ -641,7 +647,8 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
647
max_completion_tokens=10,
648
logprobs=True,
649
top_logprobs=5,
644
- extra_body=dict(guided_choice=sample_guided_choice))
650
+ extra_body=dict(
651
+ structured_outputs={"choice": sample_structured_outputs_choices}))
652
653
assert chat_completion.choices[0].logprobs is not None
654
assert chat_completion.choices[0].logprobs.content is not None
@@ -663,10 +670,23 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema,
670
}, {
671
"role":
672
"user",
666
- "content":
667
- f"Give an example JSON for an employee profile that "
668
- f"fits this schema: {sample_json_schema}"
673
+ "content": ("Give an example JSON for an employee "
674
+ "profile using the specified tool.")
675
}]
676
+ tools = [{
677
+ "type": "function",
678
+ "function": {
679
+ "name": "dummy_function_name",
680
+ "description": "This is a dummy function",
681
+ "parameters": sample_json_schema
682
+ }
683
+ }]
684
+ tool_choice = {
685
+ "type": "function",
686
+ "function": {
687
+ "name": "dummy_function_name"
688
+ }
689
+ }
690
691
# non-streaming
692
@@ -674,20 +694,8 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema,
694
model=MODEL_NAME,
695
messages=messages,
696
max_completion_tokens=1000,
677
- tools=[{
678
- "type": "function",
679
- "function": {
680
- "name": "dummy_function_name",
681
- "description": "This is a dummy function",
682
- "parameters": sample_json_schema
683
- }
684
- }],
685
- tool_choice={
686
- "type": "function",
687
- "function": {
688
- "name": "dummy_function_name"
689
- }
690
- },
697
+ tools=tools,
698
+ tool_choice=tool_choice,
699
)
700
message = chat_completion.choices[0].message
701
assert len(message.content) == 0
@@ -705,25 +713,12 @@ async def test_named_tool_use(client: openai.AsyncOpenAI, sample_json_schema,
713
714
# streaming
715
708
- stream = await client.chat.completions.create(
709
- model=MODEL_NAME,
710
- messages=messages,
711
- max_completion_tokens=1000,
712
- tools=[{
713
- "type": "function",
714
- "function": {
715
- "name": "dummy_function_name",
716
- "description": "This is a dummy function",
717
- "parameters": sample_json_schema
718
- }
719
- }],
720
- tool_choice={
721
- "type": "function",
722
- "function": {
723
- "name": "dummy_function_name"
724
- }
725
- },
726
- stream=True)
716
+ stream = await client.chat.completions.create(model=MODEL_NAME,
717
+ messages=messages,
718
+ max_completion_tokens=1000,
719
+ tools=tools,
720
+ tool_choice=tool_choice,
721
+ stream=True)
722
723
output = []
724
finish_reason_count = 0
tests/entrypoints/openai/test_completion.py
+39
-40
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-# imports for guided decoding tests
3
+# imports for structured outputs tests
4
import json
5
import os
6
from typing import Optional
@@ -23,8 +23,6 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
23
# technically these adapters use a different base model,
24
# but we're not testing generation quality here
25
26
-GUIDED_DECODING_BACKENDS = ["outlines", "xgrammar", "guidance"]
27
-
26
27
@pytest.fixture(scope="module")
28
def default_server_args(zephyr_lora_files):
@@ -595,12 +593,13 @@ async def test_allowed_token_ids(client: openai.AsyncOpenAI):
593
594
595
@pytest.mark.asyncio
598
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
599
-async def test_guided_json_completion(client: openai.AsyncOpenAI,
600
- guided_decoding_backend: str,
601
- sample_json_schema, is_v1_server: bool):
596
+async def test_structured_outputs_json_completion(
597
+ client: openai.AsyncOpenAI,
598
+ sample_json_schema,
599
+ is_v1_server: bool,
600
+):
601
if not is_v1_server:
603
- pytest.skip("Guided decoding is only supported in v1 engine")
602
+ pytest.skip("structured outputs is only supported in v1 engine")
603
604
completion = await client.completions.create(
605
model=MODEL_NAME,
@@ -609,8 +608,7 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
608
n=3,
609
temperature=1.0,
610
max_tokens=500,
612
- extra_body=dict(guided_json=sample_json_schema,
613
- guided_decoding_backend=guided_decoding_backend))
611
+ extra_body=dict(structured_outputs=dict(json=sample_json_schema)))
612
613
assert completion.id is not None
614
assert len(completion.choices) == 3
@@ -620,12 +618,13 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
618
619
620
@pytest.mark.asyncio
623
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
624
-async def test_guided_regex_completion(client: openai.AsyncOpenAI,
625
- guided_decoding_backend: str,
626
- sample_regex, is_v1_server: bool):
621
+async def test_structured_outputs_regex_completion(
622
+ client: openai.AsyncOpenAI,
623
+ sample_regex,
624
+ is_v1_server: bool,
625
+):
626
if not is_v1_server:
628
- pytest.skip("Guided decoding is only supported in v1 engine")
627
+ pytest.skip("structured outputs is only supported in v1 engine")
628
629
completion = await client.completions.create(
630
model=MODEL_NAME,
@@ -633,8 +632,7 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
632
n=3,
633
temperature=1.0,
634
max_tokens=20,
636
- extra_body=dict(guided_regex=sample_regex,
637
- guided_decoding_backend=guided_decoding_backend))
635
+ extra_body=dict(structured_outputs=dict(regex=sample_regex)))
636
637
assert completion.id is not None
638
assert len(completion.choices) == 3
@@ -644,13 +642,13 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
642
643
644
@pytest.mark.asyncio
647
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
648
-async def test_guided_choice_completion(client: openai.AsyncOpenAI,
649
- guided_decoding_backend: str,
650
- sample_guided_choice,
651
- is_v1_server: bool):
645
+async def test_structured_outputs_choice_completion(
646
+ client: openai.AsyncOpenAI,
647
+ sample_structured_outputs_choices,
648
+ is_v1_server: bool,
649
+):
650
if not is_v1_server:
653
- pytest.skip("Guided decoding is only supported in v1 engine")
651
+ pytest.skip("structured outputs is only supported in v1 engine")
652
653
completion = await client.completions.create(
654
model=MODEL_NAME,
@@ -658,20 +656,21 @@ async def test_guided_choice_completion(client: openai.AsyncOpenAI,
656
n=2,
657
temperature=1.0,
658
max_tokens=10,
661
- extra_body=dict(guided_choice=sample_guided_choice,
662
- guided_decoding_backend=guided_decoding_backend))
659
+ extra_body=dict(structured_outputs=dict(
660
+ choice=sample_structured_outputs_choices)))
661
662
assert completion.id is not None
663
assert len(completion.choices) == 2
664
for i in range(2):
667
- assert completion.choices[i].text in sample_guided_choice
665
+ assert completion.choices[i].text in sample_structured_outputs_choices
666
667
668
@pytest.mark.asyncio
671
-async def test_guided_grammar(client: openai.AsyncOpenAI,
672
- sample_sql_statements, is_v1_server: bool):
669
+async def test_structured_outputs_grammar(client: openai.AsyncOpenAI,
670
+ sample_sql_statements,
671
+ is_v1_server: bool):
672
if not is_v1_server:
674
- pytest.skip("Guided grammar is only supported in v1 engine")
673
+ pytest.skip("grammar is only supported in v1 engine")
674
675
completion = await client.completions.create(
676
model=MODEL_NAME,
@@ -679,7 +678,8 @@ async def test_guided_grammar(client: openai.AsyncOpenAI,
678
"table_1 where it is equals to 1"),
679
temperature=1.0,
680
max_tokens=500,
682
- extra_body=dict(guided_grammar=sample_sql_statements))
681
+ extra_body=dict(
682
+ structured_outputs=dict(grammar=sample_sql_statements), ))
683
684
content = completion.choices[0].text
685
@@ -730,27 +730,26 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
730
731
732
@pytest.mark.asyncio
733
-@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
734
-async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
735
- guided_decoding_backend: str,
736
- sample_json_schema, sample_regex,
737
- is_v1_server: bool):
733
+async def test_structured_outputs_type_error(client: openai.AsyncOpenAI,
734
+ sample_json_schema, sample_regex,
735
+ is_v1_server: bool):
736
if not is_v1_server:
739
- pytest.skip("Guided decoding is only supported in v1 engine")
737
+ pytest.skip("structured outputs is only supported in v1 engine")
738
739
with pytest.raises(openai.BadRequestError):
740
_ = await client.completions.create(
741
model=MODEL_NAME,
742
prompt="Give an example JSON that fits this schema: 42",
745
- extra_body=dict(guided_json=42,
746
- guided_decoding_backend=guided_decoding_backend))
743
+ extra_body=dict(structured_outputs=dict(json=42)))
744
745
with pytest.raises(openai.BadRequestError):
746
_ = await client.completions.create(
747
model=MODEL_NAME,
748
prompt="Give an example string that fits this regex",
752
- extra_body=dict(guided_regex=sample_regex,
753
- guided_json=sample_json_schema))
749
+ extra_body=dict(structured_outputs=dict(
750
+ regex=sample_regex,
751
+ json=sample_json_schema,
752
+ )))
753
754
755
@pytest.mark.asyncio
tests/entrypoints/openai/test_completion_with_function_calling.py
+2
-2
@@ -142,7 +142,7 @@ def server(): # noqa: F811
142
"--dtype",
143
"half",
144
"--enable-auto-tool-choice",
145
- "--guided-decoding-backend",
145
+ "--structured-outputs-config.backend",
146
"xgrammar",
147
"--tool-call-parser",
148
"hermes",
@@ -225,7 +225,7 @@ def k2_server(): # noqa: F811
225
"--dtype",
226
"half",
227
"--enable-auto-tool-choice",
228
- "--guided-decoding-backend",
228
+ "--structured-outputs-config.backend",
229
"xgrammar",
230
"--tool-call-parser",
231
"hermes",
tests/entrypoints/openai/test_openai_schema.py
+5
-3
@@ -102,12 +102,14 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
102
if "custom" in tool_call:
103
return False
104
105
- # Sometimes guided_grammar is generated to be empty
105
+ # Sometimes structured_outputs.grammar is generated to be empty
106
# Causing a server error in EBNF grammar parsing
107
# https://github.com/vllm-project/vllm/pull/22587#issuecomment-3195253421
108
- guided_grammar = case.body.get("guided_grammar")
108
+ structured_outputs = case.body.get("structured_outputs", {})
109
+ grammar = structured_outputs.get("grammar") if isinstance(
110
+ structured_outputs, dict) else None
111
110
- if guided_grammar == '':
112
+ if grammar == '':
113
# Allow None (will be handled as no grammar)
114
# But skip empty strings
115
return False
tests/entrypoints/openai/test_prompt_validation.py
+1
-1
@@ -3,7 +3,7 @@
3
4
import io
5
6
-# imports for guided decoding tests
6
+# imports for structured outputs tests
7
import openai
8
import pybase64
9
import pytest
tests/entrypoints/openai/test_serving_chat.py
-4
@@ -333,7 +333,6 @@ async def test_serving_chat_should_set_correct_max_tokens():
333
"role": "user",
334
"content": "what is 1+1?"
335
}],
336
- guided_decoding_backend="outlines",
336
)
337
338
with suppress(Exception):
@@ -378,7 +377,6 @@ async def test_serving_chat_should_set_correct_max_tokens():
377
"role": "user",
378
"content": "what is 1+1?"
379
}],
381
- guided_decoding_backend="outlines",
380
)
381
382
with suppress(Exception):
@@ -433,7 +431,6 @@ async def test_serving_chat_should_set_correct_max_tokens():
431
"role": "user",
432
"content": "what is 1+1?"
433
}],
436
- guided_decoding_backend="outlines",
434
)
435
436
with suppress(Exception):
@@ -489,7 +486,6 @@ async def test_serving_chat_could_load_correct_generation_config():
486
"role": "user",
487
"content": "what is 1+1?"
488
}],
492
- guided_decoding_backend="outlines",
489
)
490
491
with suppress(Exception):
tests/entrypoints/openai/test_transcription_validation.py
+1
-1
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-# imports for guided decoding tests
4
+# imports for structured outputs tests
5
import io
6
import json
7
tests/entrypoints/openai/test_translation_validation.py
+1
-1
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import io
5
-# imports for guided decoding tests
5
+# imports for structured outputs tests
6
import json
7
8
import httpx
tests/test_sampling_params.py
deleted
-84
@@ -1,84 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""Tests for the SamplingParams class.
4
-"""
5
-
6
-import pytest
7
-
8
-from vllm import SamplingParams
9
-from vllm.config import ModelConfig
10
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest
11
-
12
-MODEL_NAME = "Qwen/Qwen1.5-7B"
13
-
14
-
15
-def test_max_tokens_none():
16
- """max_tokens=None should be allowed"""
17
- SamplingParams(temperature=0.01, top_p=0.1, max_tokens=None)
18
-
19
-
20
-@pytest.fixture(scope="module")
21
-def model_config():
22
- return ModelConfig(
23
- MODEL_NAME,
24
- seed=0,
25
- dtype="float16",
26
- )
27
-
28
-
29
-@pytest.fixture(scope="module")
30
-def default_max_tokens():
31
- return 4096
32
-
33
-
34
-def test_sampling_params_from_request_with_no_guided_decoding_backend(
35
- model_config, default_max_tokens):
36
- # guided_decoding_backend is not present at request level
37
- request = ChatCompletionRequest.model_validate({
38
- 'messages': [{
39
- 'role': 'user',
40
- 'content': 'Hello'
41
- }],
42
- 'model':
43
- MODEL_NAME,
44
- 'response_format': {
45
- 'type': 'json_object',
46
- },
47
- })
48
-
49
- sampling_params = request.to_sampling_params(
50
- default_max_tokens,
51
- model_config.logits_processor_pattern,
52
- )
53
- # we do not expect any backend to be present and the default
54
- # guided_decoding_backend at engine level will be used.
55
- assert sampling_params.guided_decoding.backend is None
56
-
57
-
58
-@pytest.mark.parametrize("request_level_guided_decoding_backend,expected",
59
- [("xgrammar", "xgrammar"), ("guidance", "guidance"),
60
- ("outlines", "outlines")])
61
-def test_sampling_params_from_request_with_guided_decoding_backend(
62
- request_level_guided_decoding_backend: str, expected: str,
63
- model_config, default_max_tokens):
64
-
65
- request = ChatCompletionRequest.model_validate({
66
- 'messages': [{
67
- 'role': 'user',
68
- 'content': 'Hello'
69
- }],
70
- 'model':
71
- MODEL_NAME,
72
- 'response_format': {
73
- 'type': 'json_object',
74
- },
75
- 'guided_decoding_backend':
76
- request_level_guided_decoding_backend,
77
- })
78
-
79
- sampling_params = request.to_sampling_params(
80
- default_max_tokens,
81
- model_config.logits_processor_pattern,
82
- )
83
- # backend correctly identified in resulting sampling_params
84
- assert sampling_params.guided_decoding.backend == expected
tests/tool_use/test_tool_choice_required.py
+6
-5
@@ -68,7 +68,7 @@ EXAMPLE_TOOLS = [
68
def _compile_and_check(tools: list[ChatCompletionToolsParam], sample_output,
69
should_match: bool):
70
self = MagicMock(tool_choice="required", tools=tools)
71
- schema = ChatCompletionRequest._get_guided_json_from_tool(self)
71
+ schema = ChatCompletionRequest._get_json_schema_from_tool(self)
72
assert isinstance(schema, dict)
73
74
# use build_regex_from_schema used in JSONLogitsProcessor to create Guide
@@ -218,7 +218,7 @@ VALID_TOOLS = [t[0] for t in VALID_TOOL_OUTPUTS]
218
}
219
}, {}], False),
220
])
221
-def test_guided_json(sample_output, should_match):
221
+def test_structured_outputs_json(sample_output, should_match):
222
_compile_and_check(tools=TypeAdapter(
223
list[ChatCompletionToolsParam]).validate_python(EXAMPLE_TOOLS),
224
sample_output=sample_output,
@@ -273,8 +273,9 @@ def update_parameters_empty_dict(
273
@pytest.mark.parametrize(
274
"update_parameters",
275
[update_parameters_none, update_parameters_empty_dict])
276
-def test_guided_json_without_parameters(sample_output, should_match,
277
- update_parameters):
276
+def test_structured_outputs_json_without_parameters(sample_output,
277
+ should_match,
278
+ update_parameters):
279
updated_tools = [deepcopy(EXAMPLE_TOOLS[0])]
280
tools = TypeAdapter(
281
list[ChatCompletionToolsParam]).validate_python(updated_tools)
@@ -334,4 +335,4 @@ def test_streaming_output_valid(output, empty_params, delta_len):
335
combined_messages += message.tool_calls[0].function.arguments
336
combined_messages += "}]"
337
assert json.loads(combined_messages) == output
337
- assert json.dumps(json.loads(combined_messages)) == output_json
\ No newline at end of file
338
+ assert json.dumps(json.loads(combined_messages)) == output_json
tests/v1/core/test_scheduler.py
+3
-3
@@ -10,7 +10,7 @@ from vllm.config import (CacheConfig, KVTransferConfig, ModelConfig,
10
SchedulerConfig, SpeculativeConfig, VllmConfig)
11
from vllm.multimodal.inputs import (MultiModalFeatureSpec,
12
MultiModalKwargsItem, PlaceholderRange)
13
-from vllm.sampling_params import GuidedDecodingParams, SamplingParams
13
+from vllm.sampling_params import SamplingParams, StructuredOutputsParams
14
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
15
from vllm.v1.core.sched.scheduler import Scheduler
16
from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
@@ -1796,11 +1796,11 @@ def test_schedule_skip_tokenizer_init():
1796
1797
def test_schedule_skip_tokenizer_init_structured_output_request():
1798
scheduler = create_scheduler(skip_tokenizer_init=True)
1799
- guided_params = GuidedDecodingParams(regex="[0-9]+")
1799
+ structured_outputs_params = StructuredOutputsParams(regex="[0-9]+")
1800
sampling_params = SamplingParams(
1801
ignore_eos=False,
1802
max_tokens=16,
1803
- guided_decoding=guided_params,
1803
+ structured_outputs=structured_outputs_params,
1804
)
1805
request = Request(
1806
request_id="0",
tests/v1/engine/test_llm_engine.py
+2
-2
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Optional
8
import pytest
9
10
from vllm import LLM
11
-from vllm.sampling_params import GuidedDecodingParams, SamplingParams
11
+from vllm.sampling_params import SamplingParams, StructuredOutputsParams
12
from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
13
14
if TYPE_CHECKING:
@@ -97,7 +97,7 @@ def _get_test_sampling_params(
97
top_p=0.95,
98
n=n,
99
seed=seed,
100
- guided_decoding=GuidedDecodingParams(
100
+ structured_outputs=StructuredOutputsParams(
101
regex="[0-9]+") if structured_outputs else None,
102
) for n in n_list
103
], n_list
tests/v1/entrypoints/conftest.py
+1
-1
@@ -151,7 +151,7 @@ def sample_definition_json_schema():
151
152
153
@pytest.fixture
154
-def sample_guided_choice():
154
+def sample_structured_outputs_choices():
155
return [
156
"Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript",
157
"Ruby", "Swift", "Kotlin"
tests/v1/entrypoints/llm/test_struct_output_generate.py
+72
-63
@@ -15,12 +15,13 @@ import torch
15
from pydantic import BaseModel
16
17
from tests.reasoning.utils import run_reasoning_extraction
18
+from vllm.config import StructuredOutputsConfig
19
from vllm.distributed import cleanup_dist_env_and_memory
20
from vllm.entrypoints.llm import LLM
21
from vllm.outputs import RequestOutput
22
from vllm.platforms import current_platform
23
from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager
23
-from vllm.sampling_params import GuidedDecodingParams, SamplingParams
24
+from vllm.sampling_params import SamplingParams, StructuredOutputsParams
25
26
if TYPE_CHECKING:
27
from vllm.config import TokenizerMode
@@ -90,7 +91,7 @@ def _load_json(s: str, backend: str) -> str:
91
92
@pytest.mark.skip_global_cleanup
93
@pytest.mark.parametrize(
93
- "model_name, guided_decoding_backend, tokenizer_mode, speculative_config",
94
+ "model_name, backend, tokenizer_mode, speculative_config",
95
PARAMS_MODELS_BACKENDS_TOKENIZER_MODE)
96
def test_structured_output(
97
monkeypatch: pytest.MonkeyPatch,
@@ -99,8 +100,8 @@ def test_structured_output(
100
sample_sql_ebnf: str,
101
sample_sql_lark: str,
102
sample_regex: str,
102
- sample_guided_choice: str,
103
- guided_decoding_backend: str,
103
+ sample_structured_outputs_choices: str,
104
+ backend: str,
105
tokenizer_mode: str,
106
model_name: str,
107
speculative_config: dict[str, Any],
@@ -115,16 +116,15 @@ def test_structured_output(
116
enforce_eager = bool(not current_platform.is_tpu())
117
# Use a single LLM instance for several scenarios to
118
# speed up the test suite.
118
- llm = LLM(
119
- model=model_name,
120
- enforce_eager=enforce_eager,
121
- max_model_len=1024,
122
- guided_decoding_backend=guided_decoding_backend,
123
- guided_decoding_disable_any_whitespace=(guided_decoding_backend
124
- in {"xgrammar", "guidance"}),
125
- seed=120,
126
- tokenizer_mode=tokenizer_mode,
127
- speculative_config=speculative_config)
119
+ llm = LLM(model=model_name,
120
+ enforce_eager=enforce_eager,
121
+ max_model_len=1024,
122
+ structured_outputs_config=dict(backend=backend,
123
+ disable_any_whitespace=backend
124
+ in {"xgrammar", "guidance"}),
125
+ seed=120,
126
+ tokenizer_mode=tokenizer_mode,
127
+ speculative_config=speculative_config)
128
129
#
130
# Test 1: Generate JSON output based on a provided schema
@@ -132,7 +132,7 @@ def test_structured_output(
132
sampling_params = SamplingParams(
133
temperature=1.0,
134
max_tokens=4096,
135
- guided_decoding=GuidedDecodingParams(json=sample_json_schema))
135
+ structured_outputs=StructuredOutputsParams(json=sample_json_schema))
136
137
prompt = ("Give an example JSON for an employee profile that fits this "
138
"schema. Make the response as short as possible. Schema: "
@@ -152,7 +152,7 @@ def test_structured_output(
152
153
generated_text = output.outputs[0].text
154
assert generated_text is not None
155
- if guided_decoding_backend != 'lm-format-enforcer':
155
+ if backend != 'lm-format-enforcer':
156
assert "\n" not in generated_text
157
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
158
output_json = json.loads(generated_text)
@@ -161,12 +161,12 @@ def test_structured_output(
161
#
162
# Test 2: Generate JSON object without a schema
163
#
164
- if guided_decoding_backend != "outlines":
164
+ if backend != "outlines":
165
sampling_params = SamplingParams(
166
temperature=1.0,
167
max_tokens=4096,
168
n=2,
169
- guided_decoding=GuidedDecodingParams(json_object=True))
169
+ structured_outputs=StructuredOutputsParams(json_object=True))
170
171
outputs = llm.generate(prompts=(
172
"Generate a JSON object with curly braces for a person with "
@@ -195,8 +195,9 @@ def test_structured_output(
195
sampling_params = SamplingParams(
196
temperature=1.0,
197
max_tokens=4096,
198
- guided_decoding=GuidedDecodingParams(json=unsupported_json_schema))
199
- if guided_decoding_backend.startswith("xgrammar"):
198
+ structured_outputs=StructuredOutputsParams(
199
+ json=unsupported_json_schema))
200
+ if backend.startswith("xgrammar"):
201
with pytest.raises(ValueError,
202
match="The provided JSON schema contains features "
203
"not supported by xgrammar."):
@@ -230,7 +231,7 @@ def test_structured_output(
231
parsed_json = json.loads(generated_text)
232
assert isinstance(parsed_json, dict)
233
233
- if guided_decoding_backend not in ["outlines", "lm-format-enforcer"]:
234
+ if backend not in ["outlines", "lm-format-enforcer"]:
235
#
236
# Test 4: Generate SQL statement using EBNF grammar
237
#
@@ -238,7 +239,8 @@ def test_structured_output(
239
temperature=0.8,
240
top_p=0.95,
241
max_tokens=1000,
241
- guided_decoding=GuidedDecodingParams(grammar=sample_sql_ebnf))
242
+ structured_outputs=StructuredOutputsParams(
243
+ grammar=sample_sql_ebnf))
244
outputs = llm.generate(
245
("Generate a sql statement that selects col_1 from "
246
"table_1 where it is equal to 1. Make the response as short as "
@@ -271,7 +273,8 @@ def test_structured_output(
273
temperature=0.8,
274
top_p=0.95,
275
max_tokens=1000,
274
- guided_decoding=GuidedDecodingParams(grammar=sample_sql_lark))
276
+ structured_outputs=StructuredOutputsParams(
277
+ grammar=sample_sql_lark))
278
outputs = llm.generate(
279
("Generate a sql statement that selects col_1 from "
280
"table_1 where it is equal to 1. Make the response as short as "
@@ -309,7 +312,8 @@ def test_structured_output(
312
temperature=0.8,
313
top_p=0.95,
314
max_tokens=1000,
312
- guided_decoding=GuidedDecodingParams(grammar="not a grammar"))
315
+ structured_outputs=StructuredOutputsParams(
316
+ grammar="not a grammar"))
317
with pytest.raises(ValueError, match="Failed to convert the grammar "):
318
llm.generate(
319
("Generate a sql statement that selects col_1 from "
@@ -325,7 +329,7 @@ def test_structured_output(
329
sampling_params = SamplingParams(
330
temperature=0.8,
331
top_p=0.95,
328
- guided_decoding=GuidedDecodingParams(regex=sample_regex))
332
+ structured_outputs=StructuredOutputsParams(regex=sample_regex))
333
334
prompt = (f"Give an example IPv4 address with this regex: {sample_regex}. "
335
f"Make the response as short as possible.")
@@ -352,7 +356,8 @@ def test_structured_output(
356
sampling_params = SamplingParams(
357
temperature=0.8,
358
top_p=0.95,
355
- guided_decoding=GuidedDecodingParams(choice=sample_guided_choice))
359
+ structured_outputs=StructuredOutputsParams(
360
+ choice=sample_structured_outputs_choices))
361
362
outputs = llm.generate(
363
("The best language for type-safe systems programming is "
@@ -368,7 +373,7 @@ def test_structured_output(
373
generated_text = output.outputs[0].text
374
print(generated_text)
375
assert generated_text is not None
371
- assert generated_text in sample_guided_choice
376
+ assert generated_text in sample_structured_outputs_choices
377
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
378
379
#
@@ -378,7 +383,7 @@ def test_structured_output(
383
sampling_params = SamplingParams(
384
temperature=1.0,
385
max_tokens=1000,
381
- guided_decoding=GuidedDecodingParams(json=json_schema))
386
+ structured_outputs=StructuredOutputsParams(json=json_schema))
387
388
outputs = llm.generate(
389
("Generate a JSON with the brand, model and car_type of the most "
@@ -422,7 +427,7 @@ def test_structured_output(
427
sampling_params = SamplingParams(
428
temperature=1.0,
429
max_tokens=4096,
425
- guided_decoding=GuidedDecodingParams(json=json_schema))
430
+ structured_outputs=StructuredOutputsParams(json=json_schema))
431
432
outputs = llm.generate(
433
("Generate a description of a frog using 50 characters. "
@@ -444,7 +449,7 @@ def test_structured_output(
449
output_json = json.loads(generated_text)
450
jsonschema.validate(instance=output_json, schema=json_schema)
451
447
- if guided_decoding_backend not in ["outlines", "lm-format-enforcer"]:
452
+ if backend not in ["outlines", "lm-format-enforcer"]:
453
#
454
# Test 11: Generate structured output using structural_tag format
455
#
@@ -470,7 +475,7 @@ def test_structured_output(
475
sampling_params = SamplingParams(
476
temperature=0.0,
477
max_tokens=4096,
473
- guided_decoding=GuidedDecodingParams(
478
+ structured_outputs=StructuredOutputsParams(
479
structural_tag=json.dumps(structural_tag_config)))
480
481
prompt = """
@@ -547,7 +552,7 @@ Make the response as short as possible.
552
553
@pytest.mark.skip_global_cleanup
554
@pytest.mark.parametrize(
550
- "model_name, guided_decoding_backend, tokenizer_mode, reasoning_parser, speculative_config", # noqa: E501
555
+ "model_name, backend, tokenizer_mode, reasoning_parser, speculative_config", # noqa: E501
556
[
557
("deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", "xgrammar", "auto",
558
"deepseek_r1", NGRAM_SPEC_CONFIG),
@@ -556,7 +561,7 @@ Make the response as short as possible.
561
)
562
def test_structured_output_with_reasoning_matrices(
563
monkeypatch: pytest.MonkeyPatch,
559
- guided_decoding_backend: str,
564
+ backend: str,
565
tokenizer_mode: TokenizerMode,
566
reasoning_parser: str,
567
model_name: str,
@@ -576,10 +581,11 @@ def test_structured_output_with_reasoning_matrices(
581
enforce_eager=bool(not current_platform.is_tpu()),
582
max_model_len=1024,
583
max_num_seqs=16,
579
- guided_decoding_backend=guided_decoding_backend,
580
- guided_decoding_disable_any_whitespace=True,
584
+ structured_outputs_config=dict(backend=backend,
585
+ disable_any_whitespace=backend
586
+ in {"xgrammar", "guidance"},
587
+ reasoning_parser=reasoning_parser),
588
tokenizer_mode=tokenizer_mode,
582
- reasoning_parser=reasoning_parser,
589
speculative_config=speculative_config,
590
)
591
tokenizer = llm.get_tokenizer()
@@ -603,7 +609,7 @@ def test_structured_output_with_reasoning_matrices(
609
sampling_params = SamplingParams(
610
temperature=0.1,
611
max_tokens=8192,
606
- guided_decoding=GuidedDecodingParams(json=reasoning_schema),
612
+ structured_outputs=StructuredOutputsParams(json=reasoning_schema),
613
)
614
outputs = llm.generate(
615
[reasoning_prompt],
@@ -640,13 +646,14 @@ def test_structured_output_auto_mode(
646
647
llm = LLM(model=model_name,
648
max_model_len=1024,
643
- guided_decoding_backend="auto",
649
+ structured_outputs_config=dict(backend="auto"),
650
tokenizer_mode=tokenizer_mode)
651
652
sampling_params = SamplingParams(
653
temperature=1.0,
654
max_tokens=1000,
649
- guided_decoding=GuidedDecodingParams(json=unsupported_json_schema))
655
+ structured_outputs=StructuredOutputsParams(
656
+ json=unsupported_json_schema))
657
658
prompts = (
659
"Give an example JSON object for a grade "
@@ -681,9 +688,10 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
688
689
llm = LLM(model="Qwen/Qwen2.5-1.5B-Instruct",
690
max_model_len=1024,
684
- guided_decoding_backend="guidance",
685
- guided_decoding_disable_any_whitespace=True,
686
- guided_decoding_disable_additional_properties=True)
691
+ structured_outputs_config=dict(
692
+ backend="guidance",
693
+ disable_any_whitespace=True,
694
+ disable_additional_properties=True))
695
696
schema = {
697
'type': 'object',
@@ -709,14 +717,15 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
717
"<|im_end|>\n<|im_start|>assistant\n")
718
719
def generate_with_backend(backend):
712
- guided_params = GuidedDecodingParams(
720
+ structured_outputs_params = StructuredOutputsParams(
721
json=schema,
722
backend=backend,
723
disable_any_whitespace=True,
724
disable_additional_properties=True)
717
- sampling_params = SamplingParams(temperature=0,
718
- max_tokens=256,
719
- guided_decoding=guided_params)
725
+ sampling_params = SamplingParams(
726
+ temperature=0,
727
+ max_tokens=256,
728
+ structured_outputs=structured_outputs_params)
729
730
outputs = llm.generate(prompt, sampling_params=sampling_params)
731
assert outputs is not None
@@ -736,12 +745,11 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
745
assert "a6" not in generated
746
747
739
-@pytest.mark.parametrize("guided_decoding_backend",
740
- ["guidance", "xgrammar", "outlines"])
741
-def test_structured_output_batched_with_non_guided_requests(
748
+@pytest.mark.parametrize("backend", ["guidance", "xgrammar", "outlines"])
749
+def test_structured_output_batched_with_non_structured_outputs_requests(
750
monkeypatch: pytest.MonkeyPatch,
751
sample_json_schema: dict[str, Any],
744
- guided_decoding_backend: str,
752
+ backend: str,
753
):
754
monkeypatch.setenv("VLLM_USE_V1", "1")
755
@@ -753,24 +761,25 @@ def test_structured_output_batched_with_non_guided_requests(
761
model="meta-llama/Meta-Llama-3.1-8B-Instruct",
762
enforce_eager=enforce_eager,
763
max_model_len=1024,
756
- guided_decoding_backend=guided_decoding_backend,
757
- guided_decoding_disable_any_whitespace=(guided_decoding_backend
758
- in {"xgrammar", "guidance"}),
764
+ structured_outputs_config=StructuredOutputsConfig(
765
+ backend=backend,
766
+ disable_any_whitespace=backend in {"xgrammar", "guidance"},
767
+ ),
768
)
769
761
- guided_prompt = (
770
+ structured_outputs_prompt = (
771
"Give an example JSON for an employee profile that fits this "
772
"schema. Make the response as short as possible. Schema: "
773
f"{sample_json_schema}")
774
766
- non_guided_prompt = "The diameter of the Earth in kilometers is "
775
+ non_structured_outputs_prompt = "The diameter of the Earth in kilometers is "
776
768
- prompts = [guided_prompt, non_guided_prompt]
777
+ prompts = [structured_outputs_prompt, non_structured_outputs_prompt]
778
sampling_params = [
770
- SamplingParams(
771
- temperature=1.0,
772
- max_tokens=400,
773
- guided_decoding=GuidedDecodingParams(json=sample_json_schema)),
779
+ SamplingParams(temperature=1.0,
780
+ max_tokens=400,
781
+ structured_outputs=StructuredOutputsParams(
782
+ json=sample_json_schema)),
783
# No max tokens, temp=0 to assert on contents
784
SamplingParams(
785
seed=42,
@@ -801,16 +810,16 @@ def test_structured_output_batched_with_non_guided_requests(
810
print(f"Prompt:\n{prompt!r}\nGenerated text:\n{generated_text!r}")
811
812
if index == 0:
804
- # First prompt is guided, expect valid JSON
813
+ # First prompt is structured outputs, expect valid JSON
814
assert "\n" not in generated_text
815
output_json = json.loads(generated_text)
816
jsonschema.validate(instance=output_json,
817
schema=sample_json_schema)
818
else:
810
- # Second prompt is not guided, expect valid output
819
+ # Second prompt is not structured outputs, expect valid output
820
# Cannot assert on exact output, but we can expect it to be factual
821
assert "12,742" in generated_text
822
814
- # non-guided requests should not return a valid JSON here
823
+ # non-structured outputs requests should not return a valid JSON here
824
with pytest.raises(ValueError):
825
output_json = json.loads(generated_text)
tests/v1/entrypoints/openai/test_chat_completion.py
+11
-3
@@ -77,7 +77,9 @@ async def test_invalid_json_schema(client: openai.AsyncOpenAI,
77
"role": "user",
78
"content": prompt,
79
}],
80
- extra_body={"guided_json": invalid_json_schema},
80
+ extra_body={"structured_outputs": {
81
+ "json": invalid_json_schema
82
+ }},
83
)
84
85
@@ -99,7 +101,9 @@ async def test_invalid_regex(client: openai.AsyncOpenAI, model_name: str):
101
"content": prompt,
102
}],
103
extra_body={
102
- "guided_regex": r"[.*",
104
+ "structured_outputs": {
105
+ "regex": r"[.*"
106
+ },
107
"stop": ["\n"]
108
},
109
)
@@ -134,5 +138,9 @@ async def test_invalid_grammar(client: openai.AsyncOpenAI, model_name: str):
138
"role": "user",
139
"content": prompt,
140
}],
137
- extra_body={"guided_grammar": invalid_simplified_sql_grammar},
141
+ extra_body={
142
+ "structured_outputs": {
143
+ "grammar": invalid_simplified_sql_grammar
144
+ }
145
+ },
146
)
tests/v1/entrypoints/openai/test_completion.py
+11
-3
@@ -627,7 +627,9 @@ async def test_invalid_json_schema(client: openai.AsyncOpenAI,
627
await client.completions.create(
628
model=model_name,
629
prompt=prompt,
630
- extra_body={"guided_json": invalid_json_schema},
630
+ extra_body={"structured_outputs": {
631
+ "json": invalid_json_schema
632
+ }},
633
)
634
635
@@ -646,7 +648,9 @@ async def test_invalid_regex(client: openai.AsyncOpenAI, model_name: str):
648
model=model_name,
649
prompt=prompt,
650
extra_body={
649
- "guided_regex": r"[.*",
651
+ "structured_outputs": {
652
+ "regex": r"[.*"
653
+ },
654
"stop": ["\n"]
655
},
656
)
@@ -678,7 +682,11 @@ async def test_invalid_grammar(client: openai.AsyncOpenAI, model_name: str):
682
await client.completions.create(
683
model=model_name,
684
prompt=prompt,
681
- extra_body={"guided_grammar": invalid_simplified_sql_grammar},
685
+ extra_body={
686
+ "structured_outputs": {
687
+ "grammar": invalid_simplified_sql_grammar
688
+ }
689
+ },
690
)
691
692
vllm/config/__init__.py
+18
-17
@@ -2277,34 +2277,34 @@ def get_served_model_name(model: str,
2277
return served_model_name
2278
2279
2280
-GuidedDecodingBackend = Literal["auto", "xgrammar", "guidance", "outlines",
2281
- "lm-format-enforcer"]
2280
+StructuredOutputsBackend = Literal["auto", "xgrammar", "guidance", "outlines",
2281
+ "lm-format-enforcer"]
2282
2283
2284
@config
2285
@dataclass
2286
-class DecodingConfig:
2287
- """Dataclass which contains the decoding strategy of the engine."""
2286
+class StructuredOutputsConfig:
2287
+ """Dataclass which contains structured outputs config for the engine."""
2288
2289
- backend: GuidedDecodingBackend = "auto"
2290
- """Which engine will be used for guided decoding (JSON schema / regex etc)
2291
- by default. With "auto", we will make opinionated choices based on request
2292
- contents and what the backend libraries currently support, so the behavior
2293
- is subject to change in each release."""
2289
+ backend: StructuredOutputsBackend = "auto"
2290
+ """Which engine will be used for structured outputs (e.g. JSON schema,
2291
+ regex, etc) by default. With "auto", we will make opinionated choices
2292
+ based on request contents and what the backend libraries currently support,
2293
+ so the behavior is subject to change in each release."""
2294
2295
disable_fallback: bool = False
2296
"""If `True`, vLLM will not fallback to a different backend on error."""
2297
2298
disable_any_whitespace: bool = False
2299
- """If `True`, the model will not generate any whitespace during guided
2300
- decoding. This is only supported for xgrammar and guidance backends."""
2299
+ """If `True`, the model will not generate any whitespace during structured
2300
+ outputs. This is only supported for xgrammar and guidance backends."""
2301
2302
disable_additional_properties: bool = False
2303
"""If `True`, the `guidance` backend will not use `additionalProperties`
2304
in the JSON schema. This is only supported for the `guidance` backend and
2305
is used to better align its behaviour with `outlines` and `xgrammar`."""
2306
2307
- reasoning_backend: str = ""
2307
+ reasoning_parser: str = ""
2308
"""Select the reasoning parser depending on the model that you're using.
2309
This is used to parse the reasoning content into OpenAI API format."""
2310
@@ -2451,8 +2451,9 @@ class VllmConfig:
2451
"""LoRA configuration."""
2452
speculative_config: Optional[SpeculativeConfig] = None
2453
"""Speculative decoding configuration."""
2454
- decoding_config: DecodingConfig = field(default_factory=DecodingConfig)
2455
- """Decoding configuration."""
2454
+ structured_outputs_config: StructuredOutputsConfig = field(
2455
+ default_factory=StructuredOutputsConfig)
2456
+ """Structured outputs configuration."""
2457
observability_config: Optional[ObservabilityConfig] = None
2458
"""Observability configuration."""
2459
quant_config: Optional[QuantizationConfig] = None
@@ -2543,8 +2544,8 @@ class VllmConfig:
2544
vllm_factors.append(self.speculative_config.compute_hash())
2545
else:
2546
vllm_factors.append("None")
2546
- if self.decoding_config:
2547
- vllm_factors.append(self.decoding_config.compute_hash())
2547
+ if self.structured_outputs_config:
2548
+ vllm_factors.append(self.structured_outputs_config.compute_hash())
2549
else:
2550
vllm_factors.append("None")
2551
if self.observability_config:
@@ -3063,7 +3064,7 @@ class VllmConfig:
3064
f"enforce_eager={self.model_config.enforce_eager}, "
3065
f"kv_cache_dtype={self.cache_config.cache_dtype}, "
3066
f"device_config={self.device_config.device}, "
3066
- f"decoding_config={self.decoding_config!r}, "
3067
+ f"structured_outputs_config={self.structured_outputs_config!r}, "
3068
f"observability_config={self.observability_config!r}, "
3069
f"seed={self.model_config.seed}, "
3070
f"served_model_name={self.model_config.served_model_name}, "
vllm/engine/arg_utils.py
+55
-40
@@ -22,17 +22,16 @@ from typing_extensions import TypeIs, deprecated
22
23
import vllm.envs as envs
24
from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
25
- ConfigType, ConvertOption, DecodingConfig,
26
- DetailedTraceModules, Device, DeviceConfig,
27
- DistributedExecutorBackend, EPLBConfig,
28
- GuidedDecodingBackend, HfOverrides, KVEventsConfig,
25
+ ConfigType, ConvertOption, DetailedTraceModules,
26
+ Device, DeviceConfig, DistributedExecutorBackend,
27
+ EPLBConfig, HfOverrides, KVEventsConfig,
28
KVTransferConfig, LoadConfig, LogprobsMode,
29
LoRAConfig, MambaDType, MMEncoderTPMode, ModelConfig,
30
ModelDType, ModelImpl, ObservabilityConfig,
31
ParallelConfig, PoolerConfig, PrefixCachingHashAlgo,
32
RunnerOption, SchedulerConfig, SchedulerPolicy,
34
- SpeculativeConfig, TaskOption, TokenizerMode,
35
- VllmConfig, get_attr_docs)
33
+ SpeculativeConfig, StructuredOutputsConfig,
34
+ TaskOption, TokenizerMode, VllmConfig, get_attr_docs)
35
from vllm.config.multimodal import MMCacheType, MultiModalConfig
36
from vllm.config.parallel import ExpertPlacementStrategy
37
from vllm.config.utils import get_field
@@ -418,12 +417,15 @@ class EngineArgs:
417
disable_hybrid_kv_cache_manager: bool = (
418
SchedulerConfig.disable_hybrid_kv_cache_manager)
419
421
- guided_decoding_backend: GuidedDecodingBackend = DecodingConfig.backend
422
- guided_decoding_disable_fallback: bool = DecodingConfig.disable_fallback
423
- guided_decoding_disable_any_whitespace: bool = \
424
- DecodingConfig.disable_any_whitespace
425
- guided_decoding_disable_additional_properties: bool = \
426
- DecodingConfig.disable_additional_properties
420
+ structured_outputs_config: StructuredOutputsConfig = get_field(
421
+ VllmConfig, "structured_outputs_config")
422
+ reasoning_parser: str = StructuredOutputsConfig.reasoning_parser
423
+ # Deprecated guided decoding fields
424
+ guided_decoding_backend: Optional[str] = None
425
+ guided_decoding_disable_fallback: Optional[bool] = None
426
+ guided_decoding_disable_any_whitespace: Optional[bool] = None
427
+ guided_decoding_disable_additional_properties: Optional[bool] = None
428
+
429
logits_processor_pattern: Optional[
430
str] = ModelConfig.logits_processor_pattern
431
@@ -462,7 +464,6 @@ class EngineArgs:
464
465
additional_config: dict[str, Any] = \
466
get_field(VllmConfig, "additional_config")
465
- reasoning_parser: str = DecodingConfig.reasoning_backend
467
468
use_tqdm_on_load: bool = LoadConfig.use_tqdm_on_load
469
pt_load_map_location: str = LoadConfig.pt_load_map_location
@@ -618,28 +619,29 @@ class EngineArgs:
619
load_group.add_argument('--pt-load-map-location',
620
**load_kwargs["pt_load_map_location"])
621
621
- # Guided decoding arguments
622
- guided_decoding_kwargs = get_kwargs(DecodingConfig)
623
- guided_decoding_group = parser.add_argument_group(
624
- title="DecodingConfig",
625
- description=DecodingConfig.__doc__,
622
+ # Structured outputs arguments
623
+ structured_outputs_kwargs = get_kwargs(StructuredOutputsConfig)
624
+ structured_outputs_group = parser.add_argument_group(
625
+ title="StructuredOutputsConfig",
626
+ description=StructuredOutputsConfig.__doc__,
627
)
627
- guided_decoding_group.add_argument("--guided-decoding-backend",
628
- **guided_decoding_kwargs["backend"])
629
- guided_decoding_group.add_argument(
630
- "--guided-decoding-disable-fallback",
631
- **guided_decoding_kwargs["disable_fallback"])
632
- guided_decoding_group.add_argument(
633
- "--guided-decoding-disable-any-whitespace",
634
- **guided_decoding_kwargs["disable_any_whitespace"])
635
- guided_decoding_group.add_argument(
636
- "--guided-decoding-disable-additional-properties",
637
- **guided_decoding_kwargs["disable_additional_properties"])
638
- guided_decoding_group.add_argument(
628
+ structured_outputs_group.add_argument(
629
"--reasoning-parser",
630
# This choice is a special case because it's not static
631
choices=list(ReasoningParserManager.reasoning_parsers),
642
- **guided_decoding_kwargs["reasoning_backend"])
632
+ **structured_outputs_kwargs["reasoning_parser"])
633
+ # Deprecated guided decoding arguments
634
+ for arg, type in [
635
+ ("--guided-decoding-backend", str),
636
+ ("--guided-decoding-disable-fallback", bool),
637
+ ("--guided-decoding-disable-any-whitespace", bool),
638
+ ("--guided-decoding-disable-additional-properties", bool),
639
+ ]:
640
+ structured_outputs_group.add_argument(
641
+ arg,
642
+ type=type,
643
+ help=(f"[DEPRECATED] {arg} will be removed in v0.12.0."),
644
+ deprecated=True)
645
646
# Parallel arguments
647
parallel_kwargs = get_kwargs(ParallelConfig)
@@ -934,6 +936,8 @@ class EngineArgs:
936
**vllm_kwargs["compilation_config"])
937
vllm_group.add_argument("--additional-config",
938
**vllm_kwargs["additional_config"])
939
+ vllm_group.add_argument('--structured-outputs-config',
940
+ **vllm_kwargs["structured_outputs_config"])
941
942
# Other arguments
943
parser.add_argument('--disable-log-stats',
@@ -1421,14 +1425,25 @@ class EngineArgs:
1425
1426
load_config = self.create_load_config()
1427
1424
- decoding_config = DecodingConfig(
1425
- backend=self.guided_decoding_backend,
1426
- disable_fallback=self.guided_decoding_disable_fallback,
1427
- disable_any_whitespace=self.guided_decoding_disable_any_whitespace,
1428
- disable_additional_properties=\
1429
- self.guided_decoding_disable_additional_properties,
1430
- reasoning_backend=self.reasoning_parser
1431
- )
1428
+ # Pass reasoning_parser into StructuredOutputsConfig
1429
+ if self.reasoning_parser:
1430
+ self.structured_outputs_config.reasoning_parser = \
1431
+ self.reasoning_parser
1432
+
1433
+ # Forward the deprecated CLI args to the StructuredOutputsConfig
1434
+ so_config = self.structured_outputs_config
1435
+ if self.guided_decoding_backend is not None:
1436
+ so_config.guided_decoding_backend = \
1437
+ self.guided_decoding_backend
1438
+ if self.guided_decoding_disable_fallback is not None:
1439
+ so_config.guided_decoding_disable_fallback = \
1440
+ self.guided_decoding_disable_fallback
1441
+ if self.guided_decoding_disable_any_whitespace is not None:
1442
+ so_config.guided_decoding_disable_any_whitespace = \
1443
+ self.guided_decoding_disable_any_whitespace
1444
+ if self.guided_decoding_disable_additional_properties is not None:
1445
+ so_config.guided_decoding_disable_additional_properties = \
1446
+ self.guided_decoding_disable_additional_properties
1447
1448
observability_config = ObservabilityConfig(
1449
show_hidden_metrics_for_version=(
@@ -1446,7 +1461,7 @@ class EngineArgs:
1461
lora_config=lora_config,
1462
speculative_config=speculative_config,
1463
load_config=load_config,
1449
- decoding_config=decoding_config,
1464
+ structured_outputs_config=self.structured_outputs_config,
1465
observability_config=observability_config,
1466
compilation_config=self.compilation_config,
1467
kv_transfer_config=self.kv_transfer_config,
vllm/engine/async_llm_engine.py
+1
-6
@@ -10,9 +10,8 @@ from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
10
from weakref import ReferenceType
11
12
import vllm.envs as envs
13
-from vllm.config import (DecodingConfig, ModelConfig, ParallelConfig,
13
+from vllm.config import (LoRAConfig, ModelConfig, ParallelConfig,
14
SchedulerConfig, VllmConfig)
15
-from vllm.config.lora import LoRAConfig
15
from vllm.core.scheduler import SchedulerOutputs
16
from vllm.engine.arg_utils import AsyncEngineArgs
17
from vllm.engine.async_timeout import asyncio_timeout
@@ -955,10 +954,6 @@ class AsyncLLMEngine(EngineClient):
954
"""Get the parallel configuration of the vLLM engine."""
955
return self.engine.get_parallel_config()
956
958
- async def get_decoding_config(self) -> DecodingConfig:
959
- """Get the decoding configuration of the vLLM engine."""
960
- return self.engine.get_decoding_config()
961
-
957
async def get_scheduler_config(self) -> SchedulerConfig:
958
"""Get the scheduling configuration of the vLLM engine."""
959
return self.engine.get_scheduler_config()
vllm/engine/llm_engine.py
+6
-12
@@ -16,9 +16,8 @@ import torch
16
from typing_extensions import TypeVar
17
18
import vllm.envs as envs
19
-from vllm.config import (DecodingConfig, ModelConfig, ObservabilityConfig,
19
+from vllm.config import (LoRAConfig, ModelConfig, ObservabilityConfig,
20
ParallelConfig, SchedulerConfig, VllmConfig)
21
-from vllm.config.lora import LoRAConfig
21
from vllm.core.scheduler import ScheduledSequenceGroup, SchedulerOutputs
22
from vllm.engine.arg_utils import EngineArgs
23
from vllm.engine.metrics_types import StatLoggerBase, Stats
@@ -213,8 +212,7 @@ class LLMEngine:
212
self.device_config = vllm_config.device_config
213
self.speculative_config = vllm_config.speculative_config # noqa
214
self.load_config = vllm_config.load_config
216
- self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
217
- )
215
+ self.structured_outputs_config = vllm_config.structured_outputs_config
216
self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
217
)
218
@@ -364,10 +362,9 @@ class LLMEngine:
362
self.observability_config.otlp_traces_endpoint)
363
364
# Initialize reasoning parser if reasoning backend is set.
367
- if self.decoding_config.reasoning_backend and \
368
- self.tokenizer:
365
+ if self.structured_outputs_config.reasoning_parser and self.tokenizer:
366
reasoner_class = ReasoningParserManager.get_reasoning_parser(
370
- self.decoding_config.reasoning_backend)
367
+ self.structured_outputs_config.reasoning_parser)
368
self.reasoner: ReasoningParser = reasoner_class(
369
self.tokenizer.get_lora_tokenizer())
370
@@ -381,7 +378,8 @@ class LLMEngine:
378
self.seq_counter,
379
stop_checker=StopChecker(
380
self.scheduler_config.max_model_len,
384
- self.reasoner if self.decoding_config.reasoning_backend
381
+ self.reasoner
382
+ if self.structured_outputs_config.reasoning_parser
383
and self.tokenizer else None,
384
),
385
))
@@ -772,10 +770,6 @@ class LLMEngine:
770
"""Gets the parallel configuration."""
771
return self.parallel_config
772
775
- def get_decoding_config(self) -> DecodingConfig:
776
- """Gets the decoding configuration."""
777
- return self.decoding_config
778
-
773
def get_scheduler_config(self) -> SchedulerConfig:
774
"""Gets the scheduler configuration."""
775
return self.scheduler_config
vllm/engine/protocol.py
+1
-6
@@ -6,7 +6,7 @@ from abc import ABC, abstractmethod
6
from typing import Any, AsyncGenerator, Iterable, Mapping, Optional, Union
7
8
from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
9
-from vllm.config import DecodingConfig, ModelConfig, VllmConfig
9
+from vllm.config import ModelConfig, VllmConfig
10
from vllm.core.scheduler import SchedulerOutputs
11
from vllm.inputs.data import PromptType, TokensPrompt
12
from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
@@ -248,11 +248,6 @@ class EngineClient(ABC):
248
"""Get the model configuration of the vLLM engine."""
249
...
250
251
- @abstractmethod
252
- async def get_decoding_config(self) -> DecodingConfig:
253
- """Get the decoding configuration of the vLLM engine."""
254
- ...
255
-
251
@abstractmethod
252
async def get_input_preprocessor(self) -> InputPreprocessor:
253
"""Get the input processor of the vLLM engine."""
vllm/entrypoints/llm.py
+23
-4
@@ -15,8 +15,8 @@ import vllm.envs as envs
15
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
16
BeamSearchSequence,
17
create_sort_beams_key_function)
18
-from vllm.config import (CompilationConfig, ModelDType, TokenizerMode,
19
- is_init_field)
18
+from vllm.config import (CompilationConfig, ModelDType,
19
+ StructuredOutputsConfig, TokenizerMode, is_init_field)
20
from vllm.engine.arg_utils import (ConvertOption, EngineArgs, HfOverrides,
21
PoolerConfig, RunnerOption)
22
from vllm.engine.llm_engine import LLMEngine
@@ -192,6 +192,8 @@ class LLM:
192
hf_overrides: Optional[HfOverrides] = None,
193
mm_processor_kwargs: Optional[dict[str, Any]] = None,
194
override_pooler_config: Optional[PoolerConfig] = None,
195
+ structured_outputs_config: Optional[Union[dict[
196
+ str, Any], StructuredOutputsConfig]] = None,
197
kv_cache_memory_bytes: Optional[int] = None,
198
compilation_config: Optional[Union[int, dict[str, Any],
199
CompilationConfig]] = None,
@@ -236,14 +238,30 @@ class LLM:
238
compilation_config_instance = CompilationConfig(
239
level=compilation_config)
240
elif isinstance(compilation_config, dict):
239
- predicate = lambda x: is_init_field(CompilationConfig, x[0])
241
compilation_config_instance = CompilationConfig(
241
- **dict(filter(predicate, compilation_config.items())))
242
+ **{
243
+ k: v
244
+ for k, v in compilation_config.items()
245
+ if is_init_field(CompilationConfig, k)
246
+ })
247
else:
248
compilation_config_instance = compilation_config
249
else:
250
compilation_config_instance = CompilationConfig()
251
252
+ if structured_outputs_config is not None:
253
+ if isinstance(structured_outputs_config, dict):
254
+ structured_outputs_instance = StructuredOutputsConfig(
255
+ **{
256
+ k: v
257
+ for k, v in structured_outputs_config.items()
258
+ if is_init_field(StructuredOutputsConfig, k)
259
+ })
260
+ else:
261
+ structured_outputs_instance = structured_outputs_config
262
+ else:
263
+ structured_outputs_instance = StructuredOutputsConfig()
264
+
265
engine_args = EngineArgs(
266
model=model,
267
runner=runner,
@@ -271,6 +289,7 @@ class LLM:
289
hf_overrides=hf_overrides,
290
mm_processor_kwargs=mm_processor_kwargs,
291
override_pooler_config=override_pooler_config,
292
+ structured_outputs_config=structured_outputs_instance,
293
compilation_config=compilation_config_instance,
294
logits_processors=logits_processors,
295
**kwargs,
vllm/entrypoints/openai/api_server.py
+5
-5
@@ -1678,7 +1678,7 @@ async def init_app_state(
1678
enable_auto_tools=args.enable_auto_tool_choice,
1679
tool_parser=args.tool_call_parser,
1680
tool_server=tool_server,
1681
- reasoning_parser=args.reasoning_parser,
1681
+ reasoning_parser=args.structured_outputs_config.reasoning_parser,
1682
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
1683
enable_force_include_usage=args.enable_force_include_usage,
1684
enable_log_outputs=args.enable_log_outputs,
@@ -1697,7 +1697,7 @@ async def init_app_state(
1697
exclude_tools_when_tool_choice_none=args.
1698
exclude_tools_when_tool_choice_none,
1699
tool_parser=args.tool_call_parser,
1700
- reasoning_parser=args.reasoning_parser,
1700
+ reasoning_parser=args.structured_outputs_config.reasoning_parser,
1701
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
1702
enable_force_include_usage=args.enable_force_include_usage,
1703
enable_log_outputs=args.enable_log_outputs,
@@ -1800,10 +1800,10 @@ def validate_api_server_args(args):
1800
f"(chose from {{ {','.join(valid_tool_parses)} }})")
1801
1802
valid_reasoning_parses = ReasoningParserManager.reasoning_parsers.keys()
1803
- if args.reasoning_parser \
1804
- and args.reasoning_parser not in valid_reasoning_parses:
1803
+ if ((reasoning_parser := args.structured_outputs_config.reasoning_parser)
1804
+ and reasoning_parser not in valid_reasoning_parses):
1805
raise KeyError(
1806
- f"invalid reasoning parser: {args.reasoning_parser} "
1806
+ f"invalid reasoning parser: {reasoning_parser} "
1807
f"(chose from {{ {','.join(valid_reasoning_parses)} }})")
1808
1809
vllm/entrypoints/openai/protocol.py
+72
-136
@@ -54,8 +54,8 @@ from vllm.entrypoints.score_utils import (ScoreContentPartParam,
54
from vllm.logger import init_logger
55
from vllm.logprobs import Logprob
56
from vllm.pooling_params import PoolingParams
57
-from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
58
- RequestOutputKind, SamplingParams)
57
+from vllm.sampling_params import (BeamSearchParams, RequestOutputKind,
58
+ SamplingParams, StructuredOutputsParams)
59
from vllm.utils import random_uuid, resolve_obj_by_qualname
60
61
logger = init_logger(__name__)
@@ -373,11 +373,12 @@ class ResponsesRequest(OpenAIBaseModel):
373
stop_token_ids = default_sampling_params.get("stop_token_ids")
374
375
# Structured output
376
- guided_decoding = None
376
+ structured_outputs = None
377
if self.text is not None and self.text.format is not None:
378
response_format = self.text.format
379
- if response_format.type == "json_schema":
380
- guided_decoding = GuidedDecodingParams.from_optional(
379
+ if (response_format.type == "json_schema"
380
+ and response_format.schema_ is not None):
381
+ structured_outputs = StructuredOutputsParams(
382
json=response_format.schema_)
383
elif response_format.type == "json_object":
384
raise NotImplementedError("json_object is not supported")
@@ -392,7 +393,7 @@ class ResponsesRequest(OpenAIBaseModel):
393
stop_token_ids=stop_token_ids,
394
output_kind=(RequestOutputKind.DELTA
395
if self.stream else RequestOutputKind.FINAL_ONLY),
395
- guided_decoding=guided_decoding,
396
+ structured_outputs=structured_outputs,
397
)
398
399
def is_include_output_logprobs(self) -> bool:
@@ -547,42 +548,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
548
default=None,
549
description=("Additional kwargs to pass to the HF processor."),
550
)
550
- guided_json: Optional[Union[str, dict, BaseModel]] = Field(
551
+ structured_outputs: Optional[StructuredOutputsParams] = Field(
552
default=None,
552
- description=("If specified, the output will follow the JSON schema."),
553
- )
554
- guided_regex: Optional[str] = Field(
555
- default=None,
556
- description=(
557
- "If specified, the output will follow the regex pattern."),
558
- )
559
- guided_choice: Optional[list[str]] = Field(
560
- default=None,
561
- description=(
562
- "If specified, the output will be exactly one of the choices."),
563
- )
564
- guided_grammar: Optional[str] = Field(
565
- default=None,
566
- description=(
567
- "If specified, the output will follow the context free grammar."),
568
- )
569
- structural_tag: Optional[str] = Field(
570
- default=None,
571
- description=(
572
- "If specified, the output will follow the structural tag schema."),
573
- )
574
- guided_decoding_backend: Optional[str] = Field(
575
- default=None,
576
- description=(
577
- "If specified, will override the default guided decoding backend "
578
- "of the server for this specific request. If set, must be either "
579
- "'outlines' / 'lm-format-enforcer'"),
580
- )
581
- guided_whitespace_pattern: Optional[str] = Field(
582
- default=None,
583
- description=(
584
- "If specified, will override the default whitespace pattern "
585
- "for guided json decoding."),
553
+ description="Additional kwargs for structured outputs",
554
)
555
priority: int = Field(
556
default=0,
@@ -701,31 +669,33 @@ class ChatCompletionRequest(OpenAIBaseModel):
669
if prompt_logprobs is None and self.echo:
670
prompt_logprobs = self.top_logprobs
671
704
- guided_json_object = None
705
- if self.response_format is not None:
706
- if self.response_format.type == "json_object":
707
- guided_json_object = True
708
- elif self.response_format.type == "json_schema":
709
- json_schema = self.response_format.json_schema
710
- assert json_schema is not None
711
- self.guided_json = json_schema.json_schema
712
- elif self.response_format.type == "structural_tag":
713
- structural_tag = self.response_format
714
- assert structural_tag is not None and isinstance(
715
- structural_tag, StructuralTagResponseFormat)
716
- s_tag_obj = structural_tag.model_dump(by_alias=True)
717
- self.structural_tag = json.dumps(s_tag_obj)
718
-
719
- guided_decoding = GuidedDecodingParams.from_optional(
720
- json=self._get_guided_json_from_tool() or self.guided_json,
721
- regex=self.guided_regex,
722
- choice=self.guided_choice,
723
- grammar=self.guided_grammar,
724
- json_object=guided_json_object,
725
- backend=self.guided_decoding_backend,
726
- whitespace_pattern=self.guided_whitespace_pattern,
727
- structural_tag=self.structural_tag,
728
- )
672
+ response_format = self.response_format
673
+ json_schema_from_tool = self._get_json_schema_from_tool()
674
+ if response_format is not None or json_schema_from_tool is not None:
675
+ # If structured outputs wasn't already enabled,
676
+ # we must enable it for these features to work
677
+ if self.structured_outputs is None:
678
+ self.structured_outputs = StructuredOutputsParams()
679
+
680
+ # Set structured output params for response format
681
+ if response_format is not None:
682
+ if response_format.type == "json_object":
683
+ self.structured_outputs.json_object = True
684
+ elif response_format.type == "json_schema":
685
+ json_schema = response_format.json_schema
686
+ assert json_schema is not None
687
+ self.structured_outputs.json = json_schema.json_schema
688
+ elif response_format.type == "structural_tag":
689
+ structural_tag = response_format
690
+ assert structural_tag is not None and isinstance(
691
+ structural_tag, StructuralTagResponseFormat)
692
+ s_tag_obj = structural_tag.model_dump(by_alias=True)
693
+ self.structured_outputs.structural_tag = json.dumps(
694
+ s_tag_obj)
695
+
696
+ # Set structured output params for tool calling
697
+ if json_schema_from_tool is not None:
698
+ self.structured_outputs.json = json_schema_from_tool
699
700
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
701
if self.kv_transfer_params:
@@ -757,15 +727,14 @@ class ChatCompletionRequest(OpenAIBaseModel):
727
truncate_prompt_tokens=self.truncate_prompt_tokens,
728
output_kind=RequestOutputKind.DELTA if self.stream \
729
else RequestOutputKind.FINAL_ONLY,
760
- guided_decoding=guided_decoding,
730
+ structured_outputs=self.structured_outputs,
731
logit_bias=self.logit_bias,
762
- bad_words= self.bad_words,
732
+ bad_words=self.bad_words,
733
allowed_token_ids=self.allowed_token_ids,
734
extra_args=extra_args or None,
735
)
736
767
- def _get_guided_json_from_tool(
768
- self) -> Optional[Union[str, dict, BaseModel]]:
737
+ def _get_json_schema_from_tool(self) -> Optional[Union[str, dict]]:
738
# user has chosen to not use any tool
739
if self.tool_choice == "none" or self.tools is None:
740
return None
@@ -875,28 +844,31 @@ class ChatCompletionRequest(OpenAIBaseModel):
844
845
@model_validator(mode="before")
846
@classmethod
878
- def check_guided_decoding_count(cls, data):
847
+ def check_structured_outputs_count(cls, data):
848
if isinstance(data, ValueError):
849
raise data
850
882
- guide_count = sum([
883
- "guided_json" in data and data["guided_json"] is not None,
884
- "guided_regex" in data and data["guided_regex"] is not None,
885
- "guided_choice" in data and data["guided_choice"] is not None
886
- ])
887
- # you can only use one kind of guided decoding
888
- if guide_count > 1:
851
+ if "structured_outputs" not in data:
852
+ return data
853
+
854
+ structured_outputs_kwargs = data['structured_outputs']
855
+ count = sum(
856
+ structured_outputs_kwargs.get(k) is not None
857
+ for k in ("json", "regex", "choice"))
858
+ # you can only use one kind of constraints for structured outputs
859
+ if count > 1:
860
raise ValueError(
890
- "You can only use one kind of guided decoding "
891
- "('guided_json', 'guided_regex' or 'guided_choice').")
892
- # you can only either use guided decoding or tools, not both
893
- if guide_count > 1 and data.get("tool_choice", "none") not in (
861
+ "You can only use one kind of constraints for structured "
862
+ "outputs ('json', 'regex' or 'choice').")
863
+ # you can only either use structured outputs or tools, not both
864
+ if count > 1 and data.get("tool_choice", "none") not in (
865
"none",
866
"auto",
867
"required",
868
):
869
raise ValueError(
899
- "You can only either use guided decoding or tools, not both.")
870
+ "You can only either use constraints for structured outputs "
871
+ "or tools, not both.")
872
return data
873
874
@model_validator(mode="before")
@@ -1049,37 +1021,9 @@ class CompletionRequest(OpenAIBaseModel):
1021
", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
1022
),
1023
)
1052
- guided_json: Optional[Union[str, dict, BaseModel]] = Field(
1053
- default=None,
1054
- description="If specified, the output will follow the JSON schema.",
1055
- )
1056
- guided_regex: Optional[str] = Field(
1057
- default=None,
1058
- description=(
1059
- "If specified, the output will follow the regex pattern."),
1060
- )
1061
- guided_choice: Optional[list[str]] = Field(
1024
+ structured_outputs: Optional[StructuredOutputsParams] = Field(
1025
default=None,
1063
- description=(
1064
- "If specified, the output will be exactly one of the choices."),
1065
- )
1066
- guided_grammar: Optional[str] = Field(
1067
- default=None,
1068
- description=(
1069
- "If specified, the output will follow the context free grammar."),
1070
- )
1071
- guided_decoding_backend: Optional[str] = Field(
1072
- default=None,
1073
- description=(
1074
- "If specified, will override the default guided decoding backend "
1075
- "of the server for this specific request. If set, must be one of "
1076
- "'outlines' / 'lm-format-enforcer'"),
1077
- )
1078
- guided_whitespace_pattern: Optional[str] = Field(
1079
- default=None,
1080
- description=(
1081
- "If specified, will override the default whitespace pattern "
1082
- "for guided json decoding."),
1026
+ description="Additional kwargs for structured outputs",
1027
)
1028
priority: int = Field(
1029
default=0,
@@ -1210,20 +1154,10 @@ class CompletionRequest(OpenAIBaseModel):
1154
1155
echo_without_generation = self.echo and self.max_tokens == 0
1156
1213
- guided_json_object = None
1214
- if (self.response_format is not None
1157
+ if (self.structured_outputs is not None
1158
+ and self.response_format is not None
1159
and self.response_format.type == "json_object"):
1216
- guided_json_object = True
1217
-
1218
- guided_decoding = GuidedDecodingParams.from_optional(
1219
- json=self.guided_json,
1220
- regex=self.guided_regex,
1221
- choice=self.guided_choice,
1222
- grammar=self.guided_grammar,
1223
- json_object=guided_json_object,
1224
- backend=self.guided_decoding_backend,
1225
- whitespace_pattern=self.guided_whitespace_pattern,
1226
- )
1160
+ self.structured_outputs.json_object = True
1161
1162
extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
1163
if self.kv_transfer_params:
@@ -1255,7 +1189,7 @@ class CompletionRequest(OpenAIBaseModel):
1189
truncate_prompt_tokens=self.truncate_prompt_tokens,
1190
output_kind=RequestOutputKind.DELTA if self.stream \
1191
else RequestOutputKind.FINAL_ONLY,
1258
- guided_decoding=guided_decoding,
1192
+ structured_outputs=self.structured_outputs,
1193
logit_bias=self.logit_bias,
1194
allowed_token_ids=self.allowed_token_ids,
1195
extra_args=extra_args or None,
@@ -1263,16 +1197,18 @@ class CompletionRequest(OpenAIBaseModel):
1197
1198
@model_validator(mode="before")
1199
@classmethod
1266
- def check_guided_decoding_count(cls, data):
1267
- guide_count = sum([
1268
- "guided_json" in data and data["guided_json"] is not None,
1269
- "guided_regex" in data and data["guided_regex"] is not None,
1270
- "guided_choice" in data and data["guided_choice"] is not None
1271
- ])
1272
- if guide_count > 1:
1200
+ def check_structured_outputs_count(cls, data):
1201
+ if "structured_outputs" not in data:
1202
+ return data
1203
+
1204
+ structured_outputs_kwargs = data['structured_outputs']
1205
+ count = sum(
1206
+ structured_outputs_kwargs.get(k) is not None
1207
+ for k in ("json", "regex", "choice"))
1208
+ if count > 1:
1209
raise ValueError(
1274
- "You can only use one kind of guided decoding "
1275
- "('guided_json', 'guided_regex' or 'guided_choice').")
1210
+ "You can only use one kind of constraints for structured "
1211
+ "outputs ('json', 'regex' or 'choice').")
1212
return data
1213
1214
@model_validator(mode="before")
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -993,7 +993,7 @@ class OpenAIServingChat(OpenAIServing):
993
# check to make sure we haven't "forgotten" to stream
994
# any tokens that were generated but previously
995
# matched by partial json parsing
996
- # only happens if we are NOT using guided decoding
996
+ # only happens if we are NOT using structured outputs
997
auto_tools_called = False
998
if tool_parser:
999
auto_tools_called = len(
vllm/model_executor/models/config.py
+3
-3
@@ -262,9 +262,9 @@ class GptOssForCausalLMConfig(VerifyAndUpdateConfig):
262
263
@staticmethod
264
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
265
- decoding_config = vllm_config.decoding_config
266
- if decoding_config.reasoning_backend == "":
267
- decoding_config.reasoning_backend = "openai_gptoss"
265
+ structured_outputs_config = vllm_config.structured_outputs_config
266
+ if structured_outputs_config.reasoning_parser == "":
267
+ structured_outputs_config.reasoning_parser = "openai_gptoss"
268
269
# Increase the max capture size from 512 to 1024 for performance.
270
# NOTE(woosuk): This will increase the number of CUDA graphs
vllm/sampling_params.py
+18
-44
@@ -2,13 +2,13 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""Sampling parameters for text generation."""
4
import copy
5
-from dataclasses import dataclass
5
+from dataclasses import field
6
from enum import Enum, IntEnum
7
from functools import cached_property
8
from typing import Annotated, Any, Optional, Union
9
10
import msgspec
11
-from pydantic import BaseModel
11
+from pydantic.dataclasses import dataclass
12
13
from vllm.logger import init_logger
14
from vllm.logits_process import LogitsProcessor
@@ -28,60 +28,35 @@ class SamplingType(IntEnum):
28
29
# maybe make msgspec?
30
@dataclass
31
-class GuidedDecodingParams:
32
- """One of these fields will be used to build a logit processor."""
31
+class StructuredOutputsParams:
32
+ # One of these fields will be used to build a logit processor.
33
json: Optional[Union[str, dict]] = None
34
regex: Optional[str] = None
35
choice: Optional[list[str]] = None
36
grammar: Optional[str] = None
37
json_object: Optional[bool] = None
38
- """These are other options that can be set"""
39
- backend: Optional[str] = None
40
- backend_was_auto: bool = False
38
+ # These are other options that can be set.
39
disable_fallback: bool = False
40
disable_any_whitespace: bool = False
41
disable_additional_properties: bool = False
42
whitespace_pattern: Optional[str] = None
43
structural_tag: Optional[str] = None
44
47
- @staticmethod
48
- def from_optional(
49
- json: Optional[Union[dict, BaseModel, str]] = None,
50
- regex: Optional[str] = None,
51
- choice: Optional[list[str]] = None,
52
- grammar: Optional[str] = None,
53
- json_object: Optional[bool] = None,
54
- backend: Optional[str] = None,
55
- whitespace_pattern: Optional[str] = None,
56
- structural_tag: Optional[str] = None,
57
- ) -> Optional["GuidedDecodingParams"]:
58
- if all(arg is None for arg in (json, regex, choice, grammar,
59
- json_object, structural_tag)):
60
- return None
61
- # Extract json schemas from pydantic models
62
- if isinstance(json, (BaseModel, type(BaseModel))):
63
- json = json.model_json_schema()
64
- return GuidedDecodingParams(
65
- json=json,
66
- regex=regex,
67
- choice=choice,
68
- grammar=grammar,
69
- json_object=json_object,
70
- backend=backend,
71
- whitespace_pattern=whitespace_pattern,
72
- structural_tag=structural_tag,
73
- )
45
+ _backend: Optional[str] = field(default=None, init=False)
46
+ """CAUTION: Should only be set by Processor._validate_structured_output"""
47
+ _backend_was_auto: bool = field(default=False, init=False)
48
+ """CAUTION: Should only be set by Processor._validate_structured_output"""
49
50
def __post_init__(self):
51
"""Validate that some fields are mutually exclusive."""
77
- guide_count = sum([
52
+ count = sum([
53
self.json is not None, self.regex is not None, self.choice
54
is not None, self.grammar is not None, self.json_object is not None
55
])
81
- if guide_count > 1:
56
+ if count > 1:
57
raise ValueError(
83
- "You can only use one kind of guided decoding but multiple are "
84
- f"specified: {self.__dict__}")
58
+ "You can only use one kind of structured outputs constraint "
59
+ f"but multiple are specified: {self.__dict__}")
60
61
62
class RequestOutputKind(Enum):
@@ -196,9 +171,8 @@ class SamplingParams(
171
_all_stop_token_ids: set[int] = msgspec.field(default_factory=set)
172
173
# Fields used to construct logits processors
199
- guided_decoding: Optional[GuidedDecodingParams] = None
200
- """If provided, the engine will construct a guided decoding logits
201
- processor from these parameters."""
174
+ structured_outputs: Optional[StructuredOutputsParams] = None
175
+ """Parameters for configuring structured outputs."""
176
logit_bias: Optional[dict[int, float]] = None
177
"""If provided, the engine will construct a logits processor that applies
178
these logit biases."""
@@ -246,7 +220,7 @@ class SamplingParams(
220
msgspec.Meta(
221
ge=-1)]] = None,
222
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
249
- guided_decoding: Optional[GuidedDecodingParams] = None,
223
+ structured_outputs: Optional[StructuredOutputsParams] = None,
224
logit_bias: Optional[Union[dict[int, float], dict[str, float]]] = None,
225
allowed_token_ids: Optional[list[int]] = None,
226
extra_args: Optional[dict[str, Any]] = None,
@@ -288,7 +262,7 @@ class SamplingParams(
262
logits_processors=logits_processors,
263
truncate_prompt_tokens=truncate_prompt_tokens,
264
output_kind=output_kind,
291
- guided_decoding=guided_decoding,
265
+ structured_outputs=structured_outputs,
266
logit_bias=logit_bias,
267
allowed_token_ids=allowed_token_ids,
268
extra_args=extra_args,
@@ -559,7 +533,7 @@ class SamplingParams(
533
"spaces_between_special_tokens="
534
f"{self.spaces_between_special_tokens}, "
535
f"truncate_prompt_tokens={self.truncate_prompt_tokens}, "
562
- f"guided_decoding={self.guided_decoding}, "
536
+ f"structured_outputs={self.structured_outputs}, "
537
f"extra_args={self.extra_args})")
538
539
vllm/transformers_utils/tokenizers/mistral.py
+1
-4
@@ -274,7 +274,7 @@ class MistralTokenizer(TokenizerBase):
274
return tokenizer_file
275
276
# the following attributes are set to fit vLLM's design and are used
277
- # by the guided structured output backends.
277
+ # by the structured output backends.
278
@property
279
def all_special_tokens_extended(self) -> list[str]:
280
from mistral_common.tokens.tokenizers.base import SpecialTokens
@@ -463,9 +463,6 @@ class MistralTokenizer(TokenizerBase):
463
464
return decoded
465
466
- # WARN: Outlines logits processors can overwrite this method.
467
- # See: guided_decoding/outlines_logits_processors.py::_adapt_tokenizer
468
- # for more.
466
def decode(self,
467
ids: Union[list[int], int],
468
skip_special_tokens: bool = True) -> str:
vllm/v1/engine/async_llm.py
-3
@@ -588,9 +588,6 @@ class AsyncLLM(EngineClient):
588
async def get_model_config(self) -> ModelConfig:
589
return self.model_config
590
591
- async def get_decoding_config(self):
592
- raise ValueError("Not Supported on V1 yet.")
593
-
591
async def get_input_preprocessor(self) -> InputPreprocessor:
592
return self.processor.input_preprocessor
593
vllm/v1/engine/processor.py
+28
-29
@@ -45,7 +45,7 @@ class Processor:
45
self.model_config = vllm_config.model_config
46
self.cache_config = vllm_config.cache_config
47
self.lora_config = vllm_config.lora_config
48
- self.decoding_config = vllm_config.decoding_config
48
+ self.structured_outputs_config = vllm_config.structured_outputs_config
49
self.tokenizer = tokenizer
50
51
self.generation_config_fields = (
@@ -219,58 +219,57 @@ class Processor:
219
"[lora_path]` to use the LoRA tokenizer.")
220
221
def _validate_structured_output(self, params: SamplingParams) -> None:
222
- if not params.guided_decoding or not self.decoding_config:
222
+ if not params.structured_outputs or not self.structured_outputs_config:
223
return
224
225
- if self.model_config.skip_tokenizer_init and params.guided_decoding:
225
+ if self.model_config.skip_tokenizer_init and params.structured_outputs:
226
raise ValueError(
227
"Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501
228
)
229
230
- engine_level_backend = self.decoding_config.backend
231
- if params.guided_decoding.backend:
232
- # Request-level backend selection is not supported in V1.
230
+ backend = self.structured_outputs_config.backend
231
+ if _backend := params.structured_outputs._backend:
232
+ # Request-level backend selection is not supported.
233
# The values may differ if `params` is reused and was set
234
# to a specific backend based on `auto` behavior in a previous
235
# request. We remember that it was set as a result of `auto`
236
- # using the `_auto` option set on the backend in the params.
237
- if (params.guided_decoding.backend != engine_level_backend
238
- and not (engine_level_backend == "auto"
239
- and params.guided_decoding.backend_was_auto)):
236
+ # using the `_backend_was_auto` field set in the params.
237
+ if (backend != _backend
238
+ and not (backend == "auto"
239
+ and params.structured_outputs._backend_was_auto)):
240
raise ValueError(
241
- "Request-level structured output backend selection is no "
242
- "longer supported. The request specified "
243
- f"'{params.guided_decoding.backend}', but vLLM was "
244
- f"initialised with '{engine_level_backend}'. This error "
245
- "can be resolved by removing backend selection from the "
246
- "request.")
241
+ "Request-level structured output backend selection is not "
242
+ f"supported. The request specified '{_backend}', but vLLM "
243
+ f"was initialised with '{backend}'. This error can be "
244
+ "resolved by removing '_backend' from the request.")
245
else:
248
- params.guided_decoding.backend = engine_level_backend
246
+ params.structured_outputs._backend = backend
247
248
# Request content validation
251
- if (isinstance(params.guided_decoding.choice, list)
252
- and not params.guided_decoding.choice):
249
+ if (isinstance(params.structured_outputs.choice, list)
250
+ and not params.structured_outputs.choice):
251
# It is invalid for choice to be an empty list
254
- raise ValueError(f"Choice '{params.guided_decoding.choice}' "
255
- "cannot be an empty list")
252
+ raise ValueError(
253
+ f"Choice '{params.structured_outputs.choice}' cannot be an empty list" # noqa: E501
254
+ )
255
257
- if engine_level_backend.startswith("xgrammar"):
256
+ if backend.startswith("xgrammar"):
257
# xgrammar with no fallback
258
validate_xgrammar_grammar(params)
260
- elif engine_level_backend.startswith("guidance"):
259
+ elif backend.startswith("guidance"):
260
# TODO: ideally we would have the LLTokenizer here as Lark syntax
261
# allows <|special_token|> and similar, see
262
# https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
263
# Without tokenizer these are disallowed in grammars.
264
validate_guidance_grammar(params, tokenizer=None)
266
- elif engine_level_backend == "outlines":
265
+ elif backend == "outlines":
266
# outlines backend
267
validate_structured_output_request_outlines(params)
269
- elif engine_level_backend == "lm-format-enforcer":
268
+ elif backend == "lm-format-enforcer":
269
# lm format enforcer backend
270
validate_structured_output_request_lm_format_enforcer(params)
271
else:
273
- # NOTE: engine_level_backend must be "auto" here, because we have
272
+ # NOTE: backend must be "auto" here, because we have
273
# checked supported_backends above.
274
# In this mode, we set opinionated defaults based on what we think
275
# will satisfy the most use cases without having to worry about
@@ -278,15 +277,15 @@ class Processor:
277
# other setting where a specific backend was specified.
278
try:
279
validate_xgrammar_grammar(params)
281
- params.guided_decoding.backend = "xgrammar"
280
+ params.structured_outputs._backend = "xgrammar"
281
except ValueError:
282
# The request either failed validation
283
# or includes some jsonschema feature(s) that
284
# are not supported in xgrammar. Fall back to guidance.
285
validate_guidance_grammar(params, tokenizer=None)
287
- params.guided_decoding.backend = "guidance"
286
+ params.structured_outputs._backend = "guidance"
287
# Remember that this backend was set automatically
289
- params.guided_decoding.backend_was_auto = True
288
+ params.structured_outputs._backend_was_auto = True
289
290
def _maybe_build_mm_uuids(
291
self,
vllm/v1/request.py
+1
-1
@@ -67,7 +67,7 @@ class Request:
67
# Generative models.
68
assert sampling_params.max_tokens is not None
69
self.max_tokens = sampling_params.max_tokens
70
- if sampling_params.guided_decoding is not None:
70
+ if sampling_params.structured_outputs is not None:
71
self.status = RequestStatus.WAITING_FOR_FSM
72
self.use_structured_output = True
73
vllm/v1/structured_output/__init__.py
+7
-6
@@ -61,11 +61,11 @@ class StructuredOutputManager:
61
self.executor = ThreadPoolExecutor(max_workers=max_workers)
62
self.tokenizer = init_tokenizer_from_configs(
63
model_config=self.vllm_config.model_config)
64
- reasoning_backend = \
65
- self.vllm_config.decoding_config.reasoning_backend
66
- if reasoning_backend:
64
+ reasoning_parser = \
65
+ self.vllm_config.structured_outputs_config.reasoning_parser
66
+ if reasoning_parser:
67
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
68
- reasoning_backend)
68
+ reasoning_parser)
69
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
70
71
def grammar_init(self, request: Request) -> None:
@@ -74,15 +74,16 @@ class StructuredOutputManager:
74
75
if TYPE_CHECKING:
76
assert request.sampling_params is not None and \
77
- request.sampling_params.guided_decoding is not None
77
+ request.sampling_params.structured_outputs is not None
78
79
# Initialize the backend the first time it is needed.
80
#
81
# NOTE: We only support a single backend. We do NOT support different
82
# backends on a per-request basis in V1 (for now, anyway...).
83
+ # _backend is set in Processor._validate_structured_output
84
if self.backend is None:
85
assert request.sampling_params is not None
85
- backend = request.sampling_params.guided_decoding.backend
86
+ backend = request.sampling_params.structured_outputs._backend
87
vocab_size = self.vllm_config.model_config.get_vocab_size()
88
if backend == "xgrammar":
89
self.backend = XgrammarBackend(
vllm/v1/structured_output/backend_guidance.py
+2
-2
@@ -60,9 +60,9 @@ class GuidanceBackend(StructuredOutputBackend):
60
61
def __post_init__(self):
62
self.disable_any_whitespace = \
63
- self.vllm_config.decoding_config.disable_any_whitespace
63
+ self.vllm_config.structured_outputs_config.disable_any_whitespace
64
self.disable_additional_properties = \
65
- self.vllm_config.decoding_config.disable_additional_properties
65
+ self.vllm_config.structured_outputs_config.disable_additional_properties
66
67
self.ll_tokenizer = llguidance_hf.from_tokenizer(
68
self.tokenizer, self.vocab_size)
vllm/v1/structured_output/backend_lm_format_enforcer.py
+11
-11
@@ -138,30 +138,30 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
138
139
def validate_structured_output_request_lm_format_enforcer(
140
params: SamplingParams):
141
- if params.guided_decoding is None:
141
+ if params.structured_outputs is None:
142
return
143
144
- gd_params = params.guided_decoding
144
+ so_params = params.structured_outputs
145
146
- if gd_params.regex:
146
+ if so_params.regex:
147
return
148
- elif gd_params.json:
149
- if isinstance(gd_params.json, str):
148
+ elif so_params.json:
149
+ if isinstance(so_params.json, str):
150
try:
151
# make sure schema is valid json
152
- json.loads(gd_params.json)
152
+ json.loads(so_params.json)
153
except json.JSONDecodeError as e:
154
raise ValueError("Invalid JSON grammar specification.") from e
155
else:
156
try:
157
- json.dumps(gd_params.json)
157
+ json.dumps(so_params.json)
158
except Exception as e:
159
raise ValueError(
160
- f"Error serializing guided decoding jsonschema: {e}"
160
+ f"Error serializing structured outputs jsonschema: {e}"
161
) from e
162
return
163
- elif gd_params.choice:
163
+ elif so_params.choice:
164
return
165
- elif gd_params.grammar:
166
- raise ValueError("LM Format Enforcer guided decoding backend "
165
+ elif so_params.grammar:
166
+ raise ValueError("LM Format Enforcer structured outputs backend "
167
"does not support grammar specifications")
vllm/v1/structured_output/backend_outlines.py
+16
-16
@@ -158,36 +158,36 @@ class OutlinesGrammar(StructuredOutputGrammar):
158
159
160
def validate_structured_output_request_outlines(params: SamplingParams):
161
- if params.guided_decoding is None:
161
+ if params.structured_outputs is None:
162
return
163
164
- gd_params = params.guided_decoding
164
+ so_params = params.structured_outputs
165
166
- if gd_params.regex:
167
- validate_regex_is_buildable(gd_params.regex)
168
- elif gd_params.json:
169
- if isinstance(gd_params.json, str):
166
+ if so_params.regex:
167
+ validate_regex_is_buildable(so_params.regex)
168
+ elif so_params.json:
169
+ if isinstance(so_params.json, str):
170
try:
171
# make sure schema is valid json
172
- json.loads(gd_params.json)
173
- schema = gd_params.json
172
+ json.loads(so_params.json)
173
+ schema = so_params.json
174
except json.JSONDecodeError as e:
175
raise ValueError("Invalid JSON grammar specification.") from e
176
else:
177
try:
178
- schema = json.dumps(gd_params.json)
178
+ schema = json.dumps(so_params.json)
179
except Exception as e:
180
raise ValueError(
181
- f"Error serializing guided decoding jsonschema: {e}"
181
+ f"Error serializing structured outputs jsonschema: {e}"
182
) from e
183
pattern = json_schema.build_regex_from_schema(schema)
184
validate_regex_is_buildable(pattern)
185
- elif gd_params.choice:
186
- choices = [regex_escape(str(choice)) for choice in gd_params.choice]
185
+ elif so_params.choice:
186
+ choices = [regex_escape(str(choice)) for choice in so_params.choice]
187
regex = "(" + "|".join(choices) + ")"
188
validate_regex_is_buildable(regex)
189
- elif gd_params.grammar:
190
- raise ValueError("Outlines guided decoding backend "
189
+ elif so_params.grammar:
190
+ raise ValueError("Outlines structured outputs backend "
191
"does not support grammar specifications")
192
193
@@ -306,7 +306,7 @@ def validate_regex_is_buildable(pattern: str) -> None:
306
_check_unsupported(parsed)
307
except ValueError as e:
308
raise ValueError(
309
- f"Regex uses unsupported feature for guided decoding: {e}. "
309
+ f"Regex uses unsupported feature for structured outputs: {e}. "
310
"Only basic matching constructs are supported—lookarounds, "
311
"backreferences, and unicode boundaries are not.") from e
312
@@ -315,6 +315,6 @@ def validate_regex_is_buildable(pattern: str) -> None:
315
"Regex does not have a anchored universal start state"
316
"This means that the Regex uses anchors (^) or look-arounds "
317
"in a way which requires context before any token is matched."
318
- "Guided decoding needs regexes that can match without needing "
318
+ "structured outputs needs regexes that can match without needing "
319
"that context. Try rewriting the pattern without using these "
320
f"constructs. Pattern:\n{pattern}")
vllm/v1/structured_output/backend_xgrammar.py
+19
-19
@@ -34,7 +34,7 @@ class XgrammarBackend(StructuredOutputBackend):
34
35
def __post_init__(self):
36
self.disable_any_whitespace = \
37
- self.vllm_config.decoding_config.disable_any_whitespace
37
+ self.vllm_config.structured_outputs_config.disable_any_whitespace
38
39
if isinstance(self.tokenizer, MistralTokenizer):
40
# NOTE: ideally, xgrammar should handle this accordingly.
@@ -248,37 +248,37 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
248
249
Raises ValueError if the request is not supported.
250
"""
251
- if sampling_params.guided_decoding is None:
251
+ if sampling_params.structured_outputs is None:
252
return
253
254
- gd_params = sampling_params.guided_decoding
254
+ so_params = sampling_params.structured_outputs
255
256
- if gd_params.regex:
256
+ if so_params.regex:
257
try:
258
- xgr.Grammar.from_regex(gd_params.regex)
258
+ xgr.Grammar.from_regex(so_params.regex)
259
except Exception as err:
260
raise ValueError("Failed to transform regex into a grammar: "
261
f"{err}") from err
262
263
- if gd_params.choice:
264
- choice_grammar = choice_as_grammar(gd_params.choice)
263
+ if so_params.choice:
264
+ choice_grammar = choice_as_grammar(so_params.choice)
265
try:
266
xgr.Grammar.from_ebnf(choice_grammar)
267
except Exception as err:
268
raise ValueError("Failed to transform choices into a grammar: "
269
"{err}") from err
270
- gd_params.choice = None
271
- gd_params.grammar = choice_grammar
270
+ so_params.choice = None
271
+ so_params.grammar = choice_grammar
272
return
273
274
- if gd_params.json:
275
- if isinstance(gd_params.json, str):
274
+ if so_params.json:
275
+ if isinstance(so_params.json, str):
276
try:
277
- schema = json.loads(gd_params.json)
277
+ schema = json.loads(so_params.json)
278
except json.JSONDecodeError as e:
279
raise ValueError("Invalid JSON grammar specification.") from e
280
else:
281
- schema = gd_params.json
281
+ schema = so_params.json
282
283
try:
284
xgr.Grammar.from_json_schema(schema)
@@ -291,11 +291,11 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
291
"supported by xgrammar.")
292
return
293
294
- if gd_params.grammar:
295
- if grammar_is_likely_lark(gd_params.grammar):
294
+ if so_params.grammar:
295
+ if grammar_is_likely_lark(so_params.grammar):
296
# xgrammar supports EBNF grammars only
297
try:
298
- gd_params.grammar = convert_lark_to_ebnf(gd_params.grammar)
298
+ so_params.grammar = convert_lark_to_ebnf(so_params.grammar)
299
except ValueError as e:
300
raise ValueError(
301
"Failed to convert the grammar from Lark to EBNF. ") from e
@@ -303,14 +303,14 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
303
# Test parsing EBNF grammar, possibly already converted from Lark
304
try:
305
# parse the grammar, but we aren't compiling it.
306
- xgr.Grammar.from_ebnf(gd_params.grammar)
306
+ xgr.Grammar.from_ebnf(so_params.grammar)
307
except Exception as e:
308
raise ValueError("Invalid grammar specification.") from e
309
return
310
311
- if gd_params.structural_tag:
311
+ if so_params.structural_tag:
312
try:
313
- s_tag = json.loads(gd_params.structural_tag)
313
+ s_tag = json.loads(so_params.structural_tag)
314
tags = [
315
xgr.StructuralTagItem(
316
begin=s["begin"],
vllm/v1/structured_output/request.py
+1
-1
@@ -60,7 +60,7 @@ class StructuredOutputRequest:
60
61
def get_structured_output_key(
62
sampling_params: SamplingParams) -> StructuredOutputKey:
63
- params = sampling_params.guided_decoding
63
+ params = sampling_params.structured_outputs
64
assert params is not None, "params can't be None."
65
if params.json is not None:
66
if not isinstance(params.json, str):