A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[V1] Update structured output (#16812)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Apr 24, 2025 at 14:57 UTC
db2f8d915ccfb4445bbac4e01ad2222260903868
4 files changed
+27
-27
benchmarks/benchmark_serving_structured_output.py
+6
-6
@@ -150,17 +150,17 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,
150
151
elif args.dataset == "grammar":
152
schema = """
153
- ?start: select_statement
153
+ root ::= select_statement
154
155
- ?select_statement: "SELECT " column_list " FROM " table_name
155
+ select_statement ::= "SELECT " column " from " table " where " condition
156
157
- ?column_list: column_name ("," column_name)*
157
+ column ::= "col_1 " | "col_2 "
158
159
- ?table_name: identifier
159
+ table ::= "table_1 " | "table_2 "
160
161
- ?column_name: identifier
161
+ condition ::= column "= " number
162
163
- ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
163
+ number ::= "1 " | "2 "
164
"""
165
prompt = "Generate an SQL query to show the 'username' \
166
and 'email' from the 'users' table."
docs/source/features/structured_outputs.md
+7
-7
@@ -50,7 +50,7 @@ completion = client.chat.completions.create(
50
"content": "Generate an example email address for Alan Turing, who works in Enigma. End in .com and new line. Example result: alan.turing@enigma.com\n",
51
}
52
],
53
- extra_body={"guided_regex": "\w+@\w+\.com\n", "stop": ["\n"]},
53
+ extra_body={"guided_regex": r"\w+@\w+\.com\n", "stop": ["\n"]},
54
)
55
print(completion.choices[0].message.content)
56
```
@@ -105,17 +105,17 @@ It works by using a context free EBNF grammar, which for example we can use to d
105
106
```python
107
simplified_sql_grammar = """
108
- ?start: select_statement
108
+ root ::= select_statement
109
110
- ?select_statement: "SELECT " column_list " FROM " table_name
110
+ select_statement ::= "SELECT " column " from " table " where " condition
111
112
- ?column_list: column_name ("," column_name)*
112
+ column ::= "col_1 " | "col_2 "
113
114
- ?table_name: identifier
114
+ table ::= "table_1 " | "table_2 "
115
116
- ?column_name: identifier
116
+ condition ::= column "= " number
117
118
- ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
118
+ number ::= "1 " | "2 "
119
"""
120
121
completion = client.chat.completions.create(
examples/online_serving/openai_chat_completion_structured_outputs.py
+8
-8
@@ -33,7 +33,7 @@ completion = client.chat.completions.create(
33
"content": prompt,
34
}],
35
extra_body={
36
- "guided_regex": "\w+@\w+\.com\n",
36
+ "guided_regex": r"\w+@\w+\.com\n",
37
"stop": ["\n"]
38
},
39
)
@@ -70,17 +70,17 @@ print(completion.choices[0].message.content)
70
71
# Guided decoding by Grammar
72
simplified_sql_grammar = """
73
- ?start: select_statement
73
+ root ::= select_statement
74
75
- ?select_statement: "SELECT " column_list " FROM " table_name
75
+ select_statement ::= "SELECT " column " from " table " where " condition
76
77
- ?column_list: column_name ("," column_name)*
77
+ column ::= "col_1 " | "col_2 "
78
79
- ?table_name: identifier
79
+ table ::= "table_1 " | "table_2 "
80
81
- ?column_name: identifier
81
+ condition ::= column "= " number
82
83
- ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
83
+ number ::= "1 " | "2 "
84
"""
85
86
prompt = ("Generate an SQL query to show the 'username' and 'email'"
@@ -110,7 +110,7 @@ try:
110
"content": prompt,
111
}],
112
extra_body={
113
- "guided_regex": "\w+@\w+\.com\n",
113
+ "guided_regex": r"\w+@\w+\.com\n",
114
"stop": ["\n"],
115
"guided_decoding_backend": "xgrammar:no-fallback"
116
},
examples/online_serving/openai_chat_completion_structured_outputs_with_reasoning.py
+6
-6
@@ -101,17 +101,17 @@ print("content: ", completion.choices[0].message.content)
101
102
# Guided decoding by Grammar
103
simplified_sql_grammar = """
104
- ?start: select_statement
104
+ root ::= select_statement
105
106
- ?select_statement: "SELECT " column_list " FROM " table_name
106
+ select_statement ::= "SELECT " column " from " table " where " condition
107
108
- ?column_list: column_name ("," column_name)*
108
+ column ::= "col_1 " | "col_2 "
109
110
- ?table_name: identifier
110
+ table ::= "table_1 " | "table_2 "
111
112
- ?column_name: identifier
112
+ condition ::= column "= " number
113
114
- ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
114
+ number ::= "1 " | "2 "
115
"""
116
117
# This may be very slow https://github.com/vllm-project/vllm/issues/12122