A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Misc] refactor example series - structured outputs (#17040)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Apr 24, 2025 at 22:49 UTC
1bcbcbf57417e6fa0427bc60f6acd11b86cb857f
2 files changed
+209
-143
examples/online_serving/openai_chat_completion_structured_outputs.py
+117
-77
@@ -1,43 +1,49 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+"""
3
+To run this example, you need to start the vLLM server:
4
+
5
+```bash
6
+vllm serve Qwen/Qwen2.5-3B-Instruct
7
+```
8
+"""
9
10
from enum import Enum
11
12
from openai import BadRequestError, OpenAI
13
from pydantic import BaseModel
14
8
-client = OpenAI(
9
- base_url="http://localhost:8000/v1",
10
- api_key="-",
11
-)
15
16
# Guided decoding by Choice (list of possible options)
14
-completion = client.chat.completions.create(
15
- model="Qwen/Qwen2.5-3B-Instruct",
16
- messages=[{
17
- "role": "user",
18
- "content": "Classify this sentiment: vLLM is wonderful!"
19
- }],
20
- extra_body={"guided_choice": ["positive", "negative"]},
21
-)
22
-print(completion.choices[0].message.content)
17
+def guided_choice_completion(client: OpenAI, model: str):
18
+ completion = client.chat.completions.create(
19
+ model=model,
20
+ messages=[{
21
+ "role": "user",
22
+ "content": "Classify this sentiment: vLLM is wonderful!"
23
+ }],
24
+ extra_body={"guided_choice": ["positive", "negative"]},
25
+ )
26
+ return completion.choices[0].message.content
27
+
28
29
# Guided decoding by Regex
25
-prompt = ("Generate an email address for Alan Turing, who works in Enigma."
26
- "End in .com and new line. Example result:"
27
- "alan.turing@enigma.com\n")
28
-
29
-completion = client.chat.completions.create(
30
- model="Qwen/Qwen2.5-3B-Instruct",
31
- messages=[{
32
- "role": "user",
33
- "content": prompt,
34
- }],
35
- extra_body={
36
- "guided_regex": r"\w+@\w+\.com\n",
37
- "stop": ["\n"]
38
- },
39
-)
40
-print(completion.choices[0].message.content)
30
+def guided_regex_completion(client: OpenAI, model: str):
31
+ prompt = ("Generate an email address for Alan Turing, who works in Enigma."
32
+ "End in .com and new line. Example result:"
33
+ "alan.turing@enigma.com\n")
34
+
35
+ completion = client.chat.completions.create(
36
+ model=model,
37
+ messages=[{
38
+ "role": "user",
39
+ "content": prompt,
40
+ }],
41
+ extra_body={
42
+ "guided_regex": r"\w+@\w+\.com\n",
43
+ "stop": ["\n"]
44
+ },
45
+ )
46
+ return completion.choices[0].message.content
47
48
49
# Guided decoding by JSON using Pydantic schema
@@ -54,66 +60,100 @@ class CarDescription(BaseModel):
60
car_type: CarType
61
62
57
-json_schema = CarDescription.model_json_schema()
58
-
59
-prompt = ("Generate a JSON with the brand, model and car_type of"
60
- "the most iconic car from the 90's")
61
-completion = client.chat.completions.create(
62
- model="Qwen/Qwen2.5-3B-Instruct",
63
- messages=[{
64
- "role": "user",
65
- "content": prompt,
66
- }],
67
- extra_body={"guided_json": json_schema},
68
-)
69
-print(completion.choices[0].message.content)
63
+def guided_json_completion(client: OpenAI, model: str):
64
+ json_schema = CarDescription.model_json_schema()
65
71
-# Guided decoding by Grammar
72
-simplified_sql_grammar = """
73
- root ::= select_statement
66
+ prompt = ("Generate a JSON with the brand, model and car_type of"
67
+ "the most iconic car from the 90's")
68
+ completion = client.chat.completions.create(
69
+ model=model,
70
+ messages=[{
71
+ "role": "user",
72
+ "content": prompt,
73
+ }],
74
+ extra_body={"guided_json": json_schema},
75
+ )
76
+ return completion.choices[0].message.content
77
75
- select_statement ::= "SELECT " column " from " table " where " condition
78
77
- column ::= "col_1 " | "col_2 "
79
+# Guided decoding by Grammar
80
+def guided_grammar_completion(client: OpenAI, model: str):
81
+ simplified_sql_grammar = """
82
+ root ::= select_statement
83
79
- table ::= "table_1 " | "table_2 "
84
+ select_statement ::= "SELECT " column " from " table " where " condition
85
81
- condition ::= column "= " number
86
+ column ::= "col_1 " | "col_2 "
87
83
- number ::= "1 " | "2 "
84
-"""
88
+ table ::= "table_1 " | "table_2 "
89
86
-prompt = ("Generate an SQL query to show the 'username' and 'email'"
87
- "from the 'users' table.")
88
-completion = client.chat.completions.create(
89
- model="Qwen/Qwen2.5-3B-Instruct",
90
- messages=[{
91
- "role": "user",
92
- "content": prompt,
93
- }],
94
- extra_body={"guided_grammar": simplified_sql_grammar},
95
-)
96
-print(completion.choices[0].message.content)
90
+ condition ::= column "= " number
91
98
-# Extra backend options
99
-prompt = ("Generate an email address for Alan Turing, who works in Enigma."
100
- "End in .com and new line. Example result:"
101
- "alan.turing@enigma.com\n")
92
+ number ::= "1 " | "2 "
93
+ """
94
103
-try:
104
- # The no-fallback option forces vLLM to use xgrammar, so when it fails
105
- # you get a 400 with the reason why
95
+ prompt = ("Generate an SQL query to show the 'username' and 'email'"
96
+ "from the 'users' table.")
97
completion = client.chat.completions.create(
107
- model="Qwen/Qwen2.5-3B-Instruct",
98
+ model=model,
99
messages=[{
100
"role": "user",
101
"content": prompt,
102
}],
112
- extra_body={
113
- "guided_regex": r"\w+@\w+\.com\n",
114
- "stop": ["\n"],
115
- "guided_decoding_backend": "xgrammar:no-fallback"
116
- },
103
+ extra_body={"guided_grammar": simplified_sql_grammar},
104
+ )
105
+ return completion.choices[0].message.content
106
+
107
+
108
+# Extra backend options
109
+def extra_backend_options_completion(client: OpenAI, model: str):
110
+ prompt = ("Generate an email address for Alan Turing, who works in Enigma."
111
+ "End in .com and new line. Example result:"
112
+ "alan.turing@enigma.com\n")
113
+
114
+ try:
115
+ # The no-fallback option forces vLLM to use xgrammar, so when it fails
116
+ # you get a 400 with the reason why
117
+ completion = client.chat.completions.create(
118
+ model=model,
119
+ messages=[{
120
+ "role": "user",
121
+ "content": prompt,
122
+ }],
123
+ extra_body={
124
+ "guided_regex": r"\w+@\w+\.com\n",
125
+ "stop": ["\n"],
126
+ "guided_decoding_backend": "xgrammar:no-fallback"
127
+ },
128
+ )
129
+ return completion.choices[0].message.content
130
+ except BadRequestError as e:
131
+ print("This error is expected:", e)
132
+
133
+
134
+def main():
135
+ client: OpenAI = OpenAI(
136
+ base_url="http://localhost:8000/v1",
137
+ api_key="-",
138
)
118
-except BadRequestError as e:
119
- print("This error is expected:", e)
139
+
140
+ model = "Qwen/Qwen2.5-3B-Instruct"
141
+
142
+ print("Guided Choice Completion:")
143
+ print(guided_choice_completion(client, model))
144
+
145
+ print("\nGuided Regex Completion:")
146
+ print(guided_regex_completion(client, model))
147
+
148
+ print("\nGuided JSON Completion:")
149
+ print(guided_json_completion(client, model))
150
+
151
+ print("\nGuided Grammar Completion:")
152
+ print(guided_grammar_completion(client, model))
153
+
154
+ print("\nExtra Backend Options Completion:")
155
+ print(extra_backend_options_completion(client, model))
156
+
157
+
158
+if __name__ == "__main__":
159
+ main()
examples/online_serving/openai_chat_completion_structured_outputs_with_reasoning.py
+92
-66
@@ -25,29 +25,28 @@ from pydantic import BaseModel
25
openai_api_key = "EMPTY"
26
openai_api_base = "http://localhost:8000/v1"
27
28
-client = OpenAI(
29
- api_key=openai_api_key,
30
- base_url=openai_api_base,
31
-)
28
33
-models = client.models.list()
34
-model = models.data[0].id
29
+def print_completion_details(completion):
30
+ print("reasoning_content: ",
31
+ completion.choices[0].message.reasoning_content)
32
+ print("content: ", completion.choices[0].message.content)
33
+
34
35
# Guided decoding by Regex
37
-prompt = ("What is the capital of France?")
38
-
39
-completion = client.chat.completions.create(
40
- model=model,
41
- messages=[{
42
- "role": "user",
43
- "content": prompt,
44
- }],
45
- extra_body={
46
- "guided_regex": "(Paris|London)",
47
- },
48
-)
49
-print("reasoning_content: ", completion.choices[0].message.reasoning_content)
50
-print("content: ", completion.choices[0].message.content)
36
+def guided_regex_completion(client: OpenAI, model: str):
37
+ prompt = ("What is the capital of France?")
38
+
39
+ completion = client.chat.completions.create(
40
+ model=model,
41
+ messages=[{
42
+ "role": "user",
43
+ "content": prompt,
44
+ }],
45
+ extra_body={
46
+ "guided_regex": "(Paris|London)",
47
+ },
48
+ )
49
+ print_completion_details(completion)
50
51
52
class People(BaseModel):
@@ -55,19 +54,19 @@ class People(BaseModel):
54
age: int
55
56
58
-json_schema = People.model_json_schema()
57
+def guided_json_completion(client: OpenAI, model: str):
58
+ json_schema = People.model_json_schema()
59
60
-prompt = ("Generate a JSON with the name and age of one random person.")
61
-completion = client.chat.completions.create(
62
- model=model,
63
- messages=[{
64
- "role": "user",
65
- "content": prompt,
66
- }],
67
- extra_body={"guided_json": json_schema},
68
-)
69
-print("reasoning_content: ", completion.choices[0].message.reasoning_content)
70
-print("content: ", completion.choices[0].message.content)
60
+ prompt = ("Generate a JSON with the name and age of one random person.")
61
+ completion = client.chat.completions.create(
62
+ model=model,
63
+ messages=[{
64
+ "role": "user",
65
+ "content": prompt,
66
+ }],
67
+ extra_body={"guided_json": json_schema},
68
+ )
69
+ print_completion_details(completion)
70
71
72
# Guided decoding by JSON using Pydantic schema
@@ -84,46 +83,73 @@ class CarDescription(BaseModel):
83
car_type: CarType
84
85
87
-json_schema = CarDescription.model_json_schema()
86
+def guided_car_json_completion(client: OpenAI, model: str):
87
+ json_schema = CarDescription.model_json_schema()
88
+
89
+ prompt = ("Generate a JSON with the brand, model and car_type of"
90
+ "the most iconic car from the 90's")
91
+ completion = client.chat.completions.create(
92
+ model=model,
93
+ messages=[{
94
+ "role": "user",
95
+ "content": prompt,
96
+ }],
97
+ extra_body={"guided_json": json_schema},
98
+ )
99
+ print_completion_details(completion)
100
89
-prompt = ("Generate a JSON with the brand, model and car_type of"
90
- "the most iconic car from the 90's")
91
-completion = client.chat.completions.create(
92
- model=model,
93
- messages=[{
94
- "role": "user",
95
- "content": prompt,
96
- }],
97
- extra_body={"guided_json": json_schema},
98
-)
99
-print("reasoning_content: ", completion.choices[0].message.reasoning_content)
100
-print("content: ", completion.choices[0].message.content)
101
102
# Guided decoding by Grammar
103
-simplified_sql_grammar = """
104
- root ::= select_statement
103
+def guided_grammar_completion(client: OpenAI, model: str):
104
+ simplified_sql_grammar = """
105
+ root ::= select_statement
106
106
- select_statement ::= "SELECT " column " from " table " where " condition
107
+ select_statement ::= "SELECT " column " from " table " where " condition
108
108
- column ::= "col_1 " | "col_2 "
109
+ column ::= "col_1 " | "col_2 "
110
110
- table ::= "table_1 " | "table_2 "
111
+ table ::= "table_1 " | "table_2 "
112
112
- condition ::= column "= " number
113
+ condition ::= column "= " number
114
+
115
+ number ::= "1 " | "2 "
116
+ """
117
+
118
+ # This may be very slow https://github.com/vllm-project/vllm/issues/12122
119
+ prompt = ("Generate an SQL query to show the 'username' and 'email'"
120
+ "from the 'users' table.")
121
+ completion = client.chat.completions.create(
122
+ model=model,
123
+ messages=[{
124
+ "role": "user",
125
+ "content": prompt,
126
+ }],
127
+ extra_body={"guided_grammar": simplified_sql_grammar},
128
+ )
129
+ print_completion_details(completion)
130
+
131
+
132
+def main():
133
+ client: OpenAI = OpenAI(
134
+ api_key=openai_api_key,
135
+ base_url=openai_api_base,
136
+ )
137
+
138
+ models = client.models.list()
139
+ model: str = models.data[0].id
140
+
141
+ print("Guided Regex Completion:")
142
+ guided_regex_completion(client, model)
143
+
144
+ print("\nGuided JSON Completion (People):")
145
+ guided_json_completion(client, model)
146
+
147
+ print("\nGuided JSON Completion (CarDescription):")
148
+ guided_car_json_completion(client, model)
149
+
150
+ print("\nGuided Grammar Completion:")
151
+ guided_grammar_completion(client, model)
152
114
- number ::= "1 " | "2 "
115
-"""
153
117
-# This may be very slow https://github.com/vllm-project/vllm/issues/12122
118
-prompt = ("Generate an SQL query to show the 'username' and 'email'"
119
- "from the 'users' table.")
120
-completion = client.chat.completions.create(
121
- model=model,
122
- messages=[{
123
- "role": "user",
124
- "content": prompt,
125
- }],
126
- extra_body={"guided_grammar": simplified_sql_grammar},
127
-)
128
-print("reasoning_content: ", completion.choices[0].message.reasoning_content)
129
-print("content: ", completion.choices[0].message.content)
154
+if __name__ == "__main__":
155
+ main()