A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Structured Output] Make the output of structured output example more complete (#22481)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
Aug 16, 2025 at 02:29 UTC
48b01fd4d442d4b9250cef4fca3ca75d5c5c1f69
1 file changed
+13
-3
examples/offline_inference/structured_outputs.py
+13
-3
@@ -15,6 +15,8 @@ from pydantic import BaseModel
15
from vllm import LLM, SamplingParams
16
from vllm.sampling_params import GuidedDecodingParams
17
18
+MAX_TOKENS = 50
19
+
20
# Guided decoding by Choice (list of possible options)
21
guided_decoding_params_choice = GuidedDecodingParams(choice=["Positive", "Negative"])
22
sampling_params_choice = SamplingParams(guided_decoding=guided_decoding_params_choice)
@@ -23,7 +25,9 @@ prompt_choice = "Classify this sentiment: vLLM is wonderful!"
25
# Guided decoding by Regex
26
guided_decoding_params_regex = GuidedDecodingParams(regex=r"\w+@\w+\.com\n")
27
sampling_params_regex = SamplingParams(
26
- guided_decoding=guided_decoding_params_regex, stop=["\n"]
28
+ guided_decoding=guided_decoding_params_regex,
29
+ stop=["\n"],
30
+ max_tokens=MAX_TOKENS,
31
)
32
prompt_regex = (
33
"Generate an email address for Alan Turing, who works in Enigma."
@@ -48,7 +52,10 @@ class CarDescription(BaseModel):
52
53
json_schema = CarDescription.model_json_schema()
54
guided_decoding_params_json = GuidedDecodingParams(json=json_schema)
51
-sampling_params_json = SamplingParams(guided_decoding=guided_decoding_params_json)
55
+sampling_params_json = SamplingParams(
56
+ guided_decoding=guided_decoding_params_json,
57
+ max_tokens=MAX_TOKENS,
58
+)
59
prompt_json = (
60
"Generate a JSON with the brand, model and car_type of"
61
"the most iconic car from the 90's"
@@ -64,7 +71,10 @@ condition ::= column "= " number
71
number ::= "1 " | "2 "
72
"""
73
guided_decoding_params_grammar = GuidedDecodingParams(grammar=simplified_sql_grammar)
67
-sampling_params_grammar = SamplingParams(guided_decoding=guided_decoding_params_grammar)
74
+sampling_params_grammar = SamplingParams(
75
+ guided_decoding=guided_decoding_params_grammar,
76
+ max_tokens=MAX_TOKENS,
77
+)
78
prompt_grammar = (
79
"Generate an SQL query to show the 'username' and 'email'from the 'users' table."
80
)