A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[V1] Update structured output offline inference example (#15721)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Apr 8, 2025 at 18:34 UTC
2755c34a8fef8d0c1b0292a7ddae45d8f92f070c
1 file changed
+7
-12
examples/offline_inference/structured_outputs.py
+7
-12
@@ -19,7 +19,7 @@ outputs = llm.generate(
19
print(outputs[0].outputs[0].text)
20
21
# Guided decoding by Regex
22
-guided_decoding_params = GuidedDecodingParams(regex="\w+@\w+\.com\n")
22
+guided_decoding_params = GuidedDecodingParams(regex=r"\w+@\w+\.com\n")
23
sampling_params = SamplingParams(guided_decoding=guided_decoding_params,
24
stop=["\n"])
25
prompt = ("Generate an email address for Alan Turing, who works in Enigma."
@@ -57,17 +57,12 @@ print(outputs[0].outputs[0].text)
57
58
# Guided decoding by Grammar
59
simplified_sql_grammar = """
60
- ?start: select_statement
61
-
62
- ?select_statement: "SELECT " column_list " FROM " table_name
63
-
64
- ?column_list: column_name ("," column_name)*
65
-
66
- ?table_name: identifier
67
-
68
- ?column_name: identifier
69
-
70
- ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
60
+root ::= select_statement
61
+select_statement ::= "SELECT " column " from " table " where " condition
62
+column ::= "col_1 " | "col_2 "
63
+table ::= "table_1 " | "table_2 "
64
+condition ::= column "= " number
65
+number ::= "1 " | "2 "
66
"""
67
guided_decoding_params = GuidedDecodingParams(grammar=simplified_sql_grammar)
68
sampling_params = SamplingParams(guided_decoding=guided_decoding_params)