A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Doc] Add documentation for Structured Outputs (#9943)
Signed-off-by: ismael-dm <ismaeldm99@gmail.com>
ismael-dm committed
Nov 18, 2024 at 18:52 UTC
31894a21559436f4a9d72f751e8bd7ba4ab18613
4 files changed
+346
docs/source/index.rst
+1
@@ -101,6 +101,7 @@ Documentation
101
models/engine_args
102
models/lora
103
models/vlm
104
+ models/structured_outputs
105
models/spec_decode
106
models/performance
107
docs/source/models/structured_outputs.rst
new
+173
@@ -0,0 +1,173 @@
1
+.. _structured_outputs:
2
+
3
+Structured Outputs
4
+==================
5
+
6
+vLLM supports the generation of structured outputs using `outlines <https://github.com/dottxt-ai/outlines>`_ or `lm-format-enforcer <https://github.com/noamgat/lm-format-enforcer>`_ as backends for the guided decoding.
7
+This document shows you some examples of the different options that are available to generate structured outputs.
8
+
9
+
10
+Online Inference (OpenAI API)
11
+-----------------------------
12
+
13
+You can generate structured outputs using the OpenAI’s `Completions <https://platform.openai.com/docs/api-reference/completions>`_ and `Chat <https://platform.openai.com/docs/api-reference/chat>`_ API.
14
+
15
+The following parameters are supported, which must be added as extra parameters:
16
+
17
+- ``guided_choice``: the output will be exactly one of the choices.
18
+- ``guided_regex``: the output will follow the regex pattern.
19
+- ``guided_json``: the output will follow the JSON schema.
20
+- ``guided_grammar``: the output will follow the context free grammar.
21
+- ``guided_whitespace_pattern``: used to override the default whitespace pattern for guided json decoding.
22
+- ``guided_decoding_backend``: used to select the guided decoding backend to use.
23
+
24
+You can see the complete list of supported parameters on the `OpenAI Compatible Server </../serving/openai_compatible_server.html>`_ page.
25
+
26
+Now let´s see an example for each of the cases, starting with the ``guided_choice``, as it´s the easiest one:
27
+
28
+.. code-block:: python
29
+
30
+ from openai import OpenAI
31
+ client = OpenAI(
32
+ base_url="http://localhost:8000/v1",
33
+ api_key="-",
34
+ )
35
+
36
+ completion = client.chat.completions.create(
37
+ model="Qwen/Qwen2.5-3B-Instruct",
38
+ messages=[
39
+ {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
40
+ ],
41
+ extra_body={"guided_choice": ["positive", "negative"]},
42
+ )
43
+ print(completion.choices[0].message.content)
44
+
45
+
46
+The next example shows how to use the ``guided_regex``. The idea is to generate an email address, given a simple regex template:
47
+
48
+.. code-block:: python
49
+
50
+ completion = client.chat.completions.create(
51
+ model="Qwen/Qwen2.5-3B-Instruct",
52
+ messages=[
53
+ {
54
+ "role": "user",
55
+ "content": "Generate an example email address for Alan Turing, who works in Enigma. End in .com and new line. Example result: alan.turing@enigma.com\n",
56
+ }
57
+ ],
58
+ extra_body={"guided_regex": "\w+@\w+\.com\n", "stop": ["\n"]},
59
+ )
60
+ print(completion.choices[0].message.content)
61
+
62
+One of the most relevant features in structured text generation is the option to generate a valid JSON with pre-defined fields and formats.
63
+For this we can use the ``guided_json`` parameter in two different ways:
64
+
65
+- Using directly a `JSON Schema <https://json-schema.org/>`_
66
+- Defining a `Pydantic model <https://docs.pydantic.dev/latest/>`_ and then extracting the JSON Schema from it (which is normally an easier option).
67
+
68
+The next example shows how to use the ``guided_json`` parameter with a Pydantic model:
69
+
70
+.. code-block:: python
71
+
72
+ from pydantic import BaseModel
73
+ from enum import Enum
74
+
75
+ class CarType(str, Enum):
76
+ sedan = "sedan"
77
+ suv = "SUV"
78
+ truck = "Truck"
79
+ coupe = "Coupe"
80
+
81
+
82
+ class CarDescription(BaseModel):
83
+ brand: str
84
+ model: str
85
+ car_type: CarType
86
+
87
+
88
+ json_schema = CarDescription.model_json_schema()
89
+
90
+ completion = client.chat.completions.create(
91
+ model="Qwen/Qwen2.5-3B-Instruct",
92
+ messages=[
93
+ {
94
+ "role": "user",
95
+ "content": "Generate a JSON with the brand, model and car_type of the most iconic car from the 90's",
96
+ }
97
+ ],
98
+ extra_body={"guided_json": json_schema},
99
+ )
100
+ print(completion.choices[0].message.content)
101
+
102
+.. tip::
103
+ While not strictly necessary, normally it´s better to indicate in the prompt that a JSON needs to be generated and which fields and how should the LLM fill them.
104
+ This can improve the results notably in most cases.
105
+
106
+
107
+Finally we have the ``guided_grammar``, which probably is the most difficult one to use but it´s really powerful, as it allows us to define complete languages like SQL queries.
108
+It works by using a context free EBNF grammar, which for example we can use to define a specific format of simplified SQL queries, like in the example below:
109
+
110
+.. code-block:: python
111
+
112
+ simplified_sql_grammar = """
113
+ ?start: select_statement
114
+
115
+ ?select_statement: "SELECT " column_list " FROM " table_name
116
+
117
+ ?column_list: column_name ("," column_name)*
118
+
119
+ ?table_name: identifier
120
+
121
+ ?column_name: identifier
122
+
123
+ ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
124
+ """
125
+
126
+ completion = client.chat.completions.create(
127
+ model="Qwen/Qwen2.5-3B-Instruct",
128
+ messages=[
129
+ {
130
+ "role": "user",
131
+ "content": "Generate an SQL query to show the 'username' and 'email' from the 'users' table.",
132
+ }
133
+ ],
134
+ extra_body={"guided_grammar": simplified_sql_grammar},
135
+ )
136
+ print(completion.choices[0].message.content)
137
+
138
+The complete code of the examples can be found on `examples/openai_chat_completion_structured_outputs.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_structured_outputs.py>`_.
139
+
140
+
141
+Offline Inference
142
+-----------------
143
+
144
+Offline inference allows for the same types of guided decoding.
145
+To use it, we´ll need to configure the guided decoding using the class ``GuidedDecodingParams`` inside ``SamplingParams``.
146
+The main available options inside ``GuidedDecodingParams`` are:
147
+
148
+- ``json``
149
+- ``regex``
150
+- ``choice``
151
+- ``grammar``
152
+- ``backend``
153
+- ``whitespace_pattern``
154
+
155
+These parameters can be used in the same way as the parameters from the Online Inference examples above.
156
+One example for the usage of the ``choices`` parameter is shown below:
157
+
158
+.. code-block:: python
159
+
160
+ from vllm import LLM, SamplingParams
161
+ from vllm.sampling_params import GuidedDecodingParams
162
+
163
+ llm = LLM(model="HuggingFaceTB/SmolLM2-1.7B-Instruct")
164
+
165
+ guided_decoding_params = GuidedDecodingParams(choice=["Positive", "Negative"])
166
+ sampling_params = SamplingParams(guided_decoding=guided_decoding_params)
167
+ outputs = llm.generate(
168
+ prompts="Classify this sentiment: vLLM is wonderful!",
169
+ sampling_params=sampling_params,
170
+ )
171
+ print(outputs[0].outputs[0].text)
172
+
173
+A complete example with all options can be found in `examples/offline_inference_structured_outputs.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_structured_outputs.py>`_.
\ No newline at end of file
examples/offline_inference_structured_outputs.py
new
+78
@@ -0,0 +1,78 @@
1
+from enum import Enum
2
+
3
+from pydantic import BaseModel
4
+
5
+from vllm import LLM, SamplingParams
6
+from vllm.sampling_params import GuidedDecodingParams
7
+
8
+llm = LLM(model="Qwen/Qwen2.5-3B-Instruct", max_model_len=100)
9
+
10
+# Guided decoding by Choice (list of possible options)
11
+guided_decoding_params = GuidedDecodingParams(choice=["Positive", "Negative"])
12
+sampling_params = SamplingParams(guided_decoding=guided_decoding_params)
13
+outputs = llm.generate(
14
+ prompts="Classify this sentiment: vLLM is wonderful!",
15
+ sampling_params=sampling_params,
16
+)
17
+print(outputs[0].outputs[0].text)
18
+
19
+# Guided decoding by Regex
20
+guided_decoding_params = GuidedDecodingParams(regex="\w+@\w+\.com\n")
21
+sampling_params = SamplingParams(guided_decoding=guided_decoding_params,
22
+ stop=["\n"])
23
+prompt = ("Generate an email address for Alan Turing, who works in Enigma."
24
+ "End in .com and new line. Example result:"
25
+ "alan.turing@enigma.com\n")
26
+outputs = llm.generate(prompts=prompt, sampling_params=sampling_params)
27
+print(outputs[0].outputs[0].text)
28
+
29
+
30
+# Guided decoding by JSON using Pydantic schema
31
+class CarType(str, Enum):
32
+ sedan = "sedan"
33
+ suv = "SUV"
34
+ truck = "Truck"
35
+ coupe = "Coupe"
36
+
37
+
38
+class CarDescription(BaseModel):
39
+ brand: str
40
+ model: str
41
+ car_type: CarType
42
+
43
+
44
+json_schema = CarDescription.model_json_schema()
45
+
46
+guided_decoding_params = GuidedDecodingParams(json=json_schema)
47
+sampling_params = SamplingParams(guided_decoding=guided_decoding_params)
48
+prompt = ("Generate a JSON with the brand, model and car_type of"
49
+ "the most iconic car from the 90's")
50
+outputs = llm.generate(
51
+ prompts=prompt,
52
+ sampling_params=sampling_params,
53
+)
54
+print(outputs[0].outputs[0].text)
55
+
56
+# Guided decoding by Grammar
57
+simplified_sql_grammar = """
58
+ ?start: select_statement
59
+
60
+ ?select_statement: "SELECT " column_list " FROM " table_name
61
+
62
+ ?column_list: column_name ("," column_name)*
63
+
64
+ ?table_name: identifier
65
+
66
+ ?column_name: identifier
67
+
68
+ ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
69
+"""
70
+guided_decoding_params = GuidedDecodingParams(grammar=simplified_sql_grammar)
71
+sampling_params = SamplingParams(guided_decoding=guided_decoding_params)
72
+prompt = ("Generate an SQL query to show the 'username' and 'email'"
73
+ "from the 'users' table.")
74
+outputs = llm.generate(
75
+ prompts=prompt,
76
+ sampling_params=sampling_params,
77
+)
78
+print(outputs[0].outputs[0].text)
examples/openai_chat_completion_structured_outputs.py
new
+94
@@ -0,0 +1,94 @@
1
+from enum import Enum
2
+
3
+from openai import OpenAI
4
+from pydantic import BaseModel
5
+
6
+client = OpenAI(
7
+ base_url="http://localhost:8000/v1",
8
+ api_key="-",
9
+)
10
+
11
+# Guided decoding by Choice (list of possible options)
12
+completion = client.chat.completions.create(
13
+ model="Qwen/Qwen2.5-3B-Instruct",
14
+ messages=[{
15
+ "role": "user",
16
+ "content": "Classify this sentiment: vLLM is wonderful!"
17
+ }],
18
+ extra_body={"guided_choice": ["positive", "negative"]},
19
+)
20
+print(completion.choices[0].message.content)
21
+
22
+# Guided decoding by Regex
23
+prompt = ("Generate an email address for Alan Turing, who works in Enigma."
24
+ "End in .com and new line. Example result:"
25
+ "alan.turing@enigma.com\n")
26
+
27
+completion = client.chat.completions.create(
28
+ model="Qwen/Qwen2.5-3B-Instruct",
29
+ messages=[{
30
+ "role": "user",
31
+ "content": prompt,
32
+ }],
33
+ extra_body={
34
+ "guided_regex": "\w+@\w+\.com\n",
35
+ "stop": ["\n"]
36
+ },
37
+)
38
+print(completion.choices[0].message.content)
39
+
40
+
41
+# Guided decoding by JSON using Pydantic schema
42
+class CarType(str, Enum):
43
+ sedan = "sedan"
44
+ suv = "SUV"
45
+ truck = "Truck"
46
+ coupe = "Coupe"
47
+
48
+
49
+class CarDescription(BaseModel):
50
+ brand: str
51
+ model: str
52
+ car_type: CarType
53
+
54
+
55
+json_schema = CarDescription.model_json_schema()
56
+
57
+prompt = ("Generate a JSON with the brand, model and car_type of"
58
+ "the most iconic car from the 90's")
59
+completion = client.chat.completions.create(
60
+ model="Qwen/Qwen2.5-3B-Instruct",
61
+ messages=[{
62
+ "role": "user",
63
+ "content": prompt,
64
+ }],
65
+ extra_body={"guided_json": json_schema},
66
+)
67
+print(completion.choices[0].message.content)
68
+
69
+# Guided decoding by Grammar
70
+simplified_sql_grammar = """
71
+ ?start: select_statement
72
+
73
+ ?select_statement: "SELECT " column_list " FROM " table_name
74
+
75
+ ?column_list: column_name ("," column_name)*
76
+
77
+ ?table_name: identifier
78
+
79
+ ?column_name: identifier
80
+
81
+ ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
82
+"""
83
+
84
+prompt = ("Generate an SQL query to show the 'username' and 'email'"
85
+ "from the 'users' table.")
86
+completion = client.chat.completions.create(
87
+ model="Qwen/Qwen2.5-3B-Instruct",
88
+ messages=[{
89
+ "role": "user",
90
+ "content": prompt,
91
+ }],
92
+ extra_body={"guided_grammar": simplified_sql_grammar},
93
+)
94
+print(completion.choices[0].message.content)