A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Doc] Add docs about OpenAI compatible server (#3288)
Simon Mo committed
Mar 18, 2024 at 22:05 UTC
ef65dcfa6f5820ce9e4a2411e9be18586f6fd467
10 files changed
+383
-161
docs/requirements-docs.txt
+7
@@ -1,3 +1,10 @@
1
sphinx == 6.2.1
2
sphinx-book-theme == 1.0.1
3
sphinx-copybutton == 0.5.2
4
+myst-parser == 2.0.0
5
+sphinx-argparse
6
+
7
+# packages to install to build the documentation
8
+pydantic
9
+-f https://download.pytorch.org/whl/cpu
10
+torch
\ No newline at end of file
docs/source/conf.py
+3
-1
@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
22
# -- Project information -----------------------------------------------------
23
24
project = 'vLLM'
25
-copyright = '2023, vLLM Team'
25
+copyright = '2024, vLLM Team'
26
author = 'the vLLM Team'
27
28
# -- General configuration ---------------------------------------------------
@@ -37,6 +37,8 @@ extensions = [
37
"sphinx_copybutton",
38
"sphinx.ext.autodoc",
39
"sphinx.ext.autosummary",
40
+ "myst_parser",
41
+ "sphinxarg.ext",
42
]
43
44
# Add any paths that contain templates here, relative to this directory.
docs/source/dev/sampling_params.rst
new
+4
@@ -0,0 +1,4 @@
1
+Sampling Params
2
+===============
3
+
4
+.. automodule:: vllm.sampling_params.SamplingParams
\ No newline at end of file
docs/source/index.rst
+4
-6
@@ -69,14 +69,11 @@ Documentation
69
:maxdepth: 1
70
:caption: Serving
71
72
- serving/distributed_serving
73
- serving/run_on_sky
74
- serving/deploying_with_kserve
75
- serving/deploying_with_triton
76
- serving/deploying_with_bentoml
72
+ serving/openai_compatible_server
73
serving/deploying_with_docker
78
- serving/serving_with_langchain
74
+ serving/distributed_serving
75
serving/metrics
76
+ serving/integrations
77
78
.. toctree::
79
:maxdepth: 1
@@ -98,6 +95,7 @@ Documentation
95
:maxdepth: 2
96
:caption: Developer Documentation
97
98
+ dev/sampling_params
99
dev/engine/engine_index
100
dev/kernel/paged_attention
101
docs/source/models/lora.rst
+1
-1
@@ -90,7 +90,7 @@ Requests can specify the LoRA adapter as if it were any other model via the ``mo
90
processed according to the server-wide LoRA configuration (i.e. in parallel with base model requests, and potentially other
91
LoRA adapter requests if they were provided and ``max_loras`` is set high enough).
92
93
-The following is an example request
93
+The following is an example request
94
95
.. code-block:: bash
96
docs/source/serving/integrations.rst
new
+11
@@ -0,0 +1,11 @@
1
+Integrations
2
+------------
3
+
4
+.. toctree::
5
+ :maxdepth: 1
6
+
7
+ run_on_sky
8
+ deploying_with_kserve
9
+ deploying_with_triton
10
+ deploying_with_bentoml
11
+ serving_with_langchain
docs/source/serving/openai_compatible_server.md
new
+114
@@ -0,0 +1,114 @@
1
+# OpenAI Compatible Server
2
+
3
+vLLM provides an HTTP server that implements OpenAI's [Completions](https://platform.openai.com/docs/api-reference/completions) and [Chat](https://platform.openai.com/docs/api-reference/chat) API.
4
+
5
+You can start the server using Python, or using [Docker](deploying_with_docker.rst):
6
+```bash
7
+python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-hf --dtype float32 --api-key token-abc123
8
+```
9
+
10
+To call the server, you can use the official OpenAI Python client library, or any other HTTP client.
11
+```python
12
+from openai import OpenAI
13
+client = OpenAI(
14
+ base_url="http://localhost:8000/v1",
15
+ api_key="token-abc123",
16
+)
17
+
18
+completion = client.chat.completions.create(
19
+ model="meta-llama/Llama-2-7b-hf",
20
+ messages=[
21
+ {"role": "system", "content": "You are a helpful assistant."},
22
+ {"role": "user", "content": "Hello!"}
23
+ ]
24
+)
25
+
26
+print(completion.choices[0].message)
27
+```
28
+
29
+## API Reference
30
+Please see the [OpenAI API Reference](https://platform.openai.com/docs/api-reference) for more information on the API. We support all parameters except:
31
+- Chat: `tools`, and `tool_choice`.
32
+- Completions: `suffix`.
33
+
34
+## Extra Parameters
35
+vLLM supports a set of parameters that are not part of the OpenAI API.
36
+In order to use them, you can pass them as extra parameters in the OpenAI client.
37
+Or directly merge them into the JSON payload if you are using HTTP call directly.
38
+
39
+```python
40
+completion = client.chat.completions.create(
41
+ model="meta-llama/Llama-2-7b-hf",
42
+ messages=[
43
+ {"role": "system", "content": "You are a helpful assistant."},
44
+ {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
45
+ ],
46
+ extra_body={
47
+ "guided_choice": ["positive", "negative"]
48
+ }
49
+)
50
+```
51
+
52
+### Extra Parameters for Chat API
53
+The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
54
+
55
+```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
56
+:language: python
57
+:start-after: begin-chat-completion-sampling-params
58
+:end-before: end-chat-completion-sampling-params
59
+```
60
+
61
+The following extra parameters are supported:
62
+
63
+```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
64
+:language: python
65
+:start-after: begin-chat-completion-extra-params
66
+:end-before: end-chat-completion-extra-params
67
+```
68
+
69
+### Extra Parameters for Completions API
70
+The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
71
+
72
+```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
73
+:language: python
74
+:start-after: begin-completion-sampling-params
75
+:end-before: end-completion-sampling-params
76
+```
77
+
78
+The following extra parameters are supported:
79
+
80
+```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
81
+:language: python
82
+:start-after: begin-completion-extra-params
83
+:end-before: end-completion-extra-params
84
+```
85
+
86
+## Chat Template
87
+
88
+In order for the language model to support chat protocol, vLLM requires the model to include
89
+a chat template in its tokenizer configuration. The chat template is a Jinja2 template that
90
+specifies how are roles, messages, and other chat-specific tokens are encoded in the input.
91
+
92
+An example chat template for `meta-llama/Llama-2-7b-chat-hf` can be found [here](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/blob/09bd0f49e16738cdfaa6e615203e126038736eb0/tokenizer_config.json#L12)
93
+
94
+Some models do not provide a chat template even though they are instruction/chat fine-tuned. For those model,
95
+you can manually specify their chat template in the `--chat-template` parameter with the file path to the chat
96
+template, or the template in string form. Without a chat template, the server will not be able to process chat
97
+and all chat requests will error.
98
+
99
+```bash
100
+python -m vllm.entrypoints.openai.api_server \
101
+ --model ... \
102
+ --chat-template ./path-to-chat-template.jinja
103
+```
104
+
105
+vLLM community provides a set of chat templates for popular models. You can find them in the examples
106
+directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
107
+
108
+## Command line arguments for the server
109
+
110
+```{argparse}
111
+:module: vllm.entrypoints.openai.cli_args
112
+:func: make_arg_parser
113
+:prog: vllm-openai-server
114
+```
\ No newline at end of file
vllm/entrypoints/openai/api_server.py
+2
-106
@@ -1,11 +1,8 @@
1
-import argparse
1
import asyncio
3
-import json
2
from contextlib import asynccontextmanager
3
import os
4
import importlib
5
import inspect
8
-import ssl
6
7
from prometheus_client import make_asgi_app
8
import fastapi
@@ -23,9 +20,9 @@ from vllm.entrypoints.openai.protocol import (CompletionRequest,
20
ChatCompletionRequest,
21
ErrorResponse)
22
from vllm.logger import init_logger
23
+from vllm.entrypoints.openai.cli_args import make_arg_parser
24
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
25
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
28
-from vllm.entrypoints.openai.serving_engine import LoRA
26
27
TIMEOUT_KEEP_ALIVE = 5 # seconds
28
@@ -51,109 +48,8 @@ async def lifespan(app: fastapi.FastAPI):
48
app = fastapi.FastAPI(lifespan=lifespan)
49
50
54
-class LoRAParserAction(argparse.Action):
55
-
56
- def __call__(self, parser, namespace, values, option_string=None):
57
- lora_list = []
58
- for item in values:
59
- name, path = item.split('=')
60
- lora_list.append(LoRA(name, path))
61
- setattr(namespace, self.dest, lora_list)
62
-
63
-
51
def parse_args():
65
- parser = argparse.ArgumentParser(
66
- description="vLLM OpenAI-Compatible RESTful API server.")
67
- parser.add_argument("--host", type=str, default=None, help="host name")
68
- parser.add_argument("--port", type=int, default=8000, help="port number")
69
- parser.add_argument(
70
- "--uvicorn-log-level",
71
- type=str,
72
- default="info",
73
- choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'],
74
- help="log level for uvicorn")
75
- parser.add_argument("--allow-credentials",
76
- action="store_true",
77
- help="allow credentials")
78
- parser.add_argument("--allowed-origins",
79
- type=json.loads,
80
- default=["*"],
81
- help="allowed origins")
82
- parser.add_argument("--allowed-methods",
83
- type=json.loads,
84
- default=["*"],
85
- help="allowed methods")
86
- parser.add_argument("--allowed-headers",
87
- type=json.loads,
88
- default=["*"],
89
- help="allowed headers")
90
- parser.add_argument("--api-key",
91
- type=str,
92
- default=None,
93
- help="If provided, the server will require this key "
94
- "to be presented in the header.")
95
- parser.add_argument("--served-model-name",
96
- type=str,
97
- default=None,
98
- help="The model name used in the API. If not "
99
- "specified, the model name will be the same as "
100
- "the huggingface name.")
101
- parser.add_argument(
102
- "--lora-modules",
103
- type=str,
104
- default=None,
105
- nargs='+',
106
- action=LoRAParserAction,
107
- help="LoRA module configurations in the format name=path. "
108
- "Multiple modules can be specified.")
109
- parser.add_argument("--chat-template",
110
- type=str,
111
- default=None,
112
- help="The file path to the chat template, "
113
- "or the template in single-line form "
114
- "for the specified model")
115
- parser.add_argument("--response-role",
116
- type=str,
117
- default="assistant",
118
- help="The role name to return if "
119
- "`request.add_generation_prompt=true`.")
120
- parser.add_argument("--ssl-keyfile",
121
- type=str,
122
- default=None,
123
- help="The file path to the SSL key file")
124
- parser.add_argument("--ssl-certfile",
125
- type=str,
126
- default=None,
127
- help="The file path to the SSL cert file")
128
- parser.add_argument("--ssl-ca-certs",
129
- type=str,
130
- default=None,
131
- help="The CA certificates file")
132
- parser.add_argument(
133
- "--ssl-cert-reqs",
134
- type=int,
135
- default=int(ssl.CERT_NONE),
136
- help="Whether client certificate is required (see stdlib ssl module's)"
137
- )
138
- parser.add_argument(
139
- "--root-path",
140
- type=str,
141
- default=None,
142
- help="FastAPI root_path when app is behind a path based routing proxy")
143
- parser.add_argument(
144
- "--middleware",
145
- type=str,
146
- action="append",
147
- default=[],
148
- help="Additional ASGI middleware to apply to the app. "
149
- "We accept multiple --middleware arguments. "
150
- "The value should be an import path. "
151
- "If a function is provided, vLLM will add it to the server "
152
- "using @app.middleware('http'). "
153
- "If a class is provided, vLLM will add it to the server "
154
- "using app.add_middleware(). ")
155
-
156
- parser = AsyncEngineArgs.add_cli_args(parser)
52
+ parser = make_arg_parser()
53
return parser.parse_args()
54
55
vllm/entrypoints/openai/cli_args.py
new
+118
@@ -0,0 +1,118 @@
1
+"""
2
+This file contains the command line arguments for the vLLM's
3
+OpenAI-compatible server. It is kept in a separate file for documentation
4
+purposes.
5
+"""
6
+
7
+import argparse
8
+import json
9
+import ssl
10
+
11
+from vllm.engine.arg_utils import AsyncEngineArgs
12
+from vllm.entrypoints.openai.serving_engine import LoRA
13
+
14
+
15
+class LoRAParserAction(argparse.Action):
16
+
17
+ def __call__(self, parser, namespace, values, option_string=None):
18
+ lora_list = []
19
+ for item in values:
20
+ name, path = item.split('=')
21
+ lora_list.append(LoRA(name, path))
22
+ setattr(namespace, self.dest, lora_list)
23
+
24
+
25
+def make_arg_parser():
26
+ parser = argparse.ArgumentParser(
27
+ description="vLLM OpenAI-Compatible RESTful API server.")
28
+ parser.add_argument("--host", type=str, default=None, help="host name")
29
+ parser.add_argument("--port", type=int, default=8000, help="port number")
30
+ parser.add_argument(
31
+ "--uvicorn-log-level",
32
+ type=str,
33
+ default="info",
34
+ choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'],
35
+ help="log level for uvicorn")
36
+ parser.add_argument("--allow-credentials",
37
+ action="store_true",
38
+ help="allow credentials")
39
+ parser.add_argument("--allowed-origins",
40
+ type=json.loads,
41
+ default=["*"],
42
+ help="allowed origins")
43
+ parser.add_argument("--allowed-methods",
44
+ type=json.loads,
45
+ default=["*"],
46
+ help="allowed methods")
47
+ parser.add_argument("--allowed-headers",
48
+ type=json.loads,
49
+ default=["*"],
50
+ help="allowed headers")
51
+ parser.add_argument("--api-key",
52
+ type=str,
53
+ default=None,
54
+ help="If provided, the server will require this key "
55
+ "to be presented in the header.")
56
+ parser.add_argument("--served-model-name",
57
+ type=str,
58
+ default=None,
59
+ help="The model name used in the API. If not "
60
+ "specified, the model name will be the same as "
61
+ "the huggingface name.")
62
+ parser.add_argument(
63
+ "--lora-modules",
64
+ type=str,
65
+ default=None,
66
+ nargs='+',
67
+ action=LoRAParserAction,
68
+ help="LoRA module configurations in the format name=path. "
69
+ "Multiple modules can be specified.")
70
+ parser.add_argument("--chat-template",
71
+ type=str,
72
+ default=None,
73
+ help="The file path to the chat template, "
74
+ "or the template in single-line form "
75
+ "for the specified model")
76
+ parser.add_argument("--response-role",
77
+ type=str,
78
+ default="assistant",
79
+ help="The role name to return if "
80
+ "`request.add_generation_prompt=true`.")
81
+ parser.add_argument("--ssl-keyfile",
82
+ type=str,
83
+ default=None,
84
+ help="The file path to the SSL key file")
85
+ parser.add_argument("--ssl-certfile",
86
+ type=str,
87
+ default=None,
88
+ help="The file path to the SSL cert file")
89
+ parser.add_argument("--ssl-ca-certs",
90
+ type=str,
91
+ default=None,
92
+ help="The CA certificates file")
93
+ parser.add_argument(
94
+ "--ssl-cert-reqs",
95
+ type=int,
96
+ default=int(ssl.CERT_NONE),
97
+ help="Whether client certificate is required (see stdlib ssl module's)"
98
+ )
99
+ parser.add_argument(
100
+ "--root-path",
101
+ type=str,
102
+ default=None,
103
+ help="FastAPI root_path when app is behind a path based routing proxy")
104
+ parser.add_argument(
105
+ "--middleware",
106
+ type=str,
107
+ action="append",
108
+ default=[],
109
+ help="Additional ASGI middleware to apply to the app. "
110
+ "We accept multiple --middleware arguments. "
111
+ "The value should be an import path. "
112
+ "If a function is provided, vLLM will add it to the server "
113
+ "using @app.middleware('http'). "
114
+ "If a class is provided, vLLM will add it to the server "
115
+ "using app.add_middleware(). ")
116
+
117
+ parser = AsyncEngineArgs.add_cli_args(parser)
118
+ return parser
vllm/entrypoints/openai/protocol.py
+119
-47
@@ -61,41 +61,80 @@ class ResponseFormat(BaseModel):
61
62
63
class ChatCompletionRequest(BaseModel):
64
- model: str
64
+ # Ordered by official OpenAI API documentation
65
+ # https://platform.openai.com/docs/api-reference/chat/create
66
messages: List[Dict[str, str]]
66
- temperature: Optional[float] = 0.7
67
- top_p: Optional[float] = 1.0
68
- n: Optional[int] = 1
67
+ model: str
68
+ frequency_penalty: Optional[float] = 0.0
69
+ logit_bias: Optional[Dict[str, float]] = None
70
+ logprobs: Optional[bool] = False
71
+ top_logprobs: Optional[int] = None
72
max_tokens: Optional[int] = None
73
+ n: Optional[int] = 1
74
+ presence_penalty: Optional[float] = 0.0
75
+ response_format: Optional[ResponseFormat] = None
76
seed: Optional[int] = None
77
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
78
stream: Optional[bool] = False
73
- logprobs: Optional[bool] = False
74
- top_logprobs: Optional[int] = None
75
- presence_penalty: Optional[float] = 0.0
76
- frequency_penalty: Optional[float] = 0.0
77
- logit_bias: Optional[Dict[str, float]] = None
79
+ temperature: Optional[float] = 0.7
80
+ top_p: Optional[float] = 1.0
81
user: Optional[str] = None
79
- # Additional parameters supported by vLLM
82
+
83
+ # doc: begin-chat-completion-sampling-params
84
best_of: Optional[int] = None
81
- top_k: Optional[int] = -1
82
- ignore_eos: Optional[bool] = False
85
use_beam_search: Optional[bool] = False
86
+ top_k: Optional[int] = -1
87
+ min_p: Optional[float] = 0.0
88
+ repetition_penalty: Optional[float] = 1.0
89
+ length_penalty: Optional[float] = 1.0
90
early_stopping: Optional[bool] = False
91
+ ignore_eos: Optional[bool] = False
92
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
93
skip_special_tokens: Optional[bool] = True
94
spaces_between_special_tokens: Optional[bool] = True
88
- add_generation_prompt: Optional[bool] = True
89
- echo: Optional[bool] = False
90
- repetition_penalty: Optional[float] = 1.0
91
- min_p: Optional[float] = 0.0
92
- include_stop_str_in_output: Optional[bool] = False
93
- length_penalty: Optional[float] = 1.0
94
- guided_json: Optional[Union[str, dict, BaseModel]] = None
95
- guided_regex: Optional[str] = None
96
- guided_choice: Optional[List[str]] = None
97
- guided_grammar: Optional[str] = None
98
- response_format: Optional[ResponseFormat] = None
95
+ # doc: end-chat-completion-sampling-params
96
+
97
+ # doc: begin-chat-completion-extra-params
98
+ echo: Optional[bool] = Field(
99
+ default=False,
100
+ description=(
101
+ "If true, the new message will be prepended with the last message "
102
+ "if they belong to the same role."),
103
+ )
104
+ add_generation_prompt: Optional[bool] = Field(
105
+ default=True,
106
+ description=
107
+ ("If true, the generation prompt will be added to the chat template. "
108
+ "This is a parameter used by chat template in tokenizer config of the "
109
+ "model."),
110
+ )
111
+ include_stop_str_in_output: Optional[bool] = Field(
112
+ default=False,
113
+ description=(
114
+ "Whether to include the stop string in the output. "
115
+ "This is only applied when the stop or stop_token_ids is set."),
116
+ )
117
+ guided_json: Optional[Union[str, dict, BaseModel]] = Field(
118
+ default=None,
119
+ description=("If specified, the output will follow the JSON schema."),
120
+ )
121
+ guided_regex: Optional[str] = Field(
122
+ default=None,
123
+ description=(
124
+ "If specified, the output will follow the regex pattern."),
125
+ )
126
+ guided_choice: Optional[List[str]] = Field(
127
+ default=None,
128
+ description=(
129
+ "If specified, the output will be exactly one of the choices."),
130
+ )
131
+ guided_grammar: Optional[str] = Field(
132
+ default=None,
133
+ description=(
134
+ "If specified, the output will follow the context free grammar."),
135
+ )
136
+
137
+ # doc: end-chat-completion-extra-params
138
139
def to_sampling_params(self) -> SamplingParams:
140
if self.logprobs and not self.top_logprobs:
@@ -157,41 +196,74 @@ class ChatCompletionRequest(BaseModel):
196
197
198
class CompletionRequest(BaseModel):
199
+ # Ordered by official OpenAI API documentation
200
+ # https://platform.openai.com/docs/api-reference/completions/create
201
model: str
161
- # a string, array of strings, array of tokens, or array of token arrays
202
prompt: Union[List[int], List[List[int]], str, List[str]]
163
- suffix: Optional[str] = None
164
- max_tokens: Optional[int] = 16
165
- temperature: Optional[float] = 1.0
166
- top_p: Optional[float] = 1.0
167
- n: Optional[int] = 1
168
- stream: Optional[bool] = False
169
- logprobs: Optional[int] = None
203
+ best_of: Optional[int] = None
204
echo: Optional[bool] = False
171
- stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
172
- seed: Optional[int] = None
173
- presence_penalty: Optional[float] = 0.0
205
frequency_penalty: Optional[float] = 0.0
175
- best_of: Optional[int] = None
206
logit_bias: Optional[Dict[str, float]] = None
207
+ logprobs: Optional[int] = None
208
+ max_tokens: Optional[int] = 16
209
+ n: Optional[int] = 1
210
+ presence_penalty: Optional[float] = 0.0
211
+ seed: Optional[int] = None
212
+ stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
213
+ stream: Optional[bool] = False
214
+ suffix: Optional[str] = None
215
+ temperature: Optional[float] = 1.0
216
+ top_p: Optional[float] = 1.0
217
user: Optional[str] = None
178
- # Additional parameters supported by vLLM
179
- top_k: Optional[int] = -1
180
- ignore_eos: Optional[bool] = False
218
+
219
+ # doc: begin-completion-sampling-params
220
use_beam_search: Optional[bool] = False
221
+ top_k: Optional[int] = -1
222
+ min_p: Optional[float] = 0.0
223
+ repetition_penalty: Optional[float] = 1.0
224
+ length_penalty: Optional[float] = 1.0
225
early_stopping: Optional[bool] = False
226
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
227
+ ignore_eos: Optional[bool] = False
228
skip_special_tokens: Optional[bool] = True
229
spaces_between_special_tokens: Optional[bool] = True
186
- repetition_penalty: Optional[float] = 1.0
187
- min_p: Optional[float] = 0.0
188
- include_stop_str_in_output: Optional[bool] = False
189
- length_penalty: Optional[float] = 1.0
190
- guided_json: Optional[Union[str, dict, BaseModel]] = None
191
- guided_regex: Optional[str] = None
192
- guided_choice: Optional[List[str]] = None
193
- guided_grammar: Optional[str] = None
194
- response_format: Optional[ResponseFormat] = None
230
+ # doc: end-completion-sampling-params
231
+
232
+ # doc: begin-completion-extra-params
233
+ include_stop_str_in_output: Optional[bool] = Field(
234
+ default=False,
235
+ description=(
236
+ "Whether to include the stop string in the output. "
237
+ "This is only applied when the stop or stop_token_ids is set."),
238
+ )
239
+ response_format: Optional[ResponseFormat] = Field(
240
+ default=None,
241
+ description=
242
+ ("Similar to chat completion, this parameter specifies the format of "
243
+ "output. Only {'type': 'json_object'} or {'type': 'text' } is "
244
+ "supported."),
245
+ )
246
+ guided_json: Optional[Union[str, dict, BaseModel]] = Field(
247
+ default=None,
248
+ description=("If specified, the output will follow the JSON schema."),
249
+ )
250
+ guided_regex: Optional[str] = Field(
251
+ default=None,
252
+ description=(
253
+ "If specified, the output will follow the regex pattern."),
254
+ )
255
+ guided_choice: Optional[List[str]] = Field(
256
+ default=None,
257
+ description=(
258
+ "If specified, the output will be exactly one of the choices."),
259
+ )
260
+ guided_grammar: Optional[str] = Field(
261
+ default=None,
262
+ description=(
263
+ "If specified, the output will follow the context free grammar."),
264
+ )
265
+
266
+ # doc: end-completion-extra-params
267
268
def to_sampling_params(self):
269
echo_without_generation = self.echo and self.max_tokens == 0