@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef65dcfa6f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Doc] Add docs about OpenAI compatible server (#3288)

Simon Mo committed Mar 18, 2024 at 22:05 UTC ef65dcfa6f5820ce9e4a2411e9be18586f6fd467
10 files changed +383 -161
docs/requirements-docs.txt
+7
@@ -1,3 +1,10 @@
1 sphinx == 6.2.1
2 sphinx-book-theme == 1.0.1
3 sphinx-copybutton == 0.5.2
4 +myst-parser == 2.0.0
5 +sphinx-argparse
6 +
7 +# packages to install to build the documentation
8 +pydantic
9 +-f https://download.pytorch.org/whl/cpu
10 +torch
\ No newline at end of file
docs/source/conf.py
+3 -1
@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
22 # -- Project information -----------------------------------------------------
23
24 project = 'vLLM'
25 -copyright = '2023, vLLM Team'
25 +copyright = '2024, vLLM Team'
26 author = 'the vLLM Team'
27
28 # -- General configuration ---------------------------------------------------
@@ -37,6 +37,8 @@ extensions = [
37 "sphinx_copybutton",
38 "sphinx.ext.autodoc",
39 "sphinx.ext.autosummary",
40 + "myst_parser",
41 + "sphinxarg.ext",
42 ]
43
44 # Add any paths that contain templates here, relative to this directory.
docs/source/dev/sampling_params.rst new
+4
@@ -0,0 +1,4 @@
1 +Sampling Params
2 +===============
3 +
4 +.. automodule:: vllm.sampling_params.SamplingParams
\ No newline at end of file
docs/source/index.rst
+4 -6
@@ -69,14 +69,11 @@ Documentation
69 :maxdepth: 1
70 :caption: Serving
71
72 - serving/distributed_serving
73 - serving/run_on_sky
74 - serving/deploying_with_kserve
75 - serving/deploying_with_triton
76 - serving/deploying_with_bentoml
72 + serving/openai_compatible_server
73 serving/deploying_with_docker
78 - serving/serving_with_langchain
74 + serving/distributed_serving
75 serving/metrics
76 + serving/integrations
77
78 .. toctree::
79 :maxdepth: 1
@@ -98,6 +95,7 @@ Documentation
95 :maxdepth: 2
96 :caption: Developer Documentation
97
98 + dev/sampling_params
99 dev/engine/engine_index
100 dev/kernel/paged_attention
101
docs/source/models/lora.rst
+1 -1
@@ -90,7 +90,7 @@ Requests can specify the LoRA adapter as if it were any other model via the ``mo
90 processed according to the server-wide LoRA configuration (i.e. in parallel with base model requests, and potentially other
91 LoRA adapter requests if they were provided and ``max_loras`` is set high enough).
92
93 -The following is an example request
93 +The following is an example request
94
95 .. code-block:: bash
96
docs/source/serving/integrations.rst new
+11
@@ -0,0 +1,11 @@
1 +Integrations
2 +------------
3 +
4 +.. toctree::
5 + :maxdepth: 1
6 +
7 + run_on_sky
8 + deploying_with_kserve
9 + deploying_with_triton
10 + deploying_with_bentoml
11 + serving_with_langchain
docs/source/serving/openai_compatible_server.md new
+114
@@ -0,0 +1,114 @@
1 +# OpenAI Compatible Server
2 +
3 +vLLM provides an HTTP server that implements OpenAI's [Completions](https://platform.openai.com/docs/api-reference/completions) and [Chat](https://platform.openai.com/docs/api-reference/chat) API.
4 +
5 +You can start the server using Python, or using [Docker](deploying_with_docker.rst):
6 +```bash
7 +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-hf --dtype float32 --api-key token-abc123
8 +```
9 +
10 +To call the server, you can use the official OpenAI Python client library, or any other HTTP client.
11 +```python
12 +from openai import OpenAI
13 +client = OpenAI(
14 + base_url="http://localhost:8000/v1",
15 + api_key="token-abc123",
16 +)
17 +
18 +completion = client.chat.completions.create(
19 + model="meta-llama/Llama-2-7b-hf",
20 + messages=[
21 + {"role": "system", "content": "You are a helpful assistant."},
22 + {"role": "user", "content": "Hello!"}
23 + ]
24 +)
25 +
26 +print(completion.choices[0].message)
27 +```
28 +
29 +## API Reference
30 +Please see the [OpenAI API Reference](https://platform.openai.com/docs/api-reference) for more information on the API. We support all parameters except:
31 +- Chat: `tools`, and `tool_choice`.
32 +- Completions: `suffix`.
33 +
34 +## Extra Parameters
35 +vLLM supports a set of parameters that are not part of the OpenAI API.
36 +In order to use them, you can pass them as extra parameters in the OpenAI client.
37 +Or directly merge them into the JSON payload if you are using HTTP call directly.
38 +
39 +```python
40 +completion = client.chat.completions.create(
41 + model="meta-llama/Llama-2-7b-hf",
42 + messages=[
43 + {"role": "system", "content": "You are a helpful assistant."},
44 + {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
45 + ],
46 + extra_body={
47 + "guided_choice": ["positive", "negative"]
48 + }
49 +)
50 +```
51 +
52 +### Extra Parameters for Chat API
53 +The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
54 +
55 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
56 +:language: python
57 +:start-after: begin-chat-completion-sampling-params
58 +:end-before: end-chat-completion-sampling-params
59 +```
60 +
61 +The following extra parameters are supported:
62 +
63 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
64 +:language: python
65 +:start-after: begin-chat-completion-extra-params
66 +:end-before: end-chat-completion-extra-params
67 +```
68 +
69 +### Extra Parameters for Completions API
70 +The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
71 +
72 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
73 +:language: python
74 +:start-after: begin-completion-sampling-params
75 +:end-before: end-completion-sampling-params
76 +```
77 +
78 +The following extra parameters are supported:
79 +
80 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
81 +:language: python
82 +:start-after: begin-completion-extra-params
83 +:end-before: end-completion-extra-params
84 +```
85 +
86 +## Chat Template
87 +
88 +In order for the language model to support chat protocol, vLLM requires the model to include
89 +a chat template in its tokenizer configuration. The chat template is a Jinja2 template that
90 +specifies how are roles, messages, and other chat-specific tokens are encoded in the input.
91 +
92 +An example chat template for `meta-llama/Llama-2-7b-chat-hf` can be found [here](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/blob/09bd0f49e16738cdfaa6e615203e126038736eb0/tokenizer_config.json#L12)
93 +
94 +Some models do not provide a chat template even though they are instruction/chat fine-tuned. For those model,
95 +you can manually specify their chat template in the `--chat-template` parameter with the file path to the chat
96 +template, or the template in string form. Without a chat template, the server will not be able to process chat
97 +and all chat requests will error.
98 +
99 +```bash
100 +python -m vllm.entrypoints.openai.api_server \
101 + --model ... \
102 + --chat-template ./path-to-chat-template.jinja
103 +```
104 +
105 +vLLM community provides a set of chat templates for popular models. You can find them in the examples
106 +directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
107 +
108 +## Command line arguments for the server
109 +
110 +```{argparse}
111 +:module: vllm.entrypoints.openai.cli_args
112 +:func: make_arg_parser
113 +:prog: vllm-openai-server
114 +```
\ No newline at end of file
vllm/entrypoints/openai/api_server.py
+2 -106
@@ -1,11 +1,8 @@
1 -import argparse
1 import asyncio
3 -import json
2 from contextlib import asynccontextmanager
3 import os
4 import importlib
5 import inspect
8 -import ssl
6
7 from prometheus_client import make_asgi_app
8 import fastapi
@@ -23,9 +20,9 @@ from vllm.entrypoints.openai.protocol import (CompletionRequest,
20 ChatCompletionRequest,
21 ErrorResponse)
22 from vllm.logger import init_logger
23 +from vllm.entrypoints.openai.cli_args import make_arg_parser
24 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
25 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
28 -from vllm.entrypoints.openai.serving_engine import LoRA
26
27 TIMEOUT_KEEP_ALIVE = 5 # seconds
28
@@ -51,109 +48,8 @@ async def lifespan(app: fastapi.FastAPI):
48 app = fastapi.FastAPI(lifespan=lifespan)
49
50
54 -class LoRAParserAction(argparse.Action):
55 -
56 - def __call__(self, parser, namespace, values, option_string=None):
57 - lora_list = []
58 - for item in values:
59 - name, path = item.split('=')
60 - lora_list.append(LoRA(name, path))
61 - setattr(namespace, self.dest, lora_list)
62 -
63 -
51 def parse_args():
65 - parser = argparse.ArgumentParser(
66 - description="vLLM OpenAI-Compatible RESTful API server.")
67 - parser.add_argument("--host", type=str, default=None, help="host name")
68 - parser.add_argument("--port", type=int, default=8000, help="port number")
69 - parser.add_argument(
70 - "--uvicorn-log-level",
71 - type=str,
72 - default="info",
73 - choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'],
74 - help="log level for uvicorn")
75 - parser.add_argument("--allow-credentials",
76 - action="store_true",
77 - help="allow credentials")
78 - parser.add_argument("--allowed-origins",
79 - type=json.loads,
80 - default=["*"],
81 - help="allowed origins")
82 - parser.add_argument("--allowed-methods",
83 - type=json.loads,
84 - default=["*"],
85 - help="allowed methods")
86 - parser.add_argument("--allowed-headers",
87 - type=json.loads,
88 - default=["*"],
89 - help="allowed headers")
90 - parser.add_argument("--api-key",
91 - type=str,
92 - default=None,
93 - help="If provided, the server will require this key "
94 - "to be presented in the header.")
95 - parser.add_argument("--served-model-name",
96 - type=str,
97 - default=None,
98 - help="The model name used in the API. If not "
99 - "specified, the model name will be the same as "
100 - "the huggingface name.")
101 - parser.add_argument(
102 - "--lora-modules",
103 - type=str,
104 - default=None,
105 - nargs='+',
106 - action=LoRAParserAction,
107 - help="LoRA module configurations in the format name=path. "
108 - "Multiple modules can be specified.")
109 - parser.add_argument("--chat-template",
110 - type=str,
111 - default=None,
112 - help="The file path to the chat template, "
113 - "or the template in single-line form "
114 - "for the specified model")
115 - parser.add_argument("--response-role",
116 - type=str,
117 - default="assistant",
118 - help="The role name to return if "
119 - "`request.add_generation_prompt=true`.")
120 - parser.add_argument("--ssl-keyfile",
121 - type=str,
122 - default=None,
123 - help="The file path to the SSL key file")
124 - parser.add_argument("--ssl-certfile",
125 - type=str,
126 - default=None,
127 - help="The file path to the SSL cert file")
128 - parser.add_argument("--ssl-ca-certs",
129 - type=str,
130 - default=None,
131 - help="The CA certificates file")
132 - parser.add_argument(
133 - "--ssl-cert-reqs",
134 - type=int,
135 - default=int(ssl.CERT_NONE),
136 - help="Whether client certificate is required (see stdlib ssl module's)"
137 - )
138 - parser.add_argument(
139 - "--root-path",
140 - type=str,
141 - default=None,
142 - help="FastAPI root_path when app is behind a path based routing proxy")
143 - parser.add_argument(
144 - "--middleware",
145 - type=str,
146 - action="append",
147 - default=[],
148 - help="Additional ASGI middleware to apply to the app. "
149 - "We accept multiple --middleware arguments. "
150 - "The value should be an import path. "
151 - "If a function is provided, vLLM will add it to the server "
152 - "using @app.middleware('http'). "
153 - "If a class is provided, vLLM will add it to the server "
154 - "using app.add_middleware(). ")
155 -
156 - parser = AsyncEngineArgs.add_cli_args(parser)
52 + parser = make_arg_parser()
53 return parser.parse_args()
54
55
vllm/entrypoints/openai/cli_args.py new
+118
@@ -0,0 +1,118 @@
1 +"""
2 +This file contains the command line arguments for the vLLM's
3 +OpenAI-compatible server. It is kept in a separate file for documentation
4 +purposes.
5 +"""
6 +
7 +import argparse
8 +import json
9 +import ssl
10 +
11 +from vllm.engine.arg_utils import AsyncEngineArgs
12 +from vllm.entrypoints.openai.serving_engine import LoRA
13 +
14 +
15 +class LoRAParserAction(argparse.Action):
16 +
17 + def __call__(self, parser, namespace, values, option_string=None):
18 + lora_list = []
19 + for item in values:
20 + name, path = item.split('=')
21 + lora_list.append(LoRA(name, path))
22 + setattr(namespace, self.dest, lora_list)
23 +
24 +
25 +def make_arg_parser():
26 + parser = argparse.ArgumentParser(
27 + description="vLLM OpenAI-Compatible RESTful API server.")
28 + parser.add_argument("--host", type=str, default=None, help="host name")
29 + parser.add_argument("--port", type=int, default=8000, help="port number")
30 + parser.add_argument(
31 + "--uvicorn-log-level",
32 + type=str,
33 + default="info",
34 + choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'],
35 + help="log level for uvicorn")
36 + parser.add_argument("--allow-credentials",
37 + action="store_true",
38 + help="allow credentials")
39 + parser.add_argument("--allowed-origins",
40 + type=json.loads,
41 + default=["*"],
42 + help="allowed origins")
43 + parser.add_argument("--allowed-methods",
44 + type=json.loads,
45 + default=["*"],
46 + help="allowed methods")
47 + parser.add_argument("--allowed-headers",
48 + type=json.loads,
49 + default=["*"],
50 + help="allowed headers")
51 + parser.add_argument("--api-key",
52 + type=str,
53 + default=None,
54 + help="If provided, the server will require this key "
55 + "to be presented in the header.")
56 + parser.add_argument("--served-model-name",
57 + type=str,
58 + default=None,
59 + help="The model name used in the API. If not "
60 + "specified, the model name will be the same as "
61 + "the huggingface name.")
62 + parser.add_argument(
63 + "--lora-modules",
64 + type=str,
65 + default=None,
66 + nargs='+',
67 + action=LoRAParserAction,
68 + help="LoRA module configurations in the format name=path. "
69 + "Multiple modules can be specified.")
70 + parser.add_argument("--chat-template",
71 + type=str,
72 + default=None,
73 + help="The file path to the chat template, "
74 + "or the template in single-line form "
75 + "for the specified model")
76 + parser.add_argument("--response-role",
77 + type=str,
78 + default="assistant",
79 + help="The role name to return if "
80 + "`request.add_generation_prompt=true`.")
81 + parser.add_argument("--ssl-keyfile",
82 + type=str,
83 + default=None,
84 + help="The file path to the SSL key file")
85 + parser.add_argument("--ssl-certfile",
86 + type=str,
87 + default=None,
88 + help="The file path to the SSL cert file")
89 + parser.add_argument("--ssl-ca-certs",
90 + type=str,
91 + default=None,
92 + help="The CA certificates file")
93 + parser.add_argument(
94 + "--ssl-cert-reqs",
95 + type=int,
96 + default=int(ssl.CERT_NONE),
97 + help="Whether client certificate is required (see stdlib ssl module's)"
98 + )
99 + parser.add_argument(
100 + "--root-path",
101 + type=str,
102 + default=None,
103 + help="FastAPI root_path when app is behind a path based routing proxy")
104 + parser.add_argument(
105 + "--middleware",
106 + type=str,
107 + action="append",
108 + default=[],
109 + help="Additional ASGI middleware to apply to the app. "
110 + "We accept multiple --middleware arguments. "
111 + "The value should be an import path. "
112 + "If a function is provided, vLLM will add it to the server "
113 + "using @app.middleware('http'). "
114 + "If a class is provided, vLLM will add it to the server "
115 + "using app.add_middleware(). ")
116 +
117 + parser = AsyncEngineArgs.add_cli_args(parser)
118 + return parser
vllm/entrypoints/openai/protocol.py
+119 -47
@@ -61,41 +61,80 @@ class ResponseFormat(BaseModel):
61
62
63 class ChatCompletionRequest(BaseModel):
64 - model: str
64 + # Ordered by official OpenAI API documentation
65 + # https://platform.openai.com/docs/api-reference/chat/create
66 messages: List[Dict[str, str]]
66 - temperature: Optional[float] = 0.7
67 - top_p: Optional[float] = 1.0
68 - n: Optional[int] = 1
67 + model: str
68 + frequency_penalty: Optional[float] = 0.0
69 + logit_bias: Optional[Dict[str, float]] = None
70 + logprobs: Optional[bool] = False
71 + top_logprobs: Optional[int] = None
72 max_tokens: Optional[int] = None
73 + n: Optional[int] = 1
74 + presence_penalty: Optional[float] = 0.0
75 + response_format: Optional[ResponseFormat] = None
76 seed: Optional[int] = None
77 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
78 stream: Optional[bool] = False
73 - logprobs: Optional[bool] = False
74 - top_logprobs: Optional[int] = None
75 - presence_penalty: Optional[float] = 0.0
76 - frequency_penalty: Optional[float] = 0.0
77 - logit_bias: Optional[Dict[str, float]] = None
79 + temperature: Optional[float] = 0.7
80 + top_p: Optional[float] = 1.0
81 user: Optional[str] = None
79 - # Additional parameters supported by vLLM
82 +
83 + # doc: begin-chat-completion-sampling-params
84 best_of: Optional[int] = None
81 - top_k: Optional[int] = -1
82 - ignore_eos: Optional[bool] = False
85 use_beam_search: Optional[bool] = False
86 + top_k: Optional[int] = -1
87 + min_p: Optional[float] = 0.0
88 + repetition_penalty: Optional[float] = 1.0
89 + length_penalty: Optional[float] = 1.0
90 early_stopping: Optional[bool] = False
91 + ignore_eos: Optional[bool] = False
92 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
93 skip_special_tokens: Optional[bool] = True
94 spaces_between_special_tokens: Optional[bool] = True
88 - add_generation_prompt: Optional[bool] = True
89 - echo: Optional[bool] = False
90 - repetition_penalty: Optional[float] = 1.0
91 - min_p: Optional[float] = 0.0
92 - include_stop_str_in_output: Optional[bool] = False
93 - length_penalty: Optional[float] = 1.0
94 - guided_json: Optional[Union[str, dict, BaseModel]] = None
95 - guided_regex: Optional[str] = None
96 - guided_choice: Optional[List[str]] = None
97 - guided_grammar: Optional[str] = None
98 - response_format: Optional[ResponseFormat] = None
95 + # doc: end-chat-completion-sampling-params
96 +
97 + # doc: begin-chat-completion-extra-params
98 + echo: Optional[bool] = Field(
99 + default=False,
100 + description=(
101 + "If true, the new message will be prepended with the last message "
102 + "if they belong to the same role."),
103 + )
104 + add_generation_prompt: Optional[bool] = Field(
105 + default=True,
106 + description=
107 + ("If true, the generation prompt will be added to the chat template. "
108 + "This is a parameter used by chat template in tokenizer config of the "
109 + "model."),
110 + )
111 + include_stop_str_in_output: Optional[bool] = Field(
112 + default=False,
113 + description=(
114 + "Whether to include the stop string in the output. "
115 + "This is only applied when the stop or stop_token_ids is set."),
116 + )
117 + guided_json: Optional[Union[str, dict, BaseModel]] = Field(
118 + default=None,
119 + description=("If specified, the output will follow the JSON schema."),
120 + )
121 + guided_regex: Optional[str] = Field(
122 + default=None,
123 + description=(
124 + "If specified, the output will follow the regex pattern."),
125 + )
126 + guided_choice: Optional[List[str]] = Field(
127 + default=None,
128 + description=(
129 + "If specified, the output will be exactly one of the choices."),
130 + )
131 + guided_grammar: Optional[str] = Field(
132 + default=None,
133 + description=(
134 + "If specified, the output will follow the context free grammar."),
135 + )
136 +
137 + # doc: end-chat-completion-extra-params
138
139 def to_sampling_params(self) -> SamplingParams:
140 if self.logprobs and not self.top_logprobs:
@@ -157,41 +196,74 @@ class ChatCompletionRequest(BaseModel):
196
197
198 class CompletionRequest(BaseModel):
199 + # Ordered by official OpenAI API documentation
200 + # https://platform.openai.com/docs/api-reference/completions/create
201 model: str
161 - # a string, array of strings, array of tokens, or array of token arrays
202 prompt: Union[List[int], List[List[int]], str, List[str]]
163 - suffix: Optional[str] = None
164 - max_tokens: Optional[int] = 16
165 - temperature: Optional[float] = 1.0
166 - top_p: Optional[float] = 1.0
167 - n: Optional[int] = 1
168 - stream: Optional[bool] = False
169 - logprobs: Optional[int] = None
203 + best_of: Optional[int] = None
204 echo: Optional[bool] = False
171 - stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
172 - seed: Optional[int] = None
173 - presence_penalty: Optional[float] = 0.0
205 frequency_penalty: Optional[float] = 0.0
175 - best_of: Optional[int] = None
206 logit_bias: Optional[Dict[str, float]] = None
207 + logprobs: Optional[int] = None
208 + max_tokens: Optional[int] = 16
209 + n: Optional[int] = 1
210 + presence_penalty: Optional[float] = 0.0
211 + seed: Optional[int] = None
212 + stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
213 + stream: Optional[bool] = False
214 + suffix: Optional[str] = None
215 + temperature: Optional[float] = 1.0
216 + top_p: Optional[float] = 1.0
217 user: Optional[str] = None
178 - # Additional parameters supported by vLLM
179 - top_k: Optional[int] = -1
180 - ignore_eos: Optional[bool] = False
218 +
219 + # doc: begin-completion-sampling-params
220 use_beam_search: Optional[bool] = False
221 + top_k: Optional[int] = -1
222 + min_p: Optional[float] = 0.0
223 + repetition_penalty: Optional[float] = 1.0
224 + length_penalty: Optional[float] = 1.0
225 early_stopping: Optional[bool] = False
226 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
227 + ignore_eos: Optional[bool] = False
228 skip_special_tokens: Optional[bool] = True
229 spaces_between_special_tokens: Optional[bool] = True
186 - repetition_penalty: Optional[float] = 1.0
187 - min_p: Optional[float] = 0.0
188 - include_stop_str_in_output: Optional[bool] = False
189 - length_penalty: Optional[float] = 1.0
190 - guided_json: Optional[Union[str, dict, BaseModel]] = None
191 - guided_regex: Optional[str] = None
192 - guided_choice: Optional[List[str]] = None
193 - guided_grammar: Optional[str] = None
194 - response_format: Optional[ResponseFormat] = None
230 + # doc: end-completion-sampling-params
231 +
232 + # doc: begin-completion-extra-params
233 + include_stop_str_in_output: Optional[bool] = Field(
234 + default=False,
235 + description=(
236 + "Whether to include the stop string in the output. "
237 + "This is only applied when the stop or stop_token_ids is set."),
238 + )
239 + response_format: Optional[ResponseFormat] = Field(
240 + default=None,
241 + description=
242 + ("Similar to chat completion, this parameter specifies the format of "
243 + "output. Only {'type': 'json_object'} or {'type': 'text' } is "
244 + "supported."),
245 + )
246 + guided_json: Optional[Union[str, dict, BaseModel]] = Field(
247 + default=None,
248 + description=("If specified, the output will follow the JSON schema."),
249 + )
250 + guided_regex: Optional[str] = Field(
251 + default=None,
252 + description=(
253 + "If specified, the output will follow the regex pattern."),
254 + )
255 + guided_choice: Optional[List[str]] = Field(
256 + default=None,
257 + description=(
258 + "If specified, the output will be exactly one of the choices."),
259 + )
260 + guided_grammar: Optional[str] = Field(
261 + default=None,
262 + description=(
263 + "If specified, the output will follow the context free grammar."),
264 + )
265 +
266 + # doc: end-completion-extra-params
267
268 def to_sampling_params(self):
269 echo_without_generation = self.echo and self.max_tokens == 0