A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Doc]: Update `OpenAI-Compatible Server` documents (#12082)
maang-h committed
Jan 16, 2025 at 00:07 UTC
57e729e87478d734e8d0075e35aeb4c9bd440e77
2 files changed
+38
-38
vllm/engine/arg_utils.py
+8
-8
@@ -238,7 +238,7 @@ class EngineArgs:
238
choices=get_args(TaskOption),
239
help='The task to use the model for. Each vLLM instance only '
240
'supports one task, even if the same model can be used for '
241
- 'multiple tasks. When the model only supports one task, "auto" '
241
+ 'multiple tasks. When the model only supports one task, ``"auto"`` '
242
'can be used to select it; otherwise, you must specify explicitly '
243
'which task to use.')
244
parser.add_argument(
@@ -250,7 +250,7 @@ class EngineArgs:
250
parser.add_argument(
251
'--skip-tokenizer-init',
252
action='store_true',
253
- help='Skip initialization of tokenizer and detokenizer')
253
+ help='Skip initialization of tokenizer and detokenizer.')
254
parser.add_argument(
255
'--revision',
256
type=nullable_str,
@@ -401,7 +401,7 @@ class EngineArgs:
401
parser.add_argument(
402
'--worker-use-ray',
403
action='store_true',
404
- help='Deprecated, use --distributed-executor-backend=ray.')
404
+ help='Deprecated, use ``--distributed-executor-backend=ray``.')
405
parser.add_argument('--pipeline-parallel-size',
406
'-pp',
407
type=int,
@@ -430,7 +430,7 @@ class EngineArgs:
430
choices=[8, 16, 32, 64, 128],
431
help='Token block size for contiguous chunks of '
432
'tokens. This is ignored on neuron devices and '
433
- 'set to max-model-len. On CUDA devices, '
433
+ 'set to ``--max-model-len``. On CUDA devices, '
434
'only block sizes up to 32 are supported. '
435
'On HPU devices, block size defaults to 128.')
436
@@ -439,12 +439,12 @@ class EngineArgs:
439
action=argparse.BooleanOptionalAction,
440
default=EngineArgs.enable_prefix_caching,
441
help="Enables automatic prefix caching. "
442
- "Use --no-enable-prefix-caching to disable explicitly.",
442
+ "Use ``--no-enable-prefix-caching`` to disable explicitly.",
443
)
444
parser.add_argument('--disable-sliding-window',
445
action='store_true',
446
help='Disables sliding window, '
447
- 'capping to sliding window size')
447
+ 'capping to sliding window size.')
448
parser.add_argument('--use-v2-block-manager',
449
action='store_true',
450
default=True,
@@ -861,7 +861,7 @@ class EngineArgs:
861
"of the provided names. The model name in the model "
862
"field of a response will be the first name in this "
863
"list. If not specified, the model name will be the "
864
- "same as the `--model` argument. Noted that this name(s) "
864
+ "same as the ``--model`` argument. Noted that this name(s) "
865
"will also be used in `model_name` tag content of "
866
"prometheus metrics, if multiple names provided, metrics "
867
"tag will take the first one.")
@@ -881,7 +881,7 @@ class EngineArgs:
881
default=None,
882
help="Valid choices are " +
883
",".join(ALLOWED_DETAILED_TRACE_MODULES) +
884
- ". It makes sense to set this only if --otlp-traces-endpoint is"
884
+ ". It makes sense to set this only if ``--otlp-traces-endpoint`` is"
885
" set. If set, it will collect detailed traces for the specified "
886
"modules. This involves use of possibly costly and or blocking "
887
"operations and hence might have a performance impact.")
vllm/entrypoints/openai/cli_args.py
+30
-30
@@ -79,29 +79,29 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
79
parser.add_argument("--host",
80
type=nullable_str,
81
default=None,
82
- help="host name")
83
- parser.add_argument("--port", type=int, default=8000, help="port number")
82
+ help="Host name.")
83
+ parser.add_argument("--port", type=int, default=8000, help="Port number.")
84
parser.add_argument(
85
"--uvicorn-log-level",
86
type=str,
87
default="info",
88
choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'],
89
- help="log level for uvicorn")
89
+ help="Log level for uvicorn.")
90
parser.add_argument("--allow-credentials",
91
action="store_true",
92
- help="allow credentials")
92
+ help="Allow credentials.")
93
parser.add_argument("--allowed-origins",
94
type=json.loads,
95
default=["*"],
96
- help="allowed origins")
96
+ help="Allowed origins.")
97
parser.add_argument("--allowed-methods",
98
type=json.loads,
99
default=["*"],
100
- help="allowed methods")
100
+ help="Allowed methods.")
101
parser.add_argument("--allowed-headers",
102
type=json.loads,
103
default=["*"],
104
- help="allowed headers")
104
+ help="Allowed headers.")
105
parser.add_argument("--api-key",
106
type=nullable_str,
107
default=None,
@@ -115,10 +115,10 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
115
action=LoRAParserAction,
116
help="LoRA module configurations in either 'name=path' format"
117
"or JSON format. "
118
- "Example (old format): 'name=path' "
118
+ "Example (old format): ``'name=path'`` "
119
"Example (new format): "
120
- "'{\"name\": \"name\", \"local_path\": \"path\", "
121
- "\"base_model_name\": \"id\"}'")
120
+ "``{\"name\": \"name\", \"local_path\": \"path\", "
121
+ "\"base_model_name\": \"id\"}``")
122
parser.add_argument(
123
"--prompt-adapters",
124
type=nullable_str,
@@ -132,7 +132,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
132
default=None,
133
help="The file path to the chat template, "
134
"or the template in single-line form "
135
- "for the specified model")
135
+ "for the specified model.")
136
parser.add_argument(
137
'--chat-template-content-format',
138
type=str,
@@ -141,38 +141,39 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
141
help='The format to render message content within a chat template.'
142
'\n\n'
143
'* "string" will render the content as a string. '
144
- 'Example: "Hello World"\n'
144
+ 'Example: ``"Hello World"``\n'
145
'* "openai" will render the content as a list of dictionaries, '
146
'similar to OpenAI schema. '
147
- 'Example: [{"type": "text", "text": "Hello world!"}]')
147
+ 'Example: ``[{"type": "text", "text": "Hello world!"}]``')
148
parser.add_argument("--response-role",
149
type=nullable_str,
150
default="assistant",
151
help="The role name to return if "
152
- "`request.add_generation_prompt=true`.")
152
+ "``request.add_generation_prompt=true``.")
153
parser.add_argument("--ssl-keyfile",
154
type=nullable_str,
155
default=None,
156
- help="The file path to the SSL key file")
156
+ help="The file path to the SSL key file.")
157
parser.add_argument("--ssl-certfile",
158
type=nullable_str,
159
default=None,
160
- help="The file path to the SSL cert file")
160
+ help="The file path to the SSL cert file.")
161
parser.add_argument("--ssl-ca-certs",
162
type=nullable_str,
163
default=None,
164
- help="The CA certificates file")
164
+ help="The CA certificates file.")
165
parser.add_argument(
166
"--ssl-cert-reqs",
167
type=int,
168
default=int(ssl.CERT_NONE),
169
- help="Whether client certificate is required (see stdlib ssl module's)"
169
+ help="Whether client certificate is required (see stdlib ssl module's)."
170
)
171
parser.add_argument(
172
"--root-path",
173
type=nullable_str,
174
default=None,
175
- help="FastAPI root_path when app is behind a path based routing proxy")
175
+ help="FastAPI root_path when app is behind a path based routing proxy."
176
+ )
177
parser.add_argument(
178
"--middleware",
179
type=nullable_str,
@@ -182,15 +183,15 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
183
"We accept multiple --middleware arguments. "
184
"The value should be an import path. "
185
"If a function is provided, vLLM will add it to the server "
185
- "using @app.middleware('http'). "
186
+ "using ``@app.middleware('http')``. "
187
"If a class is provided, vLLM will add it to the server "
187
- "using app.add_middleware(). ")
188
+ "using ``app.add_middleware()``. ")
189
parser.add_argument(
190
"--return-tokens-as-token-ids",
191
action="store_true",
191
- help="When --max-logprobs is specified, represents single tokens as "
192
- "strings of the form 'token_id:{token_id}' so that tokens that "
193
- "are not JSON-encodable can be identified.")
192
+ help="When ``--max-logprobs`` is specified, represents single tokens "
193
+ " as strings of the form 'token_id:{token_id}' so that tokens "
194
+ "that are not JSON-encodable can be identified.")
195
parser.add_argument(
196
"--disable-frontend-multiprocessing",
197
action="store_true",
@@ -205,9 +206,8 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
206
"--enable-auto-tool-choice",
207
action="store_true",
208
default=False,
208
- help=
209
- "Enable auto tool choice for supported models. Use --tool-call-parser"
210
- " to specify which parser to use")
209
+ help="Enable auto tool choice for supported models. Use "
210
+ "``--tool-call-parser`` to specify which parser to use.")
211
212
valid_tool_parsers = ToolParserManager.tool_parsers.keys()
213
parser.add_argument(
@@ -219,7 +219,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
219
help=
220
"Select the tool call parser depending on the model that you're using."
221
" This is used to parse the model-generated tool call into OpenAI API "
222
- "format. Required for --enable-auto-tool-choice.")
222
+ "format. Required for ``--enable-auto-tool-choice``.")
223
224
parser.add_argument(
225
"--tool-parser-plugin",
@@ -228,7 +228,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
228
help=
229
"Special the tool parser plugin write to parse the model-generated tool"
230
" into OpenAI API format, the name register in this plugin can be used "
231
- "in --tool-call-parser.")
231
+ "in ``--tool-call-parser``.")
232
233
parser = AsyncEngineArgs.add_cli_args(parser)
234
@@ -243,7 +243,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
243
"--disable-fastapi-docs",
244
action='store_true',
245
default=False,
246
- help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint"
246
+ help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."
247
)
248
parser.add_argument(
249
"--enable-prompt-tokens-details",