@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 57e729e874
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Doc]: Update `OpenAI-Compatible Server` documents (#12082)

maang-h committed Jan 16, 2025 at 00:07 UTC 57e729e87478d734e8d0075e35aeb4c9bd440e77
2 files changed +38 -38
vllm/engine/arg_utils.py
+8 -8
@@ -238,7 +238,7 @@ class EngineArgs:
238 choices=get_args(TaskOption),
239 help='The task to use the model for. Each vLLM instance only '
240 'supports one task, even if the same model can be used for '
241 - 'multiple tasks. When the model only supports one task, "auto" '
241 + 'multiple tasks. When the model only supports one task, ``"auto"`` '
242 'can be used to select it; otherwise, you must specify explicitly '
243 'which task to use.')
244 parser.add_argument(
@@ -250,7 +250,7 @@ class EngineArgs:
250 parser.add_argument(
251 '--skip-tokenizer-init',
252 action='store_true',
253 - help='Skip initialization of tokenizer and detokenizer')
253 + help='Skip initialization of tokenizer and detokenizer.')
254 parser.add_argument(
255 '--revision',
256 type=nullable_str,
@@ -401,7 +401,7 @@ class EngineArgs:
401 parser.add_argument(
402 '--worker-use-ray',
403 action='store_true',
404 - help='Deprecated, use --distributed-executor-backend=ray.')
404 + help='Deprecated, use ``--distributed-executor-backend=ray``.')
405 parser.add_argument('--pipeline-parallel-size',
406 '-pp',
407 type=int,
@@ -430,7 +430,7 @@ class EngineArgs:
430 choices=[8, 16, 32, 64, 128],
431 help='Token block size for contiguous chunks of '
432 'tokens. This is ignored on neuron devices and '
433 - 'set to max-model-len. On CUDA devices, '
433 + 'set to ``--max-model-len``. On CUDA devices, '
434 'only block sizes up to 32 are supported. '
435 'On HPU devices, block size defaults to 128.')
436
@@ -439,12 +439,12 @@ class EngineArgs:
439 action=argparse.BooleanOptionalAction,
440 default=EngineArgs.enable_prefix_caching,
441 help="Enables automatic prefix caching. "
442 - "Use --no-enable-prefix-caching to disable explicitly.",
442 + "Use ``--no-enable-prefix-caching`` to disable explicitly.",
443 )
444 parser.add_argument('--disable-sliding-window',
445 action='store_true',
446 help='Disables sliding window, '
447 - 'capping to sliding window size')
447 + 'capping to sliding window size.')
448 parser.add_argument('--use-v2-block-manager',
449 action='store_true',
450 default=True,
@@ -861,7 +861,7 @@ class EngineArgs:
861 "of the provided names. The model name in the model "
862 "field of a response will be the first name in this "
863 "list. If not specified, the model name will be the "
864 - "same as the `--model` argument. Noted that this name(s) "
864 + "same as the ``--model`` argument. Noted that this name(s) "
865 "will also be used in `model_name` tag content of "
866 "prometheus metrics, if multiple names provided, metrics "
867 "tag will take the first one.")
@@ -881,7 +881,7 @@ class EngineArgs:
881 default=None,
882 help="Valid choices are " +
883 ",".join(ALLOWED_DETAILED_TRACE_MODULES) +
884 - ". It makes sense to set this only if --otlp-traces-endpoint is"
884 + ". It makes sense to set this only if ``--otlp-traces-endpoint`` is"
885 " set. If set, it will collect detailed traces for the specified "
886 "modules. This involves use of possibly costly and or blocking "
887 "operations and hence might have a performance impact.")
vllm/entrypoints/openai/cli_args.py
+30 -30
@@ -79,29 +79,29 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
79 parser.add_argument("--host",
80 type=nullable_str,
81 default=None,
82 - help="host name")
83 - parser.add_argument("--port", type=int, default=8000, help="port number")
82 + help="Host name.")
83 + parser.add_argument("--port", type=int, default=8000, help="Port number.")
84 parser.add_argument(
85 "--uvicorn-log-level",
86 type=str,
87 default="info",
88 choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'],
89 - help="log level for uvicorn")
89 + help="Log level for uvicorn.")
90 parser.add_argument("--allow-credentials",
91 action="store_true",
92 - help="allow credentials")
92 + help="Allow credentials.")
93 parser.add_argument("--allowed-origins",
94 type=json.loads,
95 default=["*"],
96 - help="allowed origins")
96 + help="Allowed origins.")
97 parser.add_argument("--allowed-methods",
98 type=json.loads,
99 default=["*"],
100 - help="allowed methods")
100 + help="Allowed methods.")
101 parser.add_argument("--allowed-headers",
102 type=json.loads,
103 default=["*"],
104 - help="allowed headers")
104 + help="Allowed headers.")
105 parser.add_argument("--api-key",
106 type=nullable_str,
107 default=None,
@@ -115,10 +115,10 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
115 action=LoRAParserAction,
116 help="LoRA module configurations in either 'name=path' format"
117 "or JSON format. "
118 - "Example (old format): 'name=path' "
118 + "Example (old format): ``'name=path'`` "
119 "Example (new format): "
120 - "'{\"name\": \"name\", \"local_path\": \"path\", "
121 - "\"base_model_name\": \"id\"}'")
120 + "``{\"name\": \"name\", \"local_path\": \"path\", "
121 + "\"base_model_name\": \"id\"}``")
122 parser.add_argument(
123 "--prompt-adapters",
124 type=nullable_str,
@@ -132,7 +132,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
132 default=None,
133 help="The file path to the chat template, "
134 "or the template in single-line form "
135 - "for the specified model")
135 + "for the specified model.")
136 parser.add_argument(
137 '--chat-template-content-format',
138 type=str,
@@ -141,38 +141,39 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
141 help='The format to render message content within a chat template.'
142 '\n\n'
143 '* "string" will render the content as a string. '
144 - 'Example: "Hello World"\n'
144 + 'Example: ``"Hello World"``\n'
145 '* "openai" will render the content as a list of dictionaries, '
146 'similar to OpenAI schema. '
147 - 'Example: [{"type": "text", "text": "Hello world!"}]')
147 + 'Example: ``[{"type": "text", "text": "Hello world!"}]``')
148 parser.add_argument("--response-role",
149 type=nullable_str,
150 default="assistant",
151 help="The role name to return if "
152 - "`request.add_generation_prompt=true`.")
152 + "``request.add_generation_prompt=true``.")
153 parser.add_argument("--ssl-keyfile",
154 type=nullable_str,
155 default=None,
156 - help="The file path to the SSL key file")
156 + help="The file path to the SSL key file.")
157 parser.add_argument("--ssl-certfile",
158 type=nullable_str,
159 default=None,
160 - help="The file path to the SSL cert file")
160 + help="The file path to the SSL cert file.")
161 parser.add_argument("--ssl-ca-certs",
162 type=nullable_str,
163 default=None,
164 - help="The CA certificates file")
164 + help="The CA certificates file.")
165 parser.add_argument(
166 "--ssl-cert-reqs",
167 type=int,
168 default=int(ssl.CERT_NONE),
169 - help="Whether client certificate is required (see stdlib ssl module's)"
169 + help="Whether client certificate is required (see stdlib ssl module's)."
170 )
171 parser.add_argument(
172 "--root-path",
173 type=nullable_str,
174 default=None,
175 - help="FastAPI root_path when app is behind a path based routing proxy")
175 + help="FastAPI root_path when app is behind a path based routing proxy."
176 + )
177 parser.add_argument(
178 "--middleware",
179 type=nullable_str,
@@ -182,15 +183,15 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
183 "We accept multiple --middleware arguments. "
184 "The value should be an import path. "
185 "If a function is provided, vLLM will add it to the server "
185 - "using @app.middleware('http'). "
186 + "using ``@app.middleware('http')``. "
187 "If a class is provided, vLLM will add it to the server "
187 - "using app.add_middleware(). ")
188 + "using ``app.add_middleware()``. ")
189 parser.add_argument(
190 "--return-tokens-as-token-ids",
191 action="store_true",
191 - help="When --max-logprobs is specified, represents single tokens as "
192 - "strings of the form 'token_id:{token_id}' so that tokens that "
193 - "are not JSON-encodable can be identified.")
192 + help="When ``--max-logprobs`` is specified, represents single tokens "
193 + " as strings of the form 'token_id:{token_id}' so that tokens "
194 + "that are not JSON-encodable can be identified.")
195 parser.add_argument(
196 "--disable-frontend-multiprocessing",
197 action="store_true",
@@ -205,9 +206,8 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
206 "--enable-auto-tool-choice",
207 action="store_true",
208 default=False,
208 - help=
209 - "Enable auto tool choice for supported models. Use --tool-call-parser"
210 - " to specify which parser to use")
209 + help="Enable auto tool choice for supported models. Use "
210 + "``--tool-call-parser`` to specify which parser to use.")
211
212 valid_tool_parsers = ToolParserManager.tool_parsers.keys()
213 parser.add_argument(
@@ -219,7 +219,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
219 help=
220 "Select the tool call parser depending on the model that you're using."
221 " This is used to parse the model-generated tool call into OpenAI API "
222 - "format. Required for --enable-auto-tool-choice.")
222 + "format. Required for ``--enable-auto-tool-choice``.")
223
224 parser.add_argument(
225 "--tool-parser-plugin",
@@ -228,7 +228,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
228 help=
229 "Special the tool parser plugin write to parse the model-generated tool"
230 " into OpenAI API format, the name register in this plugin can be used "
231 - "in --tool-call-parser.")
231 + "in ``--tool-call-parser``.")
232
233 parser = AsyncEngineArgs.add_cli_args(parser)
234
@@ -243,7 +243,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
243 "--disable-fastapi-docs",
244 action='store_true',
245 default=False,
246 - help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint"
246 + help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."
247 )
248 parser.add_argument(
249 "--enable-prompt-tokens-details",