f148c44c6a · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -10,9 +10,13 @@ import argparse

10 import json

11 import ssl

12 from collections.abc import Sequence

13 -from typing import Optional, Union, get_args

13 +from dataclasses import field

14 +from typing import Literal, Optional, Union

15 +

16 +from pydantic.dataclasses import dataclass

17

18 import vllm.envs as envs

19 +from vllm.config import config

20 from vllm.engine.arg_utils import AsyncEngineArgs, optional_type

21 from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,

22 validate_chat_template)

@@ -82,220 +86,171 @@ class PromptAdapterParserAction(argparse.Action):

86 setattr(namespace, self.dest, adapter_list)

87

88

89 +@config

90 +@dataclass

91 +class FrontendArgs:

92 + """Arguments for the OpenAI-compatible frontend server."""

93 + host: Optional[str] = None

94 + """Host name."""

95 + port: int = 8000

96 + """Port number."""

97 + uvicorn_log_level: Literal["debug", "info", "warning", "error", "critical",

98 + "trace"] = "info"

99 + """Log level for uvicorn."""

100 + disable_uvicorn_access_log: bool = False

101 + """Disable uvicorn access log."""

102 + allow_credentials: bool = False

103 + """Allow credentials."""

104 + allowed_origins: list[str] = field(default_factory=lambda: ["*"])

105 + """Allowed origins."""

106 + allowed_methods: list[str] = field(default_factory=lambda: ["*"])

107 + """Allowed methods."""

108 + allowed_headers: list[str] = field(default_factory=lambda: ["*"])

109 + """Allowed headers."""

110 + api_key: Optional[str] = None

111 + """If provided, the server will require this key to be presented in the

112 + header."""

113 + lora_modules: Optional[list[LoRAModulePath]] = None

114 + """LoRA modules configurations in either 'name=path' format or JSON format

115 + or JSON list format. Example (old format): `'name=path'` Example (new

116 + format): `{\"name\": \"name\", \"path\": \"lora_path\",

117 + \"base_model_name\": \"id\"}`"""

118 + prompt_adapters: Optional[list[PromptAdapterPath]] = None

119 + """Prompt adapter configurations in the format name=path. Multiple adapters

120 + can be specified."""

121 + chat_template: Optional[str] = None

122 + """The file path to the chat template, or the template in single-line form

123 + for the specified model."""

124 + chat_template_content_format: ChatTemplateContentFormatOption = "auto"

125 + """The format to render message content within a chat template.

126 +

127 +* "string" will render the content as a string. Example: `"Hello World"`

128 +* "openai" will render the content as a list of dictionaries, similar to OpenAI

129 +schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""

130 + response_role: str = "assistant"

131 + """The role name to return if `request.add_generation_prompt=true`."""

132 + ssl_keyfile: Optional[str] = None

133 + """The file path to the SSL key file."""

134 + ssl_certfile: Optional[str] = None

135 + """The file path to the SSL cert file."""

136 + ssl_ca_certs: Optional[str] = None

137 + """The CA certificates file."""

138 + enable_ssl_refresh: bool = False

139 + """Refresh SSL Context when SSL certificate files change"""

140 + ssl_cert_reqs: int = int(ssl.CERT_NONE)

141 + """Whether client certificate is required (see stdlib ssl module's)."""

142 + root_path: Optional[str] = None

143 + """FastAPI root_path when app is behind a path based routing proxy."""

144 + middleware: list[str] = field(default_factory=lambda: [])

145 + """Additional ASGI middleware to apply to the app. We accept multiple

146 + --middleware arguments. The value should be an import path. If a function

147 + is provided, vLLM will add it to the server using

148 + `@app.middleware('http')`. If a class is provided, vLLM will

149 + add it to the server using `app.add_middleware()`."""

150 + return_tokens_as_token_ids: bool = False

151 + """When `--max-logprobs` is specified, represents single tokens as

152 + strings of the form 'token_id:{token_id}' so that tokens that are not

153 + JSON-encodable can be identified."""

154 + disable_frontend_multiprocessing: bool = False

155 + """If specified, will run the OpenAI frontend server in the same process as

156 + the model serving engine."""

157 + enable_request_id_headers: bool = False

158 + """If specified, API server will add X-Request-Id header to responses.

159 + Caution: this hurts performance at high QPS."""

160 + enable_auto_tool_choice: bool = False

161 + """Enable auto tool choice for supported models. Use `--tool-call-parser`

162 + to specify which parser to use."""

163 + tool_call_parser: Optional[str] = None

164 + """Select the tool call parser depending on the model that you're using.

165 + This is used to parse the model-generated tool call into OpenAI API format.

166 + Required for `--enable-auto-tool-choice`. You can choose any option from

167 + the built-in parsers or register a plugin via `--tool-parser-plugin`."""

168 + tool_parser_plugin: str = ""

169 + """Special the tool parser plugin write to parse the model-generated tool

170 + into OpenAI API format, the name register in this plugin can be used in

171 + `--tool-call-parser`."""

172 + log_config_file: Optional[str] = envs.VLLM_LOGGING_CONFIG_PATH

173 + """Path to logging config JSON file for both vllm and uvicorn"""

174 + max_log_len: Optional[int] = None

175 + """Max number of prompt characters or prompt ID numbers being printed in

176 + log. The default of None means unlimited."""

177 + disable_fastapi_docs: bool = False

178 + """Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."""

179 + enable_prompt_tokens_details: bool = False

180 + """If set to True, enable prompt_tokens_details in usage."""

181 + enable_server_load_tracking: bool = False

182 + """If set to True, enable tracking server_load_metrics in the app state."""

183 + enable_force_include_usage: bool = False

184 + """If set to True, including usage on every request."""

185 + expand_tools_even_if_tool_choice_none: bool = False

186 + """Include tool definitions in prompts even when `tool_choice='none'`.

187 +

188 + This is a transitional option that will be removed in v0.10.0. In

189 + v0.10.0, tool definitions will always be included regardless of

190 + `tool_choice` setting. Use this flag to test the upcoming behavior

191 + before the breaking change."""

192 +

193 + @staticmethod

194 + def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:

195 + from vllm.engine.arg_utils import get_kwargs

196 +

197 + frontend_kwargs = get_kwargs(FrontendArgs)

198 +

199 + # Special case: allowed_origins, allowed_methods, allowed_headers all

200 + # need json.loads type

201 + # Should also remove nargs

202 + print(frontend_kwargs["allowed_origins"])

203 + frontend_kwargs["allowed_origins"]["type"] = json.loads

204 + frontend_kwargs["allowed_methods"]["type"] = json.loads

205 + frontend_kwargs["allowed_headers"]["type"] = json.loads

206 + del frontend_kwargs["allowed_origins"]["nargs"]

207 + del frontend_kwargs["allowed_methods"]["nargs"]

208 + del frontend_kwargs["allowed_headers"]["nargs"]

209 +

210 + # Special case: LoRA modules need custom parser action and

211 + # optional_type(str)

212 + frontend_kwargs["lora_modules"]["type"] = optional_type(str)

213 + frontend_kwargs["lora_modules"]["action"] = LoRAParserAction

214 +

215 + # Special case: Prompt adapters need custom parser action and

216 + # optional_type(str)

217 + frontend_kwargs["prompt_adapters"]["type"] = optional_type(str)

218 + frontend_kwargs["prompt_adapters"][

219 + "action"] = PromptAdapterParserAction

220 +

221 + # Special case: Middleware needs append action

222 + frontend_kwargs["middleware"]["action"] = "append"

223 +

224 + # Special case: Tool call parser shows built-in options.

225 + valid_tool_parsers = list(ToolParserManager.tool_parsers.keys())

226 + frontend_kwargs["tool_call_parser"]["choices"] = valid_tool_parsers

227 +

228 + # Special case for expand-tools-even-if-tool-choice-none because of

229 + # the deprecation field

230 + frontend_kwargs["expand_tools_even_if_tool_choice_none"]\

231 + ["deprecated"] = True

232 +

233 + frontend_group = parser.add_argument_group(

234 + title="Frontend",

235 + description=FrontendArgs.__doc__,

236 + )

237 +

238 + for key, value in frontend_kwargs.items():

239 + frontend_group.add_argument(f"--{key.replace('_', '-')}", **value)

240 +

241 + return parser

242 +

243 +

244 def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:

86 - parser.add_argument("--host",

87 - type=optional_type(str),

88 - default=None,

89 - help="Host name.")

90 - parser.add_argument("--port", type=int, default=8000, help="Port number.")

91 - parser.add_argument(

92 - "--uvicorn-log-level",

93 - type=str,

94 - default="info",

95 - choices=['debug', 'info', 'warning', 'error', 'critical', 'trace'],

96 - help="Log level for uvicorn.")

97 - parser.add_argument("--disable-uvicorn-access-log",

98 - action="store_true",

99 - help="Disable uvicorn access log.")

100 - parser.add_argument("--allow-credentials",

101 - action="store_true",

102 - help="Allow credentials.")

103 - parser.add_argument("--allowed-origins",

104 - type=json.loads,

105 - default=["*"],

106 - help="Allowed origins.")

107 - parser.add_argument("--allowed-methods",

108 - type=json.loads,

109 - default=["*"],

110 - help="Allowed methods.")

111 - parser.add_argument("--allowed-headers",

112 - type=json.loads,

113 - default=["*"],

114 - help="Allowed headers.")

115 - parser.add_argument("--api-key",

116 - type=optional_type(str),

117 - default=None,

118 - help="If provided, the server will require this key "

119 - "to be presented in the header.")

120 - parser.add_argument(

121 - "--lora-modules",

122 - type=optional_type(str),

123 - default=None,

124 - nargs='+',

125 - action=LoRAParserAction,

126 - help="LoRA module configurations in either 'name=path' format"

127 - "or JSON format. "

128 - "Example (old format): ``'name=path'`` "

129 - "Example (new format): "

130 - "``{\"name\": \"name\", \"path\": \"lora_path\", "

131 - "\"base_model_name\": \"id\"}``")

132 - parser.add_argument(

133 - "--prompt-adapters",

134 - type=optional_type(str),

135 - default=None,

136 - nargs='+',

137 - action=PromptAdapterParserAction,

138 - help="Prompt adapter configurations in the format name=path. "

139 - "Multiple adapters can be specified.")

140 - parser.add_argument("--chat-template",

141 - type=optional_type(str),

142 - default=None,

143 - help="The file path to the chat template, "

144 - "or the template in single-line form "

145 - "for the specified model.")

146 - parser.add_argument(

147 - '--chat-template-content-format',

148 - type=str,

149 - default="auto",

150 - choices=get_args(ChatTemplateContentFormatOption),

151 - help='The format to render message content within a chat template.'

152 - '\n\n'

153 - '* "string" will render the content as a string. '

154 - 'Example: ``"Hello World"``\n'

155 - '* "openai" will render the content as a list of dictionaries, '

156 - 'similar to OpenAI schema. '

157 - 'Example: ``[{"type": "text", "text": "Hello world!"}]``')

158 - parser.add_argument("--response-role",

159 - type=optional_type(str),

160 - default="assistant",

161 - help="The role name to return if "

162 - "``request.add_generation_prompt=true``.")

163 - parser.add_argument("--ssl-keyfile",

164 - type=optional_type(str),

165 - default=None,

166 - help="The file path to the SSL key file.")

167 - parser.add_argument("--ssl-certfile",

168 - type=optional_type(str),

169 - default=None,

170 - help="The file path to the SSL cert file.")

171 - parser.add_argument("--ssl-ca-certs",

172 - type=optional_type(str),

173 - default=None,

174 - help="The CA certificates file.")

175 - parser.add_argument(

176 - "--enable-ssl-refresh",

177 - action="store_true",

178 - default=False,

179 - help="Refresh SSL Context when SSL certificate files change")

180 - parser.add_argument(

181 - "--ssl-cert-reqs",

182 - type=int,

183 - default=int(ssl.CERT_NONE),

184 - help="Whether client certificate is required (see stdlib ssl module's)."

185 - )

186 - parser.add_argument(

187 - "--root-path",

188 - type=optional_type(str),

189 - default=None,

190 - help="FastAPI root_path when app is behind a path based routing proxy."

191 - )

192 - parser.add_argument(

193 - "--middleware",

194 - type=optional_type(str),

195 - action="append",

196 - default=[],

197 - help="Additional ASGI middleware to apply to the app. "

198 - "We accept multiple --middleware arguments. "

199 - "The value should be an import path. "

200 - "If a function is provided, vLLM will add it to the server "

201 - "using ``@app.middleware('http')``. "

202 - "If a class is provided, vLLM will add it to the server "

203 - "using ``app.add_middleware()``. ")

204 - parser.add_argument(

205 - "--return-tokens-as-token-ids",

206 - action="store_true",

207 - help="When ``--max-logprobs`` is specified, represents single tokens "

208 - " as strings of the form 'token_id:{token_id}' so that tokens "

209 - "that are not JSON-encodable can be identified.")

210 - parser.add_argument(

211 - "--disable-frontend-multiprocessing",

212 - action="store_true",

213 - help="If specified, will run the OpenAI frontend server in the same "

214 - "process as the model serving engine.")

215 - parser.add_argument(

216 - "--enable-request-id-headers",

217 - action="store_true",

218 - help="If specified, API server will add X-Request-Id header to "

219 - "responses.")

220 - parser.add_argument(

221 - "--enable-auto-tool-choice",

222 - action="store_true",

223 - default=False,

224 - help="Enable auto tool choice for supported models. Use "

225 - "``--tool-call-parser`` to specify which parser to use.")

226 - parser.add_argument(

227 - "--expand-tools-even-if-tool-choice-none",

228 - action="store_true",

229 - default=False,

230 - deprecated=True,

231 - help="Include tool definitions in prompts "

232 - "even when tool_choice='none'. "

233 - "This is a transitional option that will be removed in v0.10.0. "

234 - "In v0.10.0, tool definitions will always be included regardless of "

235 - "tool_choice setting. Use this flag now to test the new behavior "

236 - "before the breaking change.")

237 -

238 - valid_tool_parsers = ToolParserManager.tool_parsers.keys()

239 - parser.add_argument(

240 - "--tool-call-parser",

241 - type=str,

242 - metavar="{" + ",".join(valid_tool_parsers) + "} or name registered in "

243 - "--tool-parser-plugin",

244 - default=None,

245 - help=

246 - "Select the tool call parser depending on the model that you're using."

247 - " This is used to parse the model-generated tool call into OpenAI API "

248 - "format. Required for ``--enable-auto-tool-choice``.")

249 -

250 - parser.add_argument(

251 - "--tool-parser-plugin",

252 - type=str,

253 - default="",

254 - help=

255 - "Special the tool parser plugin write to parse the model-generated tool"

256 - " into OpenAI API format, the name register in this plugin can be used "

257 - "in ``--tool-call-parser``.")

258 -

259 - parser.add_argument(

260 - "--log-config-file",

261 - type=str,

262 - default=envs.VLLM_LOGGING_CONFIG_PATH,

263 - help="Path to logging config JSON file for both vllm and uvicorn",

264 - )

245 + """Create the CLI argument parser used by the OpenAI API server.

246

247 + We rely on the helper methods of `FrontendArgs` and `AsyncEngineArgs` to

248 + register all arguments instead of manually enumerating them here. This

249 + avoids code duplication and keeps the argument definitions in one place.

250 + """

251 + parser = FrontendArgs.add_cli_args(parser)

252 parser = AsyncEngineArgs.add_cli_args(parser)

253

268 - parser.add_argument('--max-log-len',

269 - type=int,

270 - default=None,

271 - help='Max number of prompt characters or prompt '

272 - 'ID numbers being printed in log.'

273 - ' The default of None means unlimited.')

274 -

275 - parser.add_argument(

276 - "--disable-fastapi-docs",

277 - action='store_true',

278 - default=False,

279 - help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."

280 - )

281 - parser.add_argument(

282 - "--enable-prompt-tokens-details",

283 - action='store_true',

284 - default=False,

285 - help="If set to True, enable prompt_tokens_details in usage.")

286 - parser.add_argument(

287 - "--enable-force-include-usage",

288 - action='store_true',

289 - default=False,

290 - help="If set to True, including usage on every request.")

291 - parser.add_argument(

292 - "--enable-server-load-tracking",

293 - action='store_true',

294 - default=False,

295 - help=

296 - "If set to True, enable tracking server_load_metrics in the app state."

297 - )

298 -

254 return parser

255

256

[frontend] Refactor CLI Args for a better modular integration (#20206)