| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """This file contains the command line arguments for the vLLM's online server. |
| 4 | It is kept in a separate file for documentation purposes. |
| 5 | """ |
| 6 | |
| 7 | import argparse |
| 8 | import json |
| 9 | import ssl |
| 10 | from collections.abc import Sequence |
| 11 | from dataclasses import field |
| 12 | from typing import Any, Literal |
| 13 | |
| 14 | import vllm.envs as envs |
| 15 | from vllm.config import config |
| 16 | from vllm.engine.arg_utils import AsyncEngineArgs, optional_type |
| 17 | from vllm.entrypoints.anthropic.protocol import ( |
| 18 | AnthropicDisabledThinkingEffortOption, |
| 19 | ) |
| 20 | from vllm.entrypoints.chat_utils import ( |
| 21 | ChatTemplateContentFormatOption, |
| 22 | validate_chat_template, |
| 23 | ) |
| 24 | from vllm.entrypoints.openai.models.protocol import LoRAModulePath |
| 25 | from vllm.logger import init_logger |
| 26 | from vllm.tool_parsers import ToolParserManager |
| 27 | from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName |
| 28 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 29 | |
| 30 | from .utils.constants import ( |
| 31 | H11_MAX_HEADER_COUNT_DEFAULT, |
| 32 | H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT, |
| 33 | ) |
| 34 | |
| 35 | logger = init_logger(__name__) |
| 36 | |
| 37 | |
| 38 | class LoRAParserAction(argparse.Action): |
| 39 | def __call__( |
| 40 | self, |
| 41 | parser: argparse.ArgumentParser, |
| 42 | namespace: argparse.Namespace, |
| 43 | values: str | Sequence[str] | None, |
| 44 | option_string: str | None = None, |
| 45 | ): |
| 46 | if values is None: |
| 47 | values = [] |
| 48 | if isinstance(values, str): |
| 49 | raise TypeError("Expected values to be a list") |
| 50 | |
| 51 | lora_list: list[LoRAModulePath] = [] |
| 52 | for item in values: |
| 53 | if item in [None, ""]: # Skip if item is None or empty string |
| 54 | continue |
| 55 | if "=" in item and "," not in item: # Old format: name=path |
| 56 | name, path = item.split("=", 1) |
| 57 | lora_list.append(LoRAModulePath(name, path)) |
| 58 | else: # Assume JSON format |
| 59 | try: |
| 60 | lora_dict = json.loads(item) |
| 61 | lora = LoRAModulePath(**lora_dict) |
| 62 | lora_list.append(lora) |
| 63 | except json.JSONDecodeError: |
| 64 | parser.error(f"Invalid JSON format for --lora-modules: {item}") |
| 65 | except TypeError as e: |
| 66 | parser.error( |
| 67 | f"Invalid fields for --lora-modules: {item} - {str(e)}" |
| 68 | ) |
| 69 | setattr(namespace, self.dest, lora_list) |
| 70 | |
| 71 | |
| 72 | @config |
| 73 | class BaseFrontendArgs: |
| 74 | """Base arguments for the OpenAI-compatible frontend server. |
| 75 | |
| 76 | This base class does not include host, port, and server-specific arguments |
| 77 | like SSL, CORS, and HTTP server settings. Those arguments are added by |
| 78 | the subclasses. |
| 79 | """ |
| 80 | |
| 81 | lora_modules: list[LoRAModulePath] | None = None |
| 82 | """LoRA modules configurations in either 'name=path' format or JSON format |
| 83 | or JSON list format. Example (old format): `'name=path'` Example (new |
| 84 | format): `{\"name\": \"name\", \"path\": \"lora_path\", |
| 85 | \"base_model_name\": \"id\"}`""" |
| 86 | chat_template: str | None = None |
| 87 | """The file path to the chat template, or the template in single-line form |
| 88 | for the specified model.""" |
| 89 | chat_template_content_format: ChatTemplateContentFormatOption = "auto" |
| 90 | """The format to render message content within a chat template. |
| 91 | |
| 92 | * "string" will render the content as a string. Example: `"Hello World"` |
| 93 | * "openai" will render the content as a list of dictionaries, similar to |
| 94 | OpenAI schema. Example: `[{"type": "text", "text": "Hello world!"}]`""" |
| 95 | trust_request_chat_template: bool = False |
| 96 | """Whether to trust the chat template provided in the request. If False, |
| 97 | the server will always use the chat template specified by `--chat-template` |
| 98 | or the ones from tokenizer.""" |
| 99 | trust_request_mm_kwargs: bool = False |
| 100 | """Whether to trust per-request multimodal kwargs (`mm_processor_kwargs` |
| 101 | and `media_io_kwargs`). If False, the server rejects non-empty values |
| 102 | because they can change multimodal preprocessing resource usage. Only |
| 103 | enable this when API clients are trusted.""" |
| 104 | default_chat_template_kwargs: dict[str, Any] | None = None |
| 105 | """Default keyword arguments to pass to the chat template renderer. |
| 106 | These will be merged with request-level chat_template_kwargs, |
| 107 | with request values taking precedence. Useful for setting default |
| 108 | behavior for reasoning models. Example: '{"enable_thinking": false}' |
| 109 | to disable thinking mode by default for Qwen3/DeepSeek models.""" |
| 110 | response_role: str = "assistant" |
| 111 | """The role name to return if `request.add_generation_prompt=true`.""" |
| 112 | return_tokens_as_token_ids: bool = False |
| 113 | """When `--max-logprobs` is specified, represents single tokens as |
| 114 | strings of the form 'token_id:{token_id}' so that tokens that are not |
| 115 | JSON-encodable can be identified.""" |
| 116 | enable_auto_tool_choice: bool = False |
| 117 | """Enable auto tool choice for supported models. Use `--tool-call-parser` |
| 118 | to specify which parser to use.""" |
| 119 | exclude_tools_when_tool_choice_none: bool = False |
| 120 | """If specified, exclude tool definitions in prompts when |
| 121 | tool_choice='none'.""" |
| 122 | tool_call_parser: str | None = None |
| 123 | """Select the tool call parser depending on the model that you're using. |
| 124 | This is used to parse the model-generated tool call into OpenAI API format. |
| 125 | Required for `--enable-auto-tool-choice`. You can choose any option from |
| 126 | the built-in parsers or register a plugin via `--tool-parser-plugin`.""" |
| 127 | tool_parser_plugin: str = "" |
| 128 | """Special the tool parser plugin write to parse the model-generated tool |
| 129 | into OpenAI API format, the name register in this plugin can be used in |
| 130 | `--tool-call-parser`.""" |
| 131 | tool_strict_level: ToolStrictLevelName = "auto" |
| 132 | """Server-side floor for structural-tag based tool calling, applied on top |
| 133 | of the per-tool `strict` field. `auto` follows the request's tool choice |
| 134 | and per-tool strictness; `function` constrains the |
| 135 | tool-call envelope (markup and function name) for every request with |
| 136 | tools; `parameter` additionally pins argument schemas, as if every tool |
| 137 | were `strict: true`.""" |
| 138 | tool_server: str | None = None |
| 139 | """Comma-separated list of host:port pairs (IPv4, IPv6, or hostname). |
| 140 | Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for |
| 141 | built-in demo tools (browser and Python code interpreter). WARNING: |
| 142 | The `demo` Python tool executes model-generated code in Docker without |
| 143 | network isolation by default. See the security guide for more |
| 144 | information.""" |
| 145 | max_log_len: int | None = None |
| 146 | """Max number of prompt characters or prompt ID numbers being printed in |
| 147 | log. The default of None means unlimited.""" |
| 148 | enable_prompt_tokens_details: bool = False |
| 149 | """If set to True, enable prompt_tokens_details in usage.""" |
| 150 | enable_per_request_metrics: bool = False |
| 151 | """If set to True, include per-request timing metrics in API responses.""" |
| 152 | enable_server_load_tracking: bool = False |
| 153 | """If set to True, enable tracking server_load_metrics in the app state.""" |
| 154 | enable_force_include_usage: bool = False |
| 155 | """If set to True, including usage on every request.""" |
| 156 | sse_keep_alive_interval: int = 0 |
| 157 | """Send an SSE keep-alive comment line every this many seconds when a |
| 158 | `/v1/chat/completions` or `/v1/completions` streaming response is idle |
| 159 | (queued, prefill, or between tokens), to prevent reverse proxies/tunnels |
| 160 | with read timeouts from closing the connection. Defaults to 0, which |
| 161 | disables keep-alive comments entirely. |
| 162 | """ |
| 163 | enable_tokenizer_info_endpoint: bool = False |
| 164 | """Enable the `/tokenizer_info` endpoint. May expose chat |
| 165 | templates and other tokenizer configuration.""" |
| 166 | enable_log_outputs: bool = False |
| 167 | """If set to True, log model outputs (generations). Requires |
| 168 | `--enable-log-requests`. Output text and finish reasons are logged at INFO, |
| 169 | while output token IDs are logged at DEBUG.""" |
| 170 | enable_log_deltas: bool = True |
| 171 | """If set to False, output deltas will not be logged. Relevant only if |
| 172 | --enable-log-outputs is set. |
| 173 | """ |
| 174 | cohere_is_reasoning_model: bool = True |
| 175 | """Cohere ``/cohere/v2/chat`` only. Whether the served model is a |
| 176 | reasoning Command-family model. When True (default), the assistant's |
| 177 | chain-of-thought is surfaced as a ``thinking`` content block (or |
| 178 | ``content-*`` events on the stream). When False, reasoning is |
| 179 | surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta`` |
| 180 | events) whenever the model emits tool calls, matching older non- |
| 181 | reasoning Command models. Has no effect on the non-Cohere |
| 182 | endpoints.""" |
| 183 | cohere_format: str = "cmd4" |
| 184 | """Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt |
| 185 | format to render: ``cmd4`` (current Command A+ models; default) or |
| 186 | ``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the |
| 187 | wrong format silently produces a prompt the model wasn't trained |
| 188 | on, which most commonly manifests as the model emitting text but no |
| 189 | citations / tool calls / thinking blocks. Equivalent to passing |
| 190 | ``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any |
| 191 | explicit request-level ``chat_template_kwargs.cohere_format`` takes |
| 192 | priority.""" |
| 193 | anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto" |
| 194 | """Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for |
| 195 | requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking |
| 196 | off for models that support it; ``low`` suits models that always think |
| 197 | (e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses |
| 198 | ``low`` when the renderer rejects ``none`` or renders it the same as a |
| 199 | thinking effort, and ``none`` otherwise.""" |
| 200 | log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE |
| 201 | """If set to True, log the stack trace of error responses""" |
| 202 | tokens_only: bool = False |
| 203 | """ |
| 204 | If set to True, only enable the Tokens In<>Out endpoint. |
| 205 | This is intended for use in a Disaggregated Everything setup. |
| 206 | """ |
| 207 | enable_scale_out: bool = False |
| 208 | """ |
| 209 | If set to True, register the scale-out endpoints (`/render`, `/derender`, |
| 210 | and `/inference/v1/generate`) on `vllm serve`. Has no effect on |
| 211 | `vllm launch render` or `vllm serve --tokens-only`, which always register |
| 212 | their required endpoints regardless of this flag. |
| 213 | """ |
| 214 | fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full" |
| 215 | """Controls the ``system_fingerprint`` field on responses. |
| 216 | |
| 217 | - ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>``. Encodes |
| 218 | server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an |
| 219 | 8-char config hash. |
| 220 | - ``hash``: ``vllm-<version>-<hash8>``. Parallelism stripped. |
| 221 | - ``custom``: emits the literal string from ``--fingerprint-value``. |
| 222 | - ``none``: the field is omitted (serialized as ``null``). |
| 223 | """ |
| 224 | fingerprint_value: str | None = None |
| 225 | """Literal fingerprint string used when ``--fingerprint-mode=custom``.""" |
| 226 | |
| 227 | @classmethod |
| 228 | def _customize_cli_kwargs( |
| 229 | cls, |
| 230 | frontend_kwargs: dict[str, Any], |
| 231 | ) -> dict[str, Any]: |
| 232 | """Customize argparse kwargs before arguments are registered. |
| 233 | |
| 234 | Subclasses should override this and call |
| 235 | ``super()._customize_cli_kwargs(frontend_kwargs)`` first. |
| 236 | """ |
| 237 | # Special case: default_chat_template_kwargs needs json.loads type |
| 238 | frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads |
| 239 | |
| 240 | # Special case: LoRA modules need custom parser action and |
| 241 | # optional_type(str) |
| 242 | frontend_kwargs["lora_modules"]["type"] = optional_type(str) |
| 243 | frontend_kwargs["lora_modules"]["action"] = LoRAParserAction |
| 244 | |
| 245 | # Special case: Tool call parser shows built-in options. |
| 246 | valid_tool_parsers = list(ToolParserManager.list_registered()) |
| 247 | parsers_str = ",".join(valid_tool_parsers) |
| 248 | frontend_kwargs["tool_call_parser"]["metavar"] = ( |
| 249 | f"{{{parsers_str}}} or name registered in --tool-parser-plugin" |
| 250 | ) |
| 251 | return frontend_kwargs |
| 252 | |
| 253 | @classmethod |
| 254 | def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser: |
| 255 | """Register CLI arguments for this frontend class. |
| 256 | |
| 257 | Subclasses should override ``_customize_cli_kwargs`` instead of |
| 258 | this method so that base-class postprocessing is always applied. |
| 259 | """ |
| 260 | from vllm.engine.arg_utils import get_kwargs |
| 261 | |
| 262 | frontend_kwargs = get_kwargs(cls) |
| 263 | frontend_kwargs = cls._customize_cli_kwargs(frontend_kwargs) |
| 264 | |
| 265 | group_name = cls.__name__.replace("Args", "") |
| 266 | frontend_group = parser.add_argument_group( |
| 267 | title=group_name, |
| 268 | description=cls.__doc__, |
| 269 | ) |
| 270 | for key, value in frontend_kwargs.items(): |
| 271 | extra_flags = value.pop("flags", []) |
| 272 | frontend_group.add_argument( |
| 273 | *extra_flags, f"--{key.replace('_', '-')}", **value |
| 274 | ) |
| 275 | |
| 276 | return parser |
| 277 | |
| 278 | |
| 279 | @config |
| 280 | class FrontendArgs(BaseFrontendArgs): |
| 281 | """Arguments for the OpenAI-compatible frontend server.""" |
| 282 | |
| 283 | host: str | None = None |
| 284 | """Host name.""" |
| 285 | port: int = 8000 |
| 286 | """Port number.""" |
| 287 | grpc_port: int | None = None |
| 288 | """Enable the Rust frontend's additional gRPC Inference and Control services |
| 289 | on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on |
| 290 | `--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined |
| 291 | with the Python gRPC server's `--grpc` flag.""" |
| 292 | data_parallel_supervisor_port: int = 9256 |
| 293 | """HTTP port for aggregated health endpoints in multi-port external LB |
| 294 | mode.""" |
| 295 | dp_supervisor_probe_interval_s: float = 5.0 |
| 296 | """Seconds between aggregated health probes in multi-port external LB mode.""" |
| 297 | dp_supervisor_probe_timeout_s: float = 5.0 |
| 298 | """Seconds to wait between retries when a child health probe fails with a |
| 299 | connection error in multi-port external LB mode.""" |
| 300 | dp_supervisor_probe_failure_threshold: int = 3 |
| 301 | """Number of consecutive connection-error retries before a child health |
| 302 | probe is declared failed in multi-port external LB mode.""" |
| 303 | uds: str | None = None |
| 304 | """Unix domain socket path. If set, host and port arguments are ignored.""" |
| 305 | uvicorn_log_level: Literal[ |
| 306 | "critical", "error", "warning", "info", "debug", "trace" |
| 307 | ] = "info" |
| 308 | """Log level for uvicorn.""" |
| 309 | disable_uvicorn_access_log: bool = False |
| 310 | """Disable uvicorn access log.""" |
| 311 | disable_access_log_for_endpoints: str | None = None |
| 312 | """Comma-separated list of endpoint paths to exclude from uvicorn access |
| 313 | logs. This is useful to reduce log noise from high-frequency endpoints |
| 314 | like health checks. Example: "/health,/metrics,/ping". |
| 315 | When set, access logs for requests to these paths will be suppressed |
| 316 | while keeping logs for other endpoints.""" |
| 317 | allow_credentials: bool = False |
| 318 | """Allow credentials.""" |
| 319 | allowed_origins: list[str] = field(default_factory=lambda: ["*"]) |
| 320 | """Allowed origins.""" |
| 321 | allowed_methods: list[str] = field(default_factory=lambda: ["*"]) |
| 322 | """Allowed methods.""" |
| 323 | allowed_headers: list[str] = field(default_factory=lambda: ["*"]) |
| 324 | """Allowed headers.""" |
| 325 | api_key: list[str] | None = None |
| 326 | """If provided, the server will require one of these keys to be presented in |
| 327 | the header. |
| 328 | |
| 329 | Warning: this only authenticates endpoints under the `/v1`, `/v2`, and |
| 330 | `/inference` path prefixes. Other endpoints on the same server, including |
| 331 | `/invocations` (which exposes the same inference capabilities as `/v1`), |
| 332 | remain unauthenticated. Do not rely on `--api-key` alone to secure vLLM; |
| 333 | see |
| 334 | https://docs.vllm.ai/en/latest/usage/security.html#api-key-authentication-limitations |
| 335 | for what it does and does not protect.""" |
| 336 | ssl_keyfile: str | None = None |
| 337 | """The file path to the SSL key file.""" |
| 338 | ssl_certfile: str | None = None |
| 339 | """The file path to the SSL cert file.""" |
| 340 | ssl_ca_certs: str | None = None |
| 341 | """The CA certificates file.""" |
| 342 | enable_ssl_refresh: bool = False |
| 343 | """Refresh SSL Context when SSL certificate files change""" |
| 344 | ssl_cert_reqs: int = int(ssl.CERT_NONE) |
| 345 | """Whether client certificate is required (see stdlib ssl module's).""" |
| 346 | ssl_ciphers: str | None = None |
| 347 | """SSL cipher suites for HTTPS (TLS 1.2 and below only). |
| 348 | Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'""" |
| 349 | root_path: str | None = None |
| 350 | """FastAPI root_path when app is behind a path based routing proxy.""" |
| 351 | middleware: list[str] = field(default_factory=lambda: []) |
| 352 | """Additional ASGI middleware to apply to the app. We accept multiple |
| 353 | --middleware arguments. The value should be an import path. If a function |
| 354 | is provided, vLLM will add it to the server using |
| 355 | `@app.middleware('http')`. If a class is provided, vLLM will |
| 356 | add it to the server using `app.add_middleware()`.""" |
| 357 | enable_request_id_headers: bool = False |
| 358 | """If specified, API server will add X-Request-Id header to responses.""" |
| 359 | disable_fastapi_docs: bool = False |
| 360 | """Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint.""" |
| 361 | h11_max_incomplete_event_size: int = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT |
| 362 | """Maximum size (bytes) of an incomplete HTTP event (header or body) for |
| 363 | h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB).""" |
| 364 | h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT |
| 365 | """Maximum number of HTTP headers allowed in a request for h11 parser. |
| 366 | Helps mitigate header abuse. Default: 256.""" |
| 367 | enable_offline_docs: bool = False |
| 368 | """ |
| 369 | Enable offline FastAPI documentation for air-gapped environments. |
| 370 | Uses vendored static assets bundled with vLLM. |
| 371 | """ |
| 372 | enable_flash_late_interaction: bool = True |
| 373 | """If set, run pooling score MaxSim on GPU in the API server process. |
| 374 | Can significantly improve late-interaction scoring performance. |
| 375 | When disabled, the setting is also propagated to the engine's |
| 376 | `PoolerConfig`, so the engine-side scorer uses the reference MaxSim |
| 377 | path instead of the fused Triton kernel.""" |
| 378 | |
| 379 | @classmethod |
| 380 | def _customize_cli_kwargs( |
| 381 | cls, |
| 382 | frontend_kwargs: dict[str, Any], |
| 383 | ) -> dict[str, Any]: |
| 384 | frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs) |
| 385 | |
| 386 | # Special case: allowed_origins, allowed_methods, allowed_headers all |
| 387 | # need json.loads type |
| 388 | # Should also remove nargs |
| 389 | frontend_kwargs["allowed_origins"]["type"] = json.loads |
| 390 | frontend_kwargs["allowed_methods"]["type"] = json.loads |
| 391 | frontend_kwargs["allowed_headers"]["type"] = json.loads |
| 392 | del frontend_kwargs["allowed_origins"]["nargs"] |
| 393 | del frontend_kwargs["allowed_methods"]["nargs"] |
| 394 | del frontend_kwargs["allowed_headers"]["nargs"] |
| 395 | |
| 396 | # Special case: Middleware needs to append action |
| 397 | frontend_kwargs["middleware"]["action"] = "append" |
| 398 | frontend_kwargs["middleware"]["type"] = str |
| 399 | if "nargs" in frontend_kwargs["middleware"]: |
| 400 | del frontend_kwargs["middleware"]["nargs"] |
| 401 | frontend_kwargs["middleware"]["default"] = [] |
| 402 | |
| 403 | # Special case: disable_access_log_for_endpoints is a single |
| 404 | # comma-separated string, not a list |
| 405 | if "nargs" in frontend_kwargs["disable_access_log_for_endpoints"]: |
| 406 | del frontend_kwargs["disable_access_log_for_endpoints"]["nargs"] |
| 407 | |
| 408 | return frontend_kwargs |
| 409 | |
| 410 | |
| 411 | def resolve_default_chat_template_kwargs( |
| 412 | args: argparse.Namespace, |
| 413 | ) -> dict[str, Any]: |
| 414 | """Resolve renderer defaults, including the dedicated Cohere format flag.""" |
| 415 | kwargs = dict(args.default_chat_template_kwargs or {}) |
| 416 | if getattr(args, "cohere_format", None): |
| 417 | kwargs.setdefault("cohere_format", args.cohere_format) |
| 418 | return kwargs |
| 419 | |
| 420 | |
| 421 | def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: |
| 422 | """Create the CLI argument parser used by the OpenAI API server. |
| 423 | |
| 424 | We rely on the helper methods of `FrontendArgs` and `AsyncEngineArgs` to |
| 425 | register all arguments instead of manually enumerating them here. This |
| 426 | avoids code duplication and keeps the argument definitions in one place. |
| 427 | """ |
| 428 | parser.add_argument( |
| 429 | "model_tag", |
| 430 | type=str, |
| 431 | nargs="?", |
| 432 | help="The model tag to serve (optional if specified in config)", |
| 433 | ) |
| 434 | parser.add_argument( |
| 435 | "--headless", |
| 436 | action="store_true", |
| 437 | default=False, |
| 438 | help="Run in headless mode. See multi-node data parallel " |
| 439 | "documentation for more details.", |
| 440 | ) |
| 441 | parser.add_argument( |
| 442 | "--api-server-count", |
| 443 | "-asc", |
| 444 | type=int, |
| 445 | default=None, |
| 446 | help="How many API server processes to run. " |
| 447 | "Defaults to data_parallel_size if not specified.", |
| 448 | ) |
| 449 | parser.add_argument( |
| 450 | "--config", |
| 451 | help="Read CLI options from a config file. " |
| 452 | "Must be a YAML with the following options: " |
| 453 | "https://docs.vllm.ai/en/latest/configuration/serve_args.html", |
| 454 | ) |
| 455 | parser.add_argument( |
| 456 | "--grpc", |
| 457 | action="store_true", |
| 458 | default=False, |
| 459 | help="Launch the Python gRPC (SMG VllmEngine) server on --port instead " |
| 460 | "of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. " |
| 461 | "Cannot be combined with --grpc-port, which enables the Rust " |
| 462 | "frontend's gRPC services.", |
| 463 | ) |
| 464 | parser = FrontendArgs.add_cli_args(parser) |
| 465 | parser = AsyncEngineArgs.add_cli_args(parser) |
| 466 | |
| 467 | return parser |
| 468 | |
| 469 | |
| 470 | def validate_grpc_port_arg(args: argparse.Namespace) -> None: |
| 471 | """Validate the Rust frontend's optional gRPC listener.""" |
| 472 | if getattr(args, "grpc_port", None) is None: |
| 473 | return |
| 474 | |
| 475 | if args.grpc: |
| 476 | raise ValueError( |
| 477 | "--grpc and --grpc-port are mutually exclusive. Use --grpc " |
| 478 | "--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 " |
| 479 | "with --grpc-port for the Rust frontend." |
| 480 | ) |
| 481 | if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve": |
| 482 | raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve") |
| 483 | if args.headless or ( |
| 484 | args.api_server_count is not None and args.api_server_count <= 0 |
| 485 | ): |
| 486 | raise ValueError( |
| 487 | "--grpc-port requires a Rust frontend; remove --headless " |
| 488 | "and non-positive --api-server-count" |
| 489 | ) |
| 490 | if args.data_parallel_multi_port_external_lb: |
| 491 | raise ValueError( |
| 492 | "--grpc-port is incompatible with " |
| 493 | "--data-parallel-multi-port-external-lb: " |
| 494 | "its frontends would share the same gRPC port" |
| 495 | ) |
| 496 | if not 0 <= args.grpc_port <= 65535: |
| 497 | raise ValueError("--grpc-port must be between 0 and 65535") |
| 498 | if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds: |
| 499 | raise ValueError("--grpc-port must differ from --port when HTTP uses TCP") |
| 500 | |
| 501 | |
| 502 | def validate_parsed_serve_args(args: argparse.Namespace): |
| 503 | """Quick checks for model serve args that raise prior to loading.""" |
| 504 | # `vllm launch <component>` builds its parser with make_arg_parser too (see |
| 505 | # LaunchSubcommandBase.add_cli_args), so its args are serve args as well. |
| 506 | if hasattr(args, "subparser") and args.subparser not in ("serve", "launch"): |
| 507 | return |
| 508 | |
| 509 | # Ensure that the chat template is valid; raises if it likely isn't |
| 510 | validate_chat_template(args.chat_template) |
| 511 | if args.chat_template is not None and "hf" in ( |
| 512 | args.tool_call_parser, |
| 513 | getattr(args, "reasoning_parser", None), |
| 514 | ): |
| 515 | logger.warning( |
| 516 | "--chat-template is set; the hf parser still expects " |
| 517 | "the checkpoint's output format." |
| 518 | ) |
| 519 | |
| 520 | validate_grpc_port_arg(args) |
| 521 | |
| 522 | # Enable auto tool needs a tool call parser to be valid |
| 523 | if args.enable_auto_tool_choice and not args.tool_call_parser: |
| 524 | raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser") |
| 525 | if args.enable_log_outputs and not args.enable_log_requests: |
| 526 | raise TypeError("Error: --enable-log-outputs requires --enable-log-requests") |
| 527 | |
| 528 | # SSE keep-alive interval must be zero (disabled) or a positive integer. |
| 529 | if getattr(args, "sse_keep_alive_interval", 0) < 0: |
| 530 | raise ValueError( |
| 531 | "Error: --sse-keep-alive-interval must be a non-negative integer " |
| 532 | "(0 disables keep-alive comments)." |
| 533 | ) |
| 534 | |
| 535 | if getattr(args, "enable_per_request_metrics", False) and getattr( |
| 536 | args, "disable_log_stats", False |
| 537 | ): |
| 538 | raise ValueError( |
| 539 | "Error: --enable-per-request-metrics requires engine statistics " |
| 540 | "logging; remove --disable-log-stats to enable per-request metrics." |
| 541 | ) |
| 542 | |
| 543 | if args.data_parallel_multi_port_external_lb: |
| 544 | from vllm.entrypoints.launchers.dp_supervisor import ( |
| 545 | validate_multi_port_external_lb_args, |
| 546 | ) |
| 547 | |
| 548 | validate_multi_port_external_lb_args(args) |
| 549 | |
| 550 | |
| 551 | def create_parser_for_docs() -> FlexibleArgumentParser: |
| 552 | parser_for_docs = FlexibleArgumentParser( |
| 553 | prog="-m vllm.entrypoints.launchers.api_server.entry" |
| 554 | ) |
| 555 | return make_arg_parser(parser_for_docs) |
| 556 | |
| 557 | |
| 558 | def propagate_flash_late_interaction(args, engine_args) -> None: |
| 559 | """Propagate `--no-enable-flash-late-interaction` into the engine config. |
| 560 | |
| 561 | The frontend flag alone only disables the API-server scoring path; |
| 562 | mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the |
| 563 | engine-side scorer fall back to the reference MaxSim path too. |
| 564 | """ |
| 565 | if getattr(args, "enable_flash_late_interaction", True): |
| 566 | return |
| 567 | from vllm.config.pooler import PoolerConfig |
| 568 | |
| 569 | if engine_args.pooler_config is None: |
| 570 | engine_args.pooler_config = PoolerConfig(enable_flash_late_interaction=False) |
| 571 | else: |
| 572 | engine_args.pooler_config.enable_flash_late_interaction = False |