@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dc837bc23e

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 20:15 UTC

feat(frontend): add --default-chat-template-kwargs CLI argument (#31343)

Signed-off-by: effortprogrammer <yhjhoward7@gmail.com>

Hojin Yang committed Dec 30, 2025 at 12:38 UTC dc837bc23e70eed89b562774c7ca8c1bec9f87ae

7 files changed +91 -2

docs/features/reasoning_outputs.md

+36

@@ -204,6 +204,42 @@ The reasoning content is also available when both tool calling and the reasoning

204

205 For more examples, please refer to [examples/online_serving/openai_chat_completion_tool_calls_with_reasoning.py](../../examples/online_serving/openai_chat_completion_tool_calls_with_reasoning.py).

206

207 +## Server-Level Default Chat Template Kwargs

208 +

209 +You can set default `chat_template_kwargs` at the server level using the `--default-chat-template-kwargs` CLI argument. This is useful for configuring reasoning behavior across all requests without requiring clients to specify it in each request.

210 +

211 +### Disabling Thinking Mode by Default

212 +

213 +For models like Qwen3 where thinking is enabled by default, you can disable it server-wide:

214 +

215 +```bash

216 +vllm serve Qwen/Qwen3-8B \

217 + --reasoning-parser qwen3 \

218 + --default-chat-template-kwargs '{"enable_thinking": false}'

219 +```

220 +

221 +### Enabling Thinking Mode by Default

222 +

223 +For models like IBM Granite 3.2 or DeepSeek-V3.1 where thinking is disabled by default, you can enable it server-wide:

224 +

225 +```bash

226 +vllm serve ibm-granite/granite-3.2-2b-instruct \

227 + --reasoning-parser granite \

228 + --default-chat-template-kwargs '{"thinking": true}'

229 +```

230 +

231 +### Request-Level Override

232 +

233 +Request-level `chat_template_kwargs` always take priority over server defaults. For example, if the server is started with `enable_thinking=false`, a client can still enable it for a specific request:

234 +

235 +```python

236 +response = client.chat.completions.create(

237 + model=model,

238 + messages=messages,

239 + extra_body={"chat_template_kwargs": {"enable_thinking": True}} # Overrides server default

240 +)

241 +```

242 +

243 ## Limitations

244

245 - The reasoning content is only available for online serving's chat completion endpoint (`/v1/chat/completions`).

tests/entrypoints/openai/test_cli_args.py

+33

@@ -208,3 +208,36 @@ def test_middleware(serve_parser, cli_args, expected_middleware):

208 """Ensure multiple middleware args are parsed properly"""

209 args = serve_parser.parse_args(args=cli_args)

210 assert args.middleware == expected_middleware

211 +

212 +

213 +def test_default_chat_template_kwargs_parsing(serve_parser):

214 + """Ensure default_chat_template_kwargs JSON is parsed correctly"""

215 + args = serve_parser.parse_args(

216 + args=["--default-chat-template-kwargs", '{"enable_thinking": false}']

217 + )

218 + assert args.default_chat_template_kwargs == {"enable_thinking": False}

219 +

220 +

221 +def test_default_chat_template_kwargs_complex(serve_parser):

222 + """Ensure complex default_chat_template_kwargs JSON is parsed correctly"""

223 + kwargs_json = '{"enable_thinking": false, "custom_param": "value", "num": 42}'

224 + args = serve_parser.parse_args(args=["--default-chat-template-kwargs", kwargs_json])

225 + assert args.default_chat_template_kwargs == {

226 + "enable_thinking": False,

227 + "custom_param": "value",

228 + "num": 42,

229 + }

230 +

231 +

232 +def test_default_chat_template_kwargs_default_none(serve_parser):

233 + """Ensure default_chat_template_kwargs defaults to None"""

234 + args = serve_parser.parse_args(args=[])

235 + assert args.default_chat_template_kwargs is None

236 +

237 +

238 +def test_default_chat_template_kwargs_invalid_json(serve_parser):

239 + """Ensure invalid JSON raises an error"""

240 + with pytest.raises(SystemExit):

241 + serve_parser.parse_args(

242 + args=["--default-chat-template-kwargs", "not valid json"]

243 + )

vllm/entrypoints/openai/api_server.py

@@ -1081,6 +1081,7 @@ async def init_app_state(

1081 request_logger=request_logger,

1082 chat_template=resolved_chat_template,

1083 chat_template_content_format=args.chat_template_content_format,

1084 + default_chat_template_kwargs=args.default_chat_template_kwargs,

1085 trust_request_chat_template=args.trust_request_chat_template,

1086 return_tokens_as_token_ids=args.return_tokens_as_token_ids,

1087 enable_auto_tools=args.enable_auto_tool_choice,

vllm/entrypoints/openai/cli_args.py

+10 -1

@@ -11,7 +11,7 @@ import json

11 import ssl

12 from collections.abc import Sequence

13 from dataclasses import field

14 -from typing import Literal

14 +from typing import Any, Literal

16 from pydantic.dataclasses import dataclass

@@ -114,6 +114,12 @@ class FrontendArgs:

114 """Whether to trust the chat template provided in the request. If False,

115 the server will always use the chat template specified by `--chat-template`

116 or the ones from tokenizer."""

117 + default_chat_template_kwargs: dict[str, Any] | None = None

118 + """Default keyword arguments to pass to the chat template renderer.

119 + These will be merged with request-level chat_template_kwargs,

120 + with request values taking precedence. Useful for setting default

121 + behavior for reasoning models. Example: '{"enable_thinking": false}'

122 + to disable thinking mode by default for Qwen3/DeepSeek models."""

123 response_role: str = "assistant"

124 """The role name to return if `request.add_generation_prompt=true`."""

125 ssl_keyfile: str | None = None

@@ -216,6 +222,9 @@ class FrontendArgs:

222 del frontend_kwargs["allowed_methods"]["nargs"]

223 del frontend_kwargs["allowed_headers"]["nargs"]

224

225 + # Special case: default_chat_template_kwargs needs json.loads type

226 + frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads

227 +

228 # Special case: LoRA modules need custom parser action and

229 # optional_type(str)

230 frontend_kwargs["lora_modules"]["type"] = optional_type(str)

vllm/entrypoints/openai/run_batch.py

@@ -468,6 +468,9 @@ async def run_batch(

468 reasoning_parser=args.structured_outputs_config.reasoning_parser,

469 enable_prompt_tokens_details=args.enable_prompt_tokens_details,

470 enable_force_include_usage=args.enable_force_include_usage,

471 + default_chat_template_kwargs=getattr(

472 + args, "default_chat_template_kwargs", None

473 + ),

474 )

475 if "generate" in supported_tasks

476 else None

vllm/entrypoints/openai/serving_chat.py

+5 -1

@@ -6,7 +6,7 @@ import json

6 import time

7 from collections.abc import AsyncGenerator, AsyncIterator

8 from collections.abc import Sequence as GenericSequence

9 -from typing import Final

9 +from typing import Any, Final

11 import jinja2

12 import partial_json_parser

@@ -102,6 +102,7 @@ class OpenAIServingChat(OpenAIServing):

102 enable_force_include_usage: bool = False,

103 enable_log_outputs: bool = False,

104 log_error_stack: bool = False,

105 + default_chat_template_kwargs: dict[str, Any] | None = None,

106 ) -> None:

107 super().__init__(

108 engine_client=engine_client,

@@ -115,6 +116,7 @@ class OpenAIServingChat(OpenAIServing):

116 self.chat_template = chat_template

117 self.chat_template_content_format: Final = chat_template_content_format

118 self.trust_request_chat_template = trust_request_chat_template

119 + self.default_chat_template_kwargs = default_chat_template_kwargs or {}

120 self.enable_log_outputs = enable_log_outputs

121

122 # set up logits processors

@@ -203,6 +205,7 @@ class OpenAIServingChat(OpenAIServing):

205 tool_dicts=None,

206 documents=None,

207 chat_template_kwargs=None,

208 + default_chat_template_kwargs=self.default_chat_template_kwargs,

209 tool_parser=None,

210 add_special_tokens=False,

211 )

@@ -310,6 +313,7 @@ class OpenAIServingChat(OpenAIServing):

313 tool_dicts=tool_dicts,

314 documents=request.documents,

315 chat_template_kwargs=request.chat_template_kwargs,

316 + default_chat_template_kwargs=self.default_chat_template_kwargs,

317 tool_parser=tool_parser,

318 add_special_tokens=request.add_special_tokens,

319 )

vllm/entrypoints/openai/serving_engine.py

@@ -1158,6 +1158,7 @@ class OpenAIServing:

1158 tool_dicts: list[dict[str, Any]] | None = None,

1159 documents: list[dict[str, str]] | None = None,

1160 chat_template_kwargs: dict[str, Any] | None = None,

1161 + default_chat_template_kwargs: dict[str, Any] | None = None,

1162 tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,

1163 add_special_tokens: bool = False,

1164 ) -> tuple[list[ConversationMessage], list[TokensPrompt]]:

@@ -1183,6 +1184,8 @@ class OpenAIServing:

1184 tools=tool_dicts,

1185 documents=documents,

1186 )

1187 + if default_chat_template_kwargs:

1188 + _chat_template_kwargs.update(default_chat_template_kwargs)

1189 _chat_template_kwargs.update(chat_template_kwargs or {})

1190

1191 request_prompt: str | list[int]