@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 52a31ccecc

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 18:15 UTC

[Bugfix] Map reasoning_effort to enable_thinking in chat template kwargs (#43401)

Signed-off-by: Ashwin Giridharan <girida@amazon.com> Signed-off-by: Chauncey <chaunceyjiang@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>

Ashwin Giridharan committed May 27, 2026 at 05:39 UTC 52a31ccecca203effd490a5b496dc5f8d9496654

4 files changed +178 -13

docs/features/reasoning_outputs.md

+38 -1

@@ -15,6 +15,7 @@ vLLM currently supports the following reasoning models:

15 | ------------ | ----------- | ---------------- | ----------- |

@@ -29,6 +30,7 @@ vLLM currently supports the following reasoning models:

30 !!! note

31 IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.

32 The reasoning feature for the Qwen3 series is enabled by default. To disable it, you must pass `enable_thinking=False` in your `chat_template_kwargs`.

33 + Gemma 4 reasoning is disabled by default; to enable it, pass `enable_thinking=True` in your `chat_template_kwargs` or set `reasoning_effort` (which enables it automatically).

34 DeepSeek-V3.1 tool calling is supported in non-thinking mode.

35 Holo2 reasoning is enabled by default. To disable it, you must also pass `thinking=False` in your `chat_template_kwargs`.

@@ -314,9 +316,44 @@ for output in outputs:

316 print("text:", output.outputs[0].text)

317 ```

318

319 +## Automatic `enable_thinking` Activation

320 +

321 +Some models (such as Gemma 4, DeepSeek-V4-Pro and IBM Granite 3.2) require `enable_thinking: true` in their chat template kwargs to activate thinking mode — without it, reasoning tokens are never generated regardless of other settings.

322 +

323 +When you set `reasoning_effort` in a Chat Completions request (or `reasoning.effort` in a Responses API request), vLLM automatically injects `enable_thinking` into the chat template kwargs:

324 +

325 +- `reasoning_effort` = `"low"`, `"medium"`, or `"high"` → `enable_thinking = true`

326 +- `reasoning_effort` = `"none"` → `enable_thinking = false`

327 +- `reasoning_effort` not set → `enable_thinking` is not injected (preserves existing behavior)

328 +

329 +This means you no longer need to manually pass `chat_template_kwargs: {"enable_thinking": true}` when using `reasoning_effort` — it is handled automatically.

330 +

331 +!!! note

332 + If you explicitly set `enable_thinking` in `chat_template_kwargs`, your value takes priority over the automatic injection. This allows you to override the behavior if needed.

333 +

334 + For models whose templates don't declare `enable_thinking` (e.g., DeepSeek R1), the injected kwarg is harmlessly filtered out by `resolve_chat_template_kwargs`.

335 +

336 +### Example

337 +

338 +```python

339 +from openai import OpenAI

340 +

341 +client = OpenAI(base_url="http://localhost:8000/v1", api_key="dummy")

342 +

343 +# reasoning_effort automatically enables thinking for models that need it

344 +response = client.chat.completions.create(

345 + model="google/gemma-4-26B-A4B-it",

346 + messages=[{"role": "user", "content": "What is 15 * 37?"}],

347 + reasoning_effort="high", # Automatically sets enable_thinking=true

348 +)

349 +

350 +print(response.choices[0].message.reasoning)

351 +print(response.choices[0].message.content)

352 +```

353 +

354 ## Limitations

355

319 -- The reasoning content is only available for online serving's chat completion endpoint (`/v1/chat/completions`).

356 +- The reasoning content is only available for online serving's chat completion endpoint (`/v1/chat/completions`), Anthropic Messages API (`/v1/messages`) and the Responses API (`/v1/responses`).

357

358 ## How to support a new reasoning model

359

tests/entrypoints/openai/test_reasoning_enable_thinking.py new

+107

@@ -0,0 +1,107 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Unit tests for reasoning_effort -> enable_thinking mapping.

4 +

5 +Models like Gemma4 require enable_thinking=True in chat_template_kwargs to

6 +activate thinking mode. This mapping ensures that when a user requests

7 +reasoning (via reasoning_effort or reasoning.effort), the template kwarg

8 +is injected automatically.

9 +"""

10 +

11 +import pytest

12 +from openai.types.shared import Reasoning

13 +

14 +from vllm.entrypoints.openai.chat_completion.protocol import (

15 + ChatCompletionRequest,

16 +)

17 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest

18 +

19 +

20 +def _build_chat_request(**kwargs) -> ChatCompletionRequest:

21 + defaults = dict(

22 + model="test-model",

23 + messages=[{"role": "user", "content": "Hello"}],

24 + )

25 + defaults.update(kwargs)

26 + return ChatCompletionRequest(**defaults)

27 +

28 +

29 +def _build_responses_request(**kwargs) -> ResponsesRequest:

30 + defaults = dict(

31 + model="test-model",

32 + input=[{"role": "user", "content": "Hello"}],

33 + )

34 + defaults.update(kwargs)

35 + return ResponsesRequest(**defaults)

36 +

37 +

38 +class TestChatCompletionReasoningEffort:

39 + """Chat Completions: reasoning_effort -> enable_thinking."""

40 +

41 + @pytest.mark.parametrize("effort", ["low", "medium", "high"])

42 + def test_non_none_effort_injects_enable_thinking_true(self, effort):

43 + request = _build_chat_request(reasoning_effort=effort)

44 + params = request.build_chat_params(None, "auto")

45 + assert params.chat_template_kwargs["enable_thinking"] is True

46 +

47 + def test_none_effort_injects_enable_thinking_false(self):

48 + request = _build_chat_request(reasoning_effort="none")

49 + params = request.build_chat_params(None, "auto")

50 + assert params.chat_template_kwargs["enable_thinking"] is False

51 +

52 + def test_no_effort_does_not_inject(self):

53 + request = _build_chat_request()

54 + params = request.build_chat_params(None, "auto")

55 + assert "enable_thinking" not in params.chat_template_kwargs

56 +

57 + def test_explicit_user_kwarg_not_overridden(self):

58 + request = _build_chat_request(

59 + reasoning_effort="high",

60 + chat_template_kwargs={"enable_thinking": False},

61 + )

62 + params = request.build_chat_params(None, "auto")

63 + assert params.chat_template_kwargs["enable_thinking"] is False

64 +

65 + def test_reasoning_effort_still_in_kwargs(self):

66 + request = _build_chat_request(reasoning_effort="high")

67 + params = request.build_chat_params(None, "auto")

68 + assert params.chat_template_kwargs["reasoning_effort"] == "high"

69 +

70 +

71 +class TestResponsesReasoningEffort:

72 + """Responses API: reasoning.effort -> enable_thinking."""

73 +

74 + @pytest.mark.parametrize("effort", ["low", "medium", "high"])

75 + def test_non_none_effort_injects_enable_thinking_true(self, effort):

76 + request = _build_responses_request(

77 + reasoning=Reasoning(effort=effort),

78 + )

79 + params = request.build_chat_params(None, "auto")

80 + assert params.chat_template_kwargs["enable_thinking"] is True

81 +

82 + def test_none_effort_injects_enable_thinking_false(self):

83 + request = _build_responses_request(

84 + reasoning=Reasoning(effort="none"),

85 + )

86 + params = request.build_chat_params(None, "auto")

87 + assert params.chat_template_kwargs["enable_thinking"] is False

88 +

89 + def test_no_reasoning_does_not_inject(self):

90 + request = _build_responses_request()

91 + params = request.build_chat_params(None, "auto")

92 + assert "enable_thinking" not in params.chat_template_kwargs

93 +

94 + def test_explicit_user_kwarg_not_overridden(self):

95 + request = _build_responses_request(

96 + reasoning=Reasoning(effort="high"),

97 + chat_template_kwargs={"enable_thinking": False},

98 + )

99 + params = request.build_chat_params(None, "auto")

100 + assert params.chat_template_kwargs["enable_thinking"] is False

101 +

102 + def test_reasoning_effort_still_in_kwargs(self):

103 + request = _build_responses_request(

104 + reasoning=Reasoning(effort="high"),

105 + )

106 + params = request.build_chat_params(None, "auto")

107 + assert params.chat_template_kwargs["reasoning_effort"] == "high"

vllm/entrypoints/openai/chat_completion/protocol.py

+16 -6

@@ -473,17 +473,27 @@ class ChatCompletionRequest(OpenAIBaseModel):

473 default_template: str | None,

474 default_template_content_format: ChatTemplateContentFormatOption,

475 ) -> ChatParams:

476 + extra_kwargs: dict[str, Any] = dict(

477 + add_generation_prompt=self.add_generation_prompt,

478 + continue_final_message=self.continue_final_message,

479 + documents=self.documents,

480 + reasoning_effort=self.reasoning_effort,

481 + )

482 +

483 + # When reasoning is requested, activate thinking for models whose

484 + # chat templates require explicit opt-in (e.g., Gemma4 defaults

485 + # enable_thinking to false). For templates that don't declare the

486 + # variable, resolve_chat_template_kwargs filters it out harmlessly.

487 + user_kwargs = self.chat_template_kwargs or {}

488 + if self.reasoning_effort is not None and "enable_thinking" not in user_kwargs:

489 + extra_kwargs["enable_thinking"] = self.reasoning_effort != "none"

490 +

491 return ChatParams(

492 chat_template=self.chat_template or default_template,

493 chat_template_content_format=default_template_content_format,

494 chat_template_kwargs=merge_kwargs(

495 self.chat_template_kwargs,

481 - dict(

482 - add_generation_prompt=self.add_generation_prompt,

483 - continue_final_message=self.continue_final_message,

484 - documents=self.documents,

485 - reasoning_effort=self.reasoning_effort,

486 - ),

496 + extra_kwargs,

497 ),

498 media_io_kwargs=self.media_io_kwargs,

499 )

vllm/entrypoints/openai/responses/protocol.py

+17 -6

@@ -298,17 +298,28 @@ class ResponsesRequest(OpenAIBaseModel):

298 continue_final = should_continue_final_message(self.input)

299

300 reasoning = self.reasoning

301 + reasoning_effort = None if reasoning is None else reasoning.effort

302 +

303 + extra_kwargs: dict[str, Any] = dict(

304 + add_generation_prompt=not continue_final,

305 + continue_final_message=continue_final,

306 + reasoning_effort=reasoning_effort,

307 + )

308 +

309 + # When reasoning is requested, activate thinking for models whose

310 + # chat templates require explicit opt-in (e.g., Gemma4 defaults

311 + # enable_thinking to false). For templates that don't declare the

312 + # variable, resolve_chat_template_kwargs filters it out harmlessly.

313 + user_kwargs = self.chat_template_kwargs or {}

314 + if reasoning_effort is not None and "enable_thinking" not in user_kwargs:

315 + extra_kwargs["enable_thinking"] = reasoning_effort != "none"

316

317 return ChatParams(

318 chat_template=default_template,

319 chat_template_content_format=default_template_content_format,

305 - chat_template_kwargs=merge_kwargs( # To remove unset values

320 + chat_template_kwargs=merge_kwargs(

321 self.chat_template_kwargs,

307 - dict(

308 - add_generation_prompt=not continue_final,

309 - continue_final_message=continue_final,

310 - reasoning_effort=None if reasoning is None else reasoning.effort,

311 - ),

322 + extra_kwargs,

323 ),

324 media_io_kwargs=self.media_io_kwargs,

325 )