@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf33700ecd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 21:00 UTC

[v0][structured output] Support reasoning output (#12955)

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

Ce Gao committed Mar 3, 2025 at 03:49 UTC bf33700ecd6db472c4aeb489c5d42aa47a735198
16 files changed +400 -76
docs/source/features/reasoning_outputs.md
+35 -8
@@ -76,7 +76,13 @@ Streaming chat completions are also supported for reasoning models. The `reasoni
76 }
77 ```
78
79 -Please note that it is not compatible with the OpenAI Python client library. You can use the `requests` library to make streaming requests.
79 +Please note that it is not compatible with the OpenAI Python client library. You can use the `requests` library to make streaming requests. You could checkout the [example](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py).
80 +
81 +## Limitations
82 +
83 +- The reasoning content is only available for online serving's chat completion endpoint (`/v1/chat/completions`).
84 +- It is not compatible with [`tool_calling`](#tool_calling).
85 +- The reasoning content is not available for all models. Check the model's documentation to see if it supports reasoning.
86
87 ## How to support a new reasoning model
88
@@ -137,15 +143,36 @@ class ExampleParser(ReasoningParser):
143 """
144 ```
145
140 -After defining the reasoning parser, you can use it by specifying the `--reasoning-parser` flag when making a request to the chat completion endpoint.
146 +Additionally, to enable structured output, you'll need to create a new `Reasoner` similar to the one in `vllm/model_executor/guided_decoding/reasoner/deepseek_reasoner.py`.
147 +
148 +```python
149 +@dataclass
150 +class DeepSeekReasoner(Reasoner):
151 + """
152 + Reasoner for DeepSeek R series models.
153 + """
154 + start_token_id: int
155 + end_token_id: int
156 +
157 + start_token: str = "<think>"
158 + end_token: str = "</think>"
159 +
160 + @classmethod
161 + def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:
162 + return cls(start_token_id=tokenizer.encode(
163 + "<think>", add_special_tokens=False)[0],
164 + end_token_id=tokenizer.encode("</think>",
165 + add_special_tokens=False)[0])
166 +
167 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
168 + return self.end_token_id in input_ids
169 +```
170 +
171 +The structured output engine like xgrammar will use `end_token_id` to check if the reasoning content is present in the model output and skip the structured output if it is the case.
172 +
173 +Finally, you can enable reasoning for the model by using the `--enable-reasoning` and `--reasoning-parser` flags.
174
175 ```bash
176 vllm serve <model_tag> \
177 --enable-reasoning --reasoning-parser example
178 ```
146 -
147 -## Limitations
148 -
149 -- The reasoning content is only available for online serving's chat completion endpoint (`/v1/chat/completions`).
150 -- It is not compatible with the [`structured_outputs`](#structured_outputs) and [`tool_calling`](#tool_calling) features.
151 -- The reasoning content is not available for all models. Check the model's documentation to see if it supports reasoning.
examples/online_serving/openai_chat_completion_structured_outputs_with_reasoning.py new
+64
@@ -0,0 +1,64 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""
3 +An example shows how to generate structured outputs from reasoning models
4 +like DeepSeekR1. The thinking process will not be guided by the JSON
5 +schema provided by the user. Only the final output will be structured.
6 +
7 +To run this example, you need to start the vLLM server with the reasoning
8 +parser:
9 +
10 +```bash
11 +vllm serve deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \
12 + --enable-reasoning --reasoning-parser deepseek_r1
13 +```
14 +
15 +This example demonstrates how to generate chat completions from reasoning models
16 +using the OpenAI Python client library.
17 +"""
18 +
19 +from enum import Enum
20 +
21 +from openai import OpenAI
22 +from pydantic import BaseModel
23 +
24 +# Modify OpenAI's API key and API base to use vLLM's API server.
25 +openai_api_key = "EMPTY"
26 +openai_api_base = "http://localhost:8000/v1"
27 +
28 +client = OpenAI(
29 + api_key=openai_api_key,
30 + base_url=openai_api_base,
31 +)
32 +
33 +models = client.models.list()
34 +model = models.data[0].id
35 +
36 +
37 +# Guided decoding by JSON using Pydantic schema
38 +class CarType(str, Enum):
39 + sedan = "sedan"
40 + suv = "SUV"
41 + truck = "Truck"
42 + coupe = "Coupe"
43 +
44 +
45 +class CarDescription(BaseModel):
46 + brand: str
47 + model: str
48 + car_type: CarType
49 +
50 +
51 +json_schema = CarDescription.model_json_schema()
52 +
53 +prompt = ("Generate a JSON with the brand, model and car_type of"
54 + "the most iconic car from the 90's, think in 100 tokens")
55 +completion = client.chat.completions.create(
56 + model=model,
57 + messages=[{
58 + "role": "user",
59 + "content": prompt,
60 + }],
61 + extra_body={"guided_json": json_schema},
62 +)
63 +print("content", completion.choices[0].message.content)
64 +print("reasoning_content: ", completion.choices[0].message.reasoning_content)
tests/model_executor/test_guided_processors.py
+102 -14
@@ -16,17 +16,33 @@ from vllm.sampling_params import GuidedDecodingParams
16
17 MODEL_NAME = 'HuggingFaceH4/zephyr-7b-beta'
18 GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
19 +GUIDED_DECODING_BACKENDS_WITH_REASONING_SUPPORT = ["outlines", "xgrammar"]
20 +REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
21
22
21 -def test_guided_logits_processors(sample_regex, sample_json_schema):
23 +# Initialize the tokenizer for the model here to avoid repeated loading
24 +@pytest.fixture(scope="module")
25 +def zephyr_7B_tokenzer():
26 + return AutoTokenizer.from_pretrained(MODEL_NAME)
27 +
28 +
29 +@pytest.fixture(scope="module")
30 +def deepseek_r1_qwen_tokenizer():
31 + return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
32 +
33 +
34 +def test_guided_logits_processors(zephyr_7B_tokenzer, sample_regex,
35 + sample_json_schema):
36 """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
23 - tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
24 - regex_LP = RegexLogitsProcessor(sample_regex, tokenizer)
37 + regex_LP = RegexLogitsProcessor(sample_regex,
38 + zephyr_7B_tokenzer,
39 + reasoner=None)
40 json_LP = JSONLogitsProcessor(sample_json_schema,
26 - tokenizer,
27 - whitespace_pattern=None)
41 + zephyr_7B_tokenzer,
42 + whitespace_pattern=None,
43 + reasoner=None)
44
29 - token_ids = tokenizer.encode(
45 + token_ids = zephyr_7B_tokenzer.encode(
46 f"Give an example IPv4 address with this regex: {sample_regex}")
47 tensor = torch.rand(32000)
48 original_tensor = torch.clone(tensor)
@@ -34,7 +50,7 @@ def test_guided_logits_processors(sample_regex, sample_json_schema):
50 assert tensor.shape == original_tensor.shape
51 assert not torch.allclose(tensor, original_tensor)
52
37 - token_ids = tokenizer.encode(
53 + token_ids = zephyr_7B_tokenzer.encode(
54 f"Give an employee profile that fits this schema: {sample_json_schema}"
55 )
56 tensor = torch.rand(32000)
@@ -49,7 +65,8 @@ def test_guided_logits_processors(sample_regex, sample_json_schema):
65 @pytest.mark.parametrize("is_local", [True, False])
66 async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
67 sample_regex,
52 - sample_json_schema):
68 + sample_json_schema,
69 + zephyr_7B_tokenzer):
70
71 config = ModelConfig(
72 MODEL_NAME,
@@ -60,15 +77,14 @@ async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
77 seed=0,
78 dtype="bfloat16",
79 )
63 - tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
64 - token_ids = tokenizer.encode(
80 + token_ids = zephyr_7B_tokenzer.encode(
81 f"Give an example IPv4 address with this regex: {sample_regex}")
82 regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
83
84 regex_lp = get_local_guided_decoding_logits_processor(
69 - regex_request, tokenizer, config) if is_local else \
85 + regex_request, zephyr_7B_tokenzer, config) if is_local else \
86 await get_guided_decoding_logits_processor(
71 - regex_request, tokenizer, config)
87 + regex_request, zephyr_7B_tokenzer, config)
88 assert regex_lp is not None
89 tensor = torch.rand(32000)
90 original_tensor = torch.clone(tensor)
@@ -76,13 +92,85 @@ async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
92 assert tensor.shape == original_tensor.shape
93 assert not torch.allclose(tensor, original_tensor)
94
79 - token_ids = tokenizer.encode(
95 + token_ids = zephyr_7B_tokenzer.encode(
96 f"Give an employee profile that fits this schema: {sample_json_schema}"
97 )
98 json_request = GuidedDecodingParams(json=sample_json_schema,
99 backend=backend)
100 json_lp = await get_guided_decoding_logits_processor(
85 - json_request, tokenizer, config)
101 + json_request, zephyr_7B_tokenzer, config)
102 + assert json_lp is not None
103 + tensor = torch.rand(32000)
104 + original_tensor = torch.clone(tensor)
105 + tensor = json_lp(token_ids, tensor)
106 + assert tensor.shape == original_tensor.shape
107 + assert not torch.allclose(tensor, original_tensor)
108 +
109 +
110 +@pytest.mark.asyncio
111 +@pytest.mark.parametrize("backend",
112 + GUIDED_DECODING_BACKENDS_WITH_REASONING_SUPPORT)
113 +@pytest.mark.parametrize("is_local", [True, False])
114 +@pytest.mark.parametrize("reasoning_backend", ["deepseek_r1"])
115 +async def test_guided_logits_processor_with_reasoning(
116 + backend: str, is_local: bool, reasoning_backend: str, sample_regex,
117 + sample_json_schema, deepseek_r1_qwen_tokenizer):
118 +
119 + config = ModelConfig(
120 + REASONING_MODEL_NAME,
121 + task="generate",
122 + tokenizer=REASONING_MODEL_NAME,
123 + tokenizer_mode="auto",
124 + trust_remote_code=False,
125 + seed=0,
126 + dtype="bfloat16",
127 + )
128 + token_ids = deepseek_r1_qwen_tokenizer.encode(
129 + f"Give an example IPv4 address with this regex: {sample_regex}."
130 + "<think>here is the thinking process")
131 + regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
132 +
133 + regex_lp = get_local_guided_decoding_logits_processor(regex_request,
134 + deepseek_r1_qwen_tokenizer, config,
135 + reasoning_backend) if is_local else \
136 + await get_guided_decoding_logits_processor(
137 + regex_request, deepseek_r1_qwen_tokenizer, config,
138 + reasoning_backend)
139 + assert regex_lp is not None
140 + tensor = torch.rand(32000)
141 + original_tensor = torch.clone(tensor)
142 + tensor = regex_lp(token_ids, tensor)
143 + assert tensor.shape == original_tensor.shape
144 + assert torch.allclose(tensor, original_tensor)
145 +
146 + token_ids = deepseek_r1_qwen_tokenizer.encode(
147 + f"Give an employee profile that fits this schema: {sample_json_schema}."
148 + "<think>here is the thinking process")
149 + json_request = GuidedDecodingParams(json=sample_json_schema,
150 + backend=backend)
151 + json_lp = get_local_guided_decoding_logits_processor(
152 + json_request, deepseek_r1_qwen_tokenizer, config,
153 + reasoning_backend) if is_local else \
154 + await get_guided_decoding_logits_processor(
155 + json_request, deepseek_r1_qwen_tokenizer, config, reasoning_backend)
156 + assert json_lp is not None
157 + tensor = torch.rand(32000)
158 + original_tensor = torch.clone(tensor)
159 + tensor = json_lp(token_ids, tensor)
160 + assert tensor.shape == original_tensor.shape
161 + assert torch.allclose(tensor, original_tensor)
162 +
163 + # Thinking is over, so the tensor should change.
164 + token_ids = deepseek_r1_qwen_tokenizer.encode(
165 + f"Give an employee profile that fits this schema: {sample_json_schema}."
166 + "<think>here is the thinking process</think> Then")
167 + json_request = GuidedDecodingParams(json=sample_json_schema,
168 + backend=backend)
169 + json_lp = get_local_guided_decoding_logits_processor(
170 + json_request, deepseek_r1_qwen_tokenizer, config,
171 + reasoning_backend) if is_local else \
172 + await get_guided_decoding_logits_processor(
173 + json_request, deepseek_r1_qwen_tokenizer, config, reasoning_backend)
174 assert json_lp is not None
175 tensor = torch.rand(32000)
176 original_tensor = torch.clone(tensor)
vllm/config.py
+2
@@ -2715,6 +2715,8 @@ class DecodingConfig:
2715 # 'outlines' / 'lm-format-enforcer' / 'xgrammar'
2716 guided_decoding_backend: str = 'xgrammar'
2717
2718 + reasoning_backend: Optional[str] = None
2719 +
2720 def compute_hash(self) -> str:
2721 """
2722 WARNING: Whenever a new field is added to this config,
vllm/engine/arg_utils.py
+25 -1
@@ -213,6 +213,8 @@ class EngineArgs:
213 calculate_kv_scales: Optional[bool] = None
214
215 additional_config: Optional[Dict[str, Any]] = None
216 + enable_reasoning: Optional[bool] = None
217 + reasoning_parser: Optional[str] = None
218
219 def __post_init__(self):
220 if not self.tokenizer:
@@ -1059,6 +1061,25 @@ class EngineArgs:
1061 "Different platforms may support different configs. Make sure the "
1062 "configs are valid for the platform you are using. The input format"
1063 " is like '{\"config_key\":\"config_value\"}'")
1064 +
1065 + parser.add_argument(
1066 + "--enable-reasoning",
1067 + action="store_true",
1068 + default=False,
1069 + help="Whether to enable reasoning_content for the model. "
1070 + "If enabled, the model will be able to generate reasoning content."
1071 + )
1072 +
1073 + parser.add_argument(
1074 + "--reasoning-parser",
1075 + type=str,
1076 + choices=["deepseek_r1"],
1077 + default=None,
1078 + help=
1079 + "Select the reasoning parser depending on the model that you're "
1080 + "using. This is used to parse the reasoning content into OpenAI "
1081 + "API format. Required for ``--enable-reasoning``.")
1082 +
1083 return parser
1084
1085 @classmethod
@@ -1332,7 +1353,10 @@ class EngineArgs:
1353 if self.enable_prompt_adapter else None
1354
1355 decoding_config = DecodingConfig(
1335 - guided_decoding_backend=self.guided_decoding_backend)
1356 + guided_decoding_backend=self.guided_decoding_backend,
1357 + reasoning_backend=self.reasoning_parser
1358 + if self.enable_reasoning else None,
1359 + )
1360
1361 show_hidden_metrics = False
1362 if self.show_hidden_metrics_for_version is not None:
vllm/engine/async_llm_engine.py
+8 -3
@@ -509,6 +509,7 @@ class _AsyncLLMEngine(LLMEngine):
509 tokenizer=await self.get_tokenizer_async(lora_request),
510 default_guided_backend=self.decoding_config.
511 guided_decoding_backend,
512 + reasoning_backend=self.decoding_config.reasoning_backend,
513 model_config=self.model_config)
514
515 self._add_processed_request(
@@ -530,7 +531,7 @@ class _AsyncLLMEngine(LLMEngine):
531
532 async def build_guided_decoding_logits_processor_async(
533 sampling_params: SamplingParams, tokenizer: AnyTokenizer,
533 - default_guided_backend: str,
534 + default_guided_backend: str, reasoning_backend: Optional[str],
535 model_config: ModelConfig) -> SamplingParams:
536 """Constructs logits processors based on the guided_decoding,
537 logits_bias, and allowed_token_ids fields in sampling_params. Deletes
@@ -545,14 +546,18 @@ async def build_guided_decoding_logits_processor_async(
546 sampling_params = copy.copy(sampling_params)
547 guided_decoding = sampling_params.guided_decoding
548
548 - logger.debug("Building guided decoding logits processor. "
549 - "Params: %s", guided_decoding)
549 + logger.info(
550 + "Building guided decoding logits processor. "
551 + "guided_decoding: %s%s", guided_decoding,
552 + f", reasoning_backend: {reasoning_backend}"
553 + if reasoning_backend is not None else "")
554
555 guided_decoding.backend = guided_decoding.backend or default_guided_backend
556
557 processor = await get_guided_decoding_logits_processor(
558 guided_params=guided_decoding,
559 tokenizer=tokenizer,
560 + reasoning_backend=reasoning_backend,
561 model_config=model_config)
562
563 if processor:
vllm/engine/llm_engine.py
+6 -1
@@ -2048,10 +2048,15 @@ class LLMEngine:
2048 guided_decoding.backend = guided_decoding.backend or \
2049 self.decoding_config.guided_decoding_backend
2050
2051 + logger.debug("Reasoning backend: %s",
2052 + self.decoding_config.reasoning_backend)
2053 +
2054 processor = get_local_guided_decoding_logits_processor(
2055 guided_params=guided_decoding,
2056 tokenizer=tokenizer,
2054 - model_config=self.model_config)
2057 + model_config=self.model_config,
2058 + reasoning_backend=self.decoding_config.reasoning_backend,
2059 + )
2060 if processor:
2061 logits_processors.append(processor)
2062
vllm/engine/multiprocessing/client.py
+2 -1
@@ -611,7 +611,8 @@ class MQLLMEngineClient(EngineClient):
611 default_guided_backend=(self.decoding_config.guided_decoding_backend
612 if self.decoding_config
613 else DecodingConfig.guided_decoding_backend),
614 - model_config=self.model_config
614 + model_config=self.model_config,
615 + reasoning_backend=self.decoding_config.reasoning_backend,
616 )
617
618 # 1) Create output queue for this requests.
vllm/entrypoints/openai/cli_args.py
-18
@@ -13,7 +13,6 @@ from typing import List, Optional, Sequence, Union, get_args
13 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
14 from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
15 validate_chat_template)
16 -from vllm.entrypoints.openai.reasoning_parsers import ReasoningParserManager
16 from vllm.entrypoints.openai.serving_models import (LoRAModulePath,
17 PromptAdapterPath)
18 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
@@ -215,23 +214,6 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
214 default=False,
215 help="Enable auto tool choice for supported models. Use "
216 "``--tool-call-parser`` to specify which parser to use.")
218 - parser.add_argument(
219 - "--enable-reasoning",
220 - action="store_true",
221 - default=False,
222 - help="Whether to enable reasoning_content for the model. "
223 - "If enabled, the model will be able to generate reasoning content.")
224 -
225 - valid_reasoning_parsers = ReasoningParserManager.reasoning_parsers.keys()
226 - parser.add_argument(
227 - "--reasoning-parser",
228 - type=str,
229 - metavar="{" + ",".join(valid_reasoning_parsers) + "}",
230 - default=None,
231 - help=
232 - "Select the reasoning parser depending on the model that you're using."
233 - " This is used to parse the reasoning content into OpenAI API "
234 - "format. Required for ``--enable-reasoning``.")
217
218 valid_tool_parsers = ToolParserManager.tool_parsers.keys()
219 parser.add_argument(
vllm/model_executor/guided_decoding/__init__.py
+21 -9
@@ -5,6 +5,7 @@ from __future__ import annotations
5 from typing import TYPE_CHECKING
6
7 from vllm.logger import init_logger
8 +from vllm.model_executor.guided_decoding.reasoner import get_reasoner
9 from vllm.model_executor.guided_decoding.utils import (
10 convert_lark_to_gbnf, grammar_is_likely_lark,
11 has_lmf_unsupported_json_features, has_xgrammar_unsupported_json_features)
@@ -103,8 +104,13 @@ def maybe_backend_fallback(
104
105
106 async def get_guided_decoding_logits_processor(
106 - guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizer,
107 - model_config: ModelConfig) -> LogitsProcessor | None:
107 + guided_params: GuidedDecodingParams,
108 + tokenizer: PreTrainedTokenizer,
109 + model_config: ModelConfig,
110 + reasoning_backend: str | None = None) -> LogitsProcessor | None:
111 +
112 + reasoner = get_reasoner(tokenizer, reasoning_backend)
113 +
114 guided_params = maybe_backend_fallback(guided_params)
115 # CFG grammar not supported by LMFE, so we use outlines instead
116 if guided_params.backend_name == 'outlines':
@@ -112,8 +118,8 @@ async def get_guided_decoding_logits_processor(
118 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
119 get_outlines_guided_decoding_logits_processor)
120 return await get_outlines_guided_decoding_logits_processor(
115 - guided_params, tokenizer)
116 - if guided_params.backend_name == 'lm-format-enforcer':
121 + guided_params, tokenizer, reasoner)
122 + if guided_params.backend == 'lm-format-enforcer':
123 from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
124 get_local_lm_format_enforcer_guided_decoding_logits_processor)
125 return get_local_lm_format_enforcer_guided_decoding_logits_processor(
@@ -122,7 +128,7 @@ async def get_guided_decoding_logits_processor(
128 from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
129 get_local_xgrammar_guided_decoding_logits_processor)
130 return get_local_xgrammar_guided_decoding_logits_processor(
125 - guided_params, tokenizer, model_config)
131 + guided_params, tokenizer, model_config, reasoner)
132
133 raise ValueError(
134 f"Unknown guided decoding backend '{guided_params.backend}'. "
@@ -130,16 +136,22 @@ async def get_guided_decoding_logits_processor(
136
137
138 def get_local_guided_decoding_logits_processor(
133 - guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizer,
134 - model_config: ModelConfig) -> LogitsProcessor | None:
139 + guided_params: GuidedDecodingParams,
140 + tokenizer: PreTrainedTokenizer,
141 + model_config: ModelConfig,
142 + reasoning_backend: str | None = None) -> LogitsProcessor | None:
143 guided_params = maybe_backend_fallback(guided_params)
144 +
145 + # Get the reasoner if needed, it will be None if reasoning_
146 + reasoner = get_reasoner(tokenizer, reasoning_backend)
147 +
148 # CFG grammar not supported by LMFE, so we use outlines instead
149 if guided_params.backend_name == 'outlines':
150 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
151 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
152 get_local_outlines_guided_decoding_logits_processor)
153 return get_local_outlines_guided_decoding_logits_processor(
142 - guided_params, tokenizer)
154 + guided_params, tokenizer, reasoner)
155 if guided_params.backend_name == 'lm-format-enforcer':
156 from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
157 get_local_lm_format_enforcer_guided_decoding_logits_processor)
@@ -149,7 +161,7 @@ def get_local_guided_decoding_logits_processor(
161 from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
162 get_local_xgrammar_guided_decoding_logits_processor)
163 return get_local_xgrammar_guided_decoding_logits_processor(
152 - guided_params, tokenizer, model_config)
164 + guided_params, tokenizer, model_config, reasoner)
165
166 raise ValueError(
167 f"Unknown guided decoding backend '{guided_params.backend}'. "
vllm/model_executor/guided_decoding/outlines_decoding.py
+20 -10
@@ -6,12 +6,13 @@ import os
6 from enum import Enum
7 from json import dumps as json_dumps
8 from re import escape as regex_escape
9 -from typing import Tuple, Union
9 +from typing import Optional, Tuple, Union
10
11 from transformers import PreTrainedTokenizerBase
12
13 from vllm.model_executor.guided_decoding.outlines_logits_processors import (
14 CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
15 +from vllm.model_executor.guided_decoding.reasoner import Reasoner
16 from vllm.sampling_params import GuidedDecodingParams
17
18
@@ -58,7 +59,9 @@ _MAX_THREADPOOL_WORKERS = 16
59
60
61 async def get_outlines_guided_decoding_logits_processor(
61 - guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
62 + guided_params: GuidedDecodingParams,
63 + tokenizer: PreTrainedTokenizerBase,
64 + reasoner: Optional[Reasoner],
65 ) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
66 None]:
67 """
@@ -82,11 +85,14 @@ async def get_outlines_guided_decoding_logits_processor(
85
86 return await loop.run_in_executor(global_thread_pool,
87 _get_logits_processor, guide, tokenizer,
85 - mode, guided_params.whitespace_pattern)
88 + mode, guided_params.whitespace_pattern,
89 + reasoner)
90
91
92 def get_local_outlines_guided_decoding_logits_processor(
89 - guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
93 + guided_params: GuidedDecodingParams,
94 + tokenizer: PreTrainedTokenizerBase,
95 + reasoner: Optional[Reasoner],
96 ) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
97 None]:
98 """
@@ -100,7 +106,7 @@ def get_local_outlines_guided_decoding_logits_processor(
106 return None
107
108 return _get_logits_processor(guide, tokenizer, mode,
103 - guided_params.whitespace_pattern)
109 + guided_params.whitespace_pattern, reasoner)
110
111
112 def _get_guide_and_mode(
@@ -131,14 +137,18 @@ def _get_guide_and_mode(
137
138
139 def _get_logits_processor(
134 - guide: str, tokenizer: PreTrainedTokenizerBase, mode: GuidedDecodingMode,
135 - whitespace_pattern: Union[str, None]
140 + guide: str,
141 + tokenizer: PreTrainedTokenizerBase,
142 + mode: GuidedDecodingMode,
143 + whitespace_pattern: Union[str, None],
144 + reasoner: Optional[Reasoner],
145 ) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor]:
146 if mode == GuidedDecodingMode.JSON:
138 - return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern)
147 + return JSONLogitsProcessor(guide, tokenizer, whitespace_pattern,
148 + reasoner)
149 elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
140 - return RegexLogitsProcessor(guide, tokenizer)
150 + return RegexLogitsProcessor(guide, tokenizer, reasoner)
151 elif mode == GuidedDecodingMode.GRAMMAR:
142 - return CFGLogitsProcessor(guide, tokenizer)
152 + return CFGLogitsProcessor(guide, tokenizer, reasoner)
153 else:
154 raise ValueError(f"Unknown guided decoding mode {mode}")
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+29 -8
@@ -19,7 +19,7 @@ import copy
19 import json
20 from collections import defaultdict
21 from functools import lru_cache
22 -from typing import Callable, DefaultDict, Dict, List, Union
22 +from typing import Callable, DefaultDict, Dict, List, Optional, Union
23
24 import numpy as np
25 import torch
@@ -32,13 +32,18 @@ from outlines_core.fsm.json_schema import build_regex_from_schema
32 from pydantic import BaseModel
33 from transformers import PreTrainedTokenizerBase
34
35 +from vllm.logger import init_logger
36 +from vllm.model_executor.guided_decoding.reasoner import Reasoner
37 from vllm.platforms import current_platform
38
39 +logger = init_logger(__name__)
40 +
41
42 class BaseLogitsProcessor:
43
40 - def __init__(self, guide: Guide):
44 + def __init__(self, guide: Guide, reasoner: Optional[Reasoner]):
45 self._guide: Guide = guide
46 + self._reasoner = reasoner
47 # CFGState is used for the FSM state for CFGGuide
48 self._fsm_state: DefaultDict[int, Union[int,
49 CFGState]] = defaultdict(int)
@@ -46,6 +51,14 @@ class BaseLogitsProcessor:
51 def __call__(self, input_ids: List[int],
52 scores: torch.Tensor) -> torch.Tensor:
53 """Use the FSM to bias the logits before sampling the next token."""
54 +
55 + # Skip the structured logits processing if reasoning is not finished.
56 + # reasoner is not None only when `--enable-reasoning` is set.
57 + if self._reasoner is not None and \
58 + not self._reasoner.is_reasoning_end(
59 + input_ids):
60 + return scores
61 +
62 seq_id = hash(tuple(input_ids))
63
64 if len(input_ids) > 0:
@@ -113,7 +126,12 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
126 tokenizer = _adapt_tokenizer(tokenizer)
127 return RegexGuide.from_regex(regex_string, tokenizer)
128
116 - def __init__(self, regex_string: str, tokenizer: PreTrainedTokenizerBase):
129 + def __init__(
130 + self,
131 + regex_string: str,
132 + tokenizer: PreTrainedTokenizerBase,
133 + reasoner: Optional[Reasoner],
134 + ):
135 """Compile the FSM that drives the regex-structured generation.
136
137 Parameters
@@ -125,14 +143,15 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
143
144 """
145 super().__init__(
128 - RegexLogitsProcessor._get_guide(regex_string, tokenizer))
146 + RegexLogitsProcessor._get_guide(regex_string, tokenizer), reasoner)
147
148
149 class JSONLogitsProcessor(RegexLogitsProcessor):
150
151 def __init__(self, schema: Union[str, Dict, BaseModel],
152 tokenizer: PreTrainedTokenizerBase,
135 - whitespace_pattern: Union[str, None]):
153 + whitespace_pattern: Union[str, None],
154 + reasoner: Optional[Reasoner]):
155 """Compile the FSM that drives the JSON-guided generation.
156
157 Parameters
@@ -160,7 +179,7 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
179 f"a Pydantic object, a dictionary or a string that contains "
180 f"the JSON Schema specification")
181 regex_string = build_regex_from_schema(schema_str, whitespace_pattern)
163 - super().__init__(regex_string, tokenizer)
182 + super().__init__(regex_string, tokenizer, reasoner)
183
184
185 class CFGLogitsProcessor(BaseLogitsProcessor):
@@ -171,7 +190,8 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
190 tokenizer = _adapt_tokenizer(tokenizer)
191 return CFGGuide(cfg, tokenizer)
192
174 - def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase):
193 + def __init__(self, cfg: str, tokenizer: PreTrainedTokenizerBase,
194 + reasoner: Optional[Reasoner]):
195 """Compile the FSM that drives the context free grammar generation.
196
197 Parameters
@@ -182,7 +202,8 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
202 The model's tokenizer
203
204 """
185 - super().__init__(CFGLogitsProcessor._get_guide(cfg, tokenizer))
205 + super().__init__(CFGLogitsProcessor._get_guide(cfg, tokenizer),
206 + reasoner)
207 self._guide = self._guide.copy()
208
209
vllm/model_executor/guided_decoding/reasoner/__init__.py new
+23
@@ -0,0 +1,23 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from __future__ import annotations
4 +
5 +from transformers import PreTrainedTokenizer
6 +
7 +from vllm.model_executor.guided_decoding.reasoner.deepseek_reasoner import ( # noqa: E501
8 + DeepSeekReasoner)
9 +from vllm.model_executor.guided_decoding.reasoner.reasoner import Reasoner
10 +
11 +
12 +def get_reasoner(tokenizer: PreTrainedTokenizer,
13 + reasoning_backend: str | None) -> Reasoner | None:
14 + if reasoning_backend is None:
15 + # No reasoning backend specified
16 + return None
17 + elif reasoning_backend == "deepseek_r1":
18 + return DeepSeekReasoner.from_tokenizer(tokenizer)
19 + else:
20 + raise ValueError(f"Unknown reasoning backend '{reasoning_backend}'")
21 +
22 +
23 +__all__ = ["Reasoner", "get_reasoner"]
vllm/model_executor/guided_decoding/reasoner/deepseek_reasoner.py new
+28
@@ -0,0 +1,28 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +from dataclasses import dataclass
3 +
4 +from transformers import PreTrainedTokenizer
5 +
6 +from vllm.model_executor.guided_decoding.reasoner.reasoner import Reasoner
7 +
8 +
9 +@dataclass
10 +class DeepSeekReasoner(Reasoner):
11 + """
12 + Reasoner for DeepSeek R series models.
13 + """
14 + start_token_id: int
15 + end_token_id: int
16 +
17 + start_token: str = "<think>"
18 + end_token: str = "</think>"
19 +
20 + @classmethod
21 + def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:
22 + return cls(start_token_id=tokenizer.encode(
23 + "<think>", add_special_tokens=False)[0],
24 + end_token_id=tokenizer.encode("</think>",
25 + add_special_tokens=False)[0])
26 +
27 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
28 + return self.end_token_id in input_ids
vllm/model_executor/guided_decoding/reasoner/reasoner.py new
+19
@@ -0,0 +1,19 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +from __future__ import annotations
3 +
4 +from abc import ABC, abstractmethod
5 +from dataclasses import dataclass
6 +
7 +from transformers import PreTrainedTokenizer
8 +
9 +
10 +@dataclass
11 +class Reasoner(ABC):
12 +
13 + @abstractmethod
14 + def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:
15 + pass
16 +
17 + @abstractmethod
18 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
19 + pass
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+16 -3
@@ -11,6 +11,8 @@ from typing import TYPE_CHECKING, Any, List
11 import torch
12 from transformers import PreTrainedTokenizerFast
13
14 +from vllm.logger import init_logger
15 +
16 try:
17 import xgrammar as xgr
18 from xgrammar.base import _core as xgr_core
@@ -19,7 +21,6 @@ except ImportError:
21 xgr_installed = False
22 pass
23
22 -from vllm.logger import init_logger
24 from vllm.model_executor.guided_decoding.utils import (convert_lark_to_gbnf,
25 grammar_is_likely_lark)
26 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
@@ -28,6 +29,7 @@ if TYPE_CHECKING:
29 from transformers import PreTrainedTokenizer
30
31 from vllm.config import ModelConfig
32 + from vllm.model_executor.guided_decoding.reasoner import Reasoner
33 from vllm.sampling_params import GuidedDecodingParams
34
35 logger = init_logger(__name__)
@@ -38,12 +40,13 @@ def get_local_xgrammar_guided_decoding_logits_processor(
40 guided_params: GuidedDecodingParams,
41 tokenizer: PreTrainedTokenizer,
42 model_config: ModelConfig,
43 + reasoner: Reasoner | None,
44 max_threads: int = 8):
45 config = GrammarConfig.from_guided_params(guided_params=guided_params,
46 model_config=model_config,
47 tokenizer=tokenizer,
48 max_threads=max_threads)
46 - return XGrammarLogitsProcessor(config)
49 + return XGrammarLogitsProcessor(config, reasoner)
50
51
52 @dataclass(frozen=True)
@@ -293,6 +296,7 @@ class GrammarConfig:
296 class XGrammarLogitsProcessor:
297 """Wrapper class to support pickle protocol"""
298 config: GrammarConfig
299 + reasoner: Reasoner | None = None
300
301 ctx: xgr.CompiledGrammar | None = None
302 token_bitmask: torch.Tensor = None # type: ignore[assignment]
@@ -301,10 +305,11 @@ class XGrammarLogitsProcessor:
305 prefilled: bool = field(default=False)
306
307 def __getstate__(self) -> dict[str, Any]:
304 - return {'config': self.config}
308 + return {'config': self.config, 'reasoner': self.reasoner}
309
310 def __setstate__(self, state: dict[str, Any]):
311 self.config = state['config']
312 + self.reasoner = state['reasoner']
313
314 self.ctx = None
315 self.matchers = []
@@ -331,6 +336,14 @@ class XGrammarLogitsProcessor:
336
337 def __call__(self, input_ids: list[int],
338 scores: torch.Tensor) -> torch.Tensor:
339 +
340 + # Skip the structured logits processing if reasoning is not finished.
341 + # reasoner is not None only when `--enable-reasoning` is set.
342 + if self.reasoner is not None and \
343 + not self.reasoner.is_reasoning_end(
344 + input_ids):
345 + return scores
346 +
347 if self.ctx is None:
348 self._ensure_ctx()
349