@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2fc9075b82
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 09:45 UTC

[V1] Structured Outputs + Thinking compatibility (#16577)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Co-authored-by: Russell Bryant <rbryant@redhat.com>

Aaron Pham committed May 14, 2025 at 18:45 UTC 2fc9075b82e05007f460992b3f9d42d2746c41cb
10 files changed +233 -75
docs/source/features/reasoning_outputs.md
+2 -2
@@ -141,10 +141,10 @@ Remember to check whether the `reasoning_content` exists in the response before
141 The reasoning content is also available in the structured output. The structured output engine like `xgrammar` will use the reasoning content to generate structured output. It is only supported in v0 engine now.
142
143 ```bash
144 -VLLM_USE_V1=0 vllm serve deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --reasoning-parser deepseek_r1
144 +vllm serve deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --reasoning-parser deepseek_r1
145 ```
146
147 -Please note that the `VLLM_USE_V1` environment variable must be set to `0` to use the v0 engine.
147 +The following is an example client:
148
149 ```python
150 from openai import OpenAI
tests/v1/entrypoints/llm/test_struct_output_generate.py
+92 -4
@@ -1,3 +1,4 @@
1 +# ruff: noqa: E501
2 # SPDX-License-Identifier: Apache-2.0
3
4 from __future__ import annotations
@@ -5,17 +6,22 @@ from __future__ import annotations
6 import json
7 import re
8 from enum import Enum
8 -from typing import Any
9 +from typing import TYPE_CHECKING, Any
10
11 import jsonschema
12 import pytest
13 from pydantic import BaseModel
14
15 +from tests.reasoning.utils import run_reasoning_extraction
16 from vllm.entrypoints.llm import LLM
17 from vllm.outputs import RequestOutput
18 from vllm.platforms import current_platform
19 +from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager
20 from vllm.sampling_params import GuidedDecodingParams, SamplingParams
21
22 +if TYPE_CHECKING:
23 + from vllm.config import TokenizerMode
24 +
25 NGRAM_SPEC_CONFIG = {
26 "model": "[ngram]",
27 "num_speculative_tokens": 5,
@@ -444,7 +450,7 @@ def test_structured_output(
450
451 prompt = """
452 You have access to the following function to retrieve the weather in a city:
447 -
453 +
454 {
455 "name": "get_weather",
456 "parameters": {
@@ -455,7 +461,7 @@ You have access to the following function to retrieve the weather in a city:
461 }
462 }
463 }
458 -
464 +
465 If a you choose to call a function ONLY reply in the following format:
466 <{start_tag}={function_name}>{parameters}{end_tag}
467 where
@@ -476,7 +482,7 @@ Reminder:
482 - Always add your sources when using search results to answer the user query
483
484 You are a helpful assistant.
479 -
485 +
486 Given the previous instructions, what is the weather in New York City? \
487 Make the response as short as possible.
488 """
@@ -514,6 +520,88 @@ Make the response as short as possible.
520 f"{generated_text!r}\nError: {str(e)}")
521
522
523 +@pytest.mark.skip_global_cleanup
524 +@pytest.mark.parametrize(
525 + "model_name, guided_decoding_backend, tokenizer_mode, reasoning_parser, speculative_config", # noqa: E501
526 + [
527 + ("deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", "xgrammar", "auto",
528 + "deepseek_r1", NGRAM_SPEC_CONFIG),
529 + ("Qwen/Qwen3-1.7B", "xgrammar", "auto", "deepseek_r1", None),
530 + ],
531 +)
532 +def test_structured_output_with_reasoning_matrices(
533 + monkeypatch: pytest.MonkeyPatch,
534 + guided_decoding_backend: str,
535 + tokenizer_mode: TokenizerMode,
536 + reasoning_parser: str,
537 + model_name: str,
538 + speculative_config: dict[str, Any] | None,
539 +):
540 + monkeypatch.setenv("VLLM_USE_V1", "1")
541 +
542 + if current_platform.is_tpu() and speculative_config:
543 + pytest.skip("TPU does not support speculative decoding")
544 +
545 + # Use a single LLM instance for several scenarios to
546 + # speed up the test suite.
547 + llm = LLM(
548 + model=model_name,
549 + # Don't use eager execution on TPUs because we want to test for no
550 + # recompilation at runtime
551 + enforce_eager=bool(not current_platform.is_tpu()),
552 + max_model_len=1024,
553 + max_num_seqs=16,
554 + guided_decoding_backend=guided_decoding_backend,
555 + guided_decoding_disable_any_whitespace=True,
556 + tokenizer_mode=tokenizer_mode,
557 + reasoning_parser=reasoning_parser,
558 + speculative_config=speculative_config,
559 + )
560 + tokenizer = llm.get_tokenizer(None)
561 + reasoner = ReasoningParserManager.get_reasoning_parser(reasoning_parser)(
562 + tokenizer=tokenizer)
563 +
564 + reasoning_prompt = "Solve the following math problem step-by-step, then provide the final answer as JSON object with a single key 'result'. Make sure to correct your reasoning if there are any issue should it arise.\nProblem: What is 5 * 8 + 2?" # noqa: E501
565 + reasoning_schema = {
566 + "type": "object",
567 + "properties": {
568 + "result": {
569 + "type": "integer"
570 + }
571 + },
572 + "required": ["result"],
573 + "additionalProperties": False
574 + }
575 + if "Qwen3" in model_name:
576 + reasoning_prompt += "<think>\n"
577 +
578 + sampling_params = SamplingParams(
579 + temperature=0.1,
580 + max_tokens=8192,
581 + guided_decoding=GuidedDecodingParams(json=reasoning_schema),
582 + )
583 + outputs = llm.generate(
584 + [reasoning_prompt],
585 + sampling_params=sampling_params,
586 + use_tqdm=True,
587 + )
588 +
589 + assert outputs is not None
590 + output = outputs[0]
591 + assert output is not None and isinstance(output, RequestOutput)
592 + prompt = output.prompt
593 + generated_text = output.outputs[0].text
594 + reasoning_content, content = run_reasoning_extraction(
595 + reasoner, [generated_text])
596 + print(
597 + f"Prompt: {prompt!r}\nReasoning: {reasoning_content!r}\nContent: {content!r}"
598 + )
599 +
600 + assert content is not None and reasoning_content is not None
601 + output_json = json.loads(content)
602 + jsonschema.validate(instance=output_json, schema=reasoning_schema)
603 +
604 +
605 @pytest.mark.skip_global_cleanup
606 @pytest.mark.parametrize("model_name, tokenizer_mode",
607 PARAMS_MODELS_TOKENIZER_MODE)
vllm/config.py
+2 -2
@@ -2332,7 +2332,7 @@ class SpeculativeConfig:
2332 `TypicalAcceptanceSampler`."""
2333
2334 speculative_token_tree: Optional[str] = None
2335 - """Specifies the tree structure for speculative token generation.
2335 + """Specifies the tree structure for speculative token generation.
2336 """
2337 # required configuration params passed from engine
2338 target_model_config: ModelConfig = field(default=None,
@@ -4024,7 +4024,7 @@ class VllmConfig:
4024 """LoRA configuration."""
4025 speculative_config: Optional[SpeculativeConfig] = None
4026 """Speculative decoding configuration."""
4027 - decoding_config: Optional[DecodingConfig] = None
4027 + decoding_config: DecodingConfig = field(default_factory=DecodingConfig)
4028 """Decoding configuration."""
4029 observability_config: Optional[ObservabilityConfig] = None
4030 """Observability configuration."""
vllm/reasoning/abs_reasoning_parsers.py
+4 -2
@@ -1,5 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 +from __future__ import annotations
4 +
5 import os
6 from abc import abstractmethod
7 from collections.abc import Sequence
@@ -33,7 +35,7 @@ class ReasoningParser:
35 return self.model_tokenizer.get_vocab()
36
37 @abstractmethod
36 - def is_reasoning_end(self, input_ids: list[int]) -> bool:
38 + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
39 """
40 Check if the reasoning content ends in the input_ids.
41
@@ -106,7 +108,7 @@ class ReasoningParserManager:
108 reasoning_parsers: dict[str, type] = {}
109
110 @classmethod
109 - def get_reasoning_parser(cls, name) -> type:
111 + def get_reasoning_parser(cls, name: str | None) -> type[ReasoningParser]:
112 """
113 Get reasoning parser by name which is registered by `register_module`.
114
vllm/v1/core/sched/scheduler.py
+4 -4
@@ -758,7 +758,8 @@ class Scheduler(SchedulerInterface):
758 # the outer lists can be of length > 1.
759 new_logprobs = logprobs.slice(req_index, req_index + 1)
760
761 - if new_token_ids and request.use_structured_output:
761 + if new_token_ids and self.structured_output_manager.should_advance(
762 + request):
763 # NOTE: structured_output_request
764 # should not be None if use_structured_output, we have
765 # check above, so safe to ignore type warning
@@ -767,11 +768,10 @@ class Scheduler(SchedulerInterface):
768
769 # Add newly generated spec token ids to the request.
770 if spec_token_ids is not None:
770 - if request.use_structured_output:
771 + if self.structured_output_manager.should_advance(request):
772 metadata = request.structured_output_request
772 - assert metadata is not None and metadata.grammar is not None
773 # Needs to happen after new_token_ids are accepted.
774 - request.spec_token_ids = metadata.grammar.validate_tokens(
774 + request.spec_token_ids = metadata.grammar.validate_tokens( # type: ignore[union-attr]
775 spec_token_ids[req_index])
776 else:
777 request.spec_token_ids = spec_token_ids[req_index]
vllm/v1/structured_output/__init__.py
+85 -16
@@ -7,16 +7,23 @@ from typing import TYPE_CHECKING, Optional
7
8 from vllm.config import VllmConfig
9 from vllm.logger import init_logger
10 +from vllm.reasoning import ReasoningParserManager
11 +from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
12 +from vllm.utils import LazyLoader
13 from vllm.v1.structured_output.backend_guidance import GuidanceBackend
14 from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
15 StructuredOutputGrammar)
16 +from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
17
18 if TYPE_CHECKING:
19 import numpy as np
20 import numpy.typing as npt
21 import torch
22
23 + from vllm.reasoning import ReasoningParser
24 from vllm.v1.request import Request
25 +else:
26 + torch = LazyLoader("torch", globals(), "torch")
27
28 logger = init_logger(__name__)
29
@@ -26,9 +33,11 @@ class StructuredOutputManager:
33
34 def __init__(self, vllm_config: VllmConfig):
35 self.backend: Optional[StructuredOutputBackend] = None
36 + self.reasoner: Optional[ReasoningParser] = None
37 self.vllm_config = vllm_config
38
39 self._grammar_bitmask: Optional[torch.Tensor] = None
40 + self._full_mask = torch.tensor(-1, dtype=torch.int32)
41
42 # The default max_workers if not specified is the number of CPUs * 5,
43 # which is way too high since these tasks are CPU-bound, not I/O bound.
@@ -36,24 +45,43 @@ class StructuredOutputManager:
45 # compilation, so we set it to half the number of CPUs.
46 max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
47 self.executor = ThreadPoolExecutor(max_workers=max_workers)
48 + self.tokenizer = init_tokenizer_from_configs(
49 + model_config=self.vllm_config.model_config,
50 + scheduler_config=self.vllm_config.scheduler_config,
51 + lora_config=self.vllm_config.lora_config,
52 + ).get_lora_tokenizer(None)
53 + reasoning_backend = vllm_config.decoding_config.reasoning_backend
54 + if reasoning_backend:
55 + reasoner_cls = ReasoningParserManager.get_reasoning_parser(
56 + reasoning_backend)
57 + self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
58
59 def grammar_init(self, request: Request) -> None:
60 if request.structured_output_request is None:
61 return
62
63 + if TYPE_CHECKING:
64 + assert request.sampling_params.guided_decoding is not None
65 +
66 # Initialize the backend the first time it is needed.
67 #
68 # NOTE: We only support a single backend. We do NOT support different
69 # backends on a per-request basis in V1 (for now, anyway...).
70 if self.backend is None:
71 backend = request.sampling_params.guided_decoding.backend
72 + vocab_size = self.vllm_config.model_config.get_vocab_size()
73 if backend == "xgrammar":
51 - from vllm.v1.structured_output.backend_xgrammar import (
52 - XgrammarBackend)
53 -
54 - self.backend = XgrammarBackend(self.vllm_config)
74 + self.backend = XgrammarBackend(
75 + self.vllm_config,
76 + tokenizer=self.tokenizer,
77 + vocab_size=vocab_size,
78 + )
79 elif backend == "guidance":
56 - self.backend = GuidanceBackend(self.vllm_config)
80 + self.backend = GuidanceBackend(
81 + self.vllm_config,
82 + tokenizer=self.tokenizer,
83 + vocab_size=vocab_size,
84 + )
85 else:
86 raise ValueError(
87 f"Unsupported structured output backend: {backend}")
@@ -87,14 +115,14 @@ class StructuredOutputManager:
115 if not structured_output_request_ids:
116 return None
117
118 + max_num_spec_tokens = 0
119 + if self.vllm_config.speculative_config is not None:
120 + max_num_spec_tokens = \
121 + self.vllm_config.speculative_config.num_speculative_tokens
122 +
123 if self._grammar_bitmask is None:
124 assert self.backend is not None
125 max_batch_size = self.vllm_config.scheduler_config.max_num_seqs
93 - if self.vllm_config.speculative_config is not None:
94 - max_num_spec_tokens = self.vllm_config.\
95 - speculative_config.num_speculative_tokens
96 - else:
97 - max_num_spec_tokens = 0
126
127 # Allocate a bitmask for each token needing to be checked:
128 # one for each speculative position, and one more for the
@@ -103,6 +131,7 @@ class StructuredOutputManager:
131 self.backend.allocate_token_bitmask(
132 max_batch_size * (1 + max_num_spec_tokens))
133
134 + bitmask_tensor = self._grammar_bitmask
135 # Generate a batched bitmask for all structured output requests.
136 # When speculative decoding is enabled, we need to include multiple
137 # masks for each request, one for each possible bonus token position.
@@ -110,16 +139,30 @@ class StructuredOutputManager:
139 cumulative_index = 0
140 ordered_seq = sorted(structured_output_request_ids.items(),
141 key=lambda x: x[1])
142 +
143 + # Note that for thinking support, we will need to
144 + # reset the relevant part of the bitmask for consequent
145 + # request here.
146 + bitmask_tensor[:(len(ordered_seq) * (1 + max_num_spec_tokens))].fill_(
147 + self._full_mask)
148 +
149 # NOTE: This outer loop can likely be parallelized to improve
150 # performance of bitmask generation for large batches.
151 for req_id, _ in ordered_seq:
152 request = requests[req_id].structured_output_request
117 - assert request is not None and request.grammar is not None
153 + if TYPE_CHECKING:
154 + assert request is not None
155 + assert request.grammar is not None
156 +
157 + apply_bitmask = (
158 + request.reasoning_ended if self.reasoner is not None else True
159 + ) # noqa: E501
160 +
161 state_advancements = 0
162 req_tokens = scheduled_spec_decode_tokens.get(req_id, []) + [None]
163 for i, token in enumerate(req_tokens):
121 - if not request.grammar.is_terminated():
122 - request.grammar.fill_bitmask(self._grammar_bitmask,
164 + if apply_bitmask and not request.grammar.is_terminated():
165 + request.grammar.fill_bitmask(bitmask_tensor,
166 cumulative_index)
167 if token is not None:
168 # In order to generate the correct bitmask for each
@@ -132,15 +175,41 @@ class StructuredOutputManager:
175 if state_advancements > 0:
176 request.grammar.rollback(state_advancements)
177
135 - bitmask_tensor = self._grammar_bitmask
136 - if cumulative_index < self._grammar_bitmask.shape[0]:
137 - bitmask_tensor = self._grammar_bitmask[:cumulative_index]
178 + if cumulative_index < bitmask_tensor.shape[0]:
179 + bitmask_tensor = bitmask_tensor[:cumulative_index]
180
181 # After finishing with the xgrammar operations, we convert to
182 # np.ndarray, because that is much more efficient for serialization
183 # and deserialization when sending this to the GPU workers.
184 return bitmask_tensor.numpy()
185
186 + def should_advance(self, request: Request) -> bool:
187 + if not request.use_structured_output:
188 + return False
189 +
190 + # To determine whether we can advance the FSM.
191 + # Supports thinking usage where we skip the reasoning components.
192 + if TYPE_CHECKING:
193 + assert request.structured_output_request is not None
194 + assert request.structured_output_request.grammar is not None
195 + # by default, we should always advance
196 + # for cases that doesn't uses thinking mode.
197 + if self.reasoner is not None:
198 + structured_req = request.structured_output_request
199 +
200 + if structured_req.reasoning_ended:
201 + return True
202 +
203 + # Check if reasoning ends in *this* step
204 + if self.reasoner.is_reasoning_end(request.all_token_ids):
205 + # Reasoning just ended, so we shouldn't advanced til
206 + # next pass
207 + structured_req.reasoning_ended = True
208 +
209 + return False
210 + else:
211 + return True
212 +
213 def clear_backend(self) -> None:
214 if self.backend is not None:
215 self.backend.destroy()
vllm/v1/structured_output/backend_guidance.py
+7 -15
@@ -1,5 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 +from __future__ import annotations
4 +
5 import copy
6 import json
7 import os
@@ -8,10 +10,8 @@ from typing import TYPE_CHECKING, Any, Optional, Union
10
11 import torch
12
11 -from vllm.config import VllmConfig
13 from vllm.logger import init_logger
14 from vllm.sampling_params import SamplingParams
14 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
15 from vllm.utils import LazyLoader
16 from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
17 StructuredOutputGrammar,
@@ -54,25 +54,17 @@ def process_for_additional_properties(
54 return guide_json_obj
55
56
57 +@dataclass
58 class GuidanceBackend(StructuredOutputBackend):
59
59 - def __init__(self, vllm_config: VllmConfig):
60 - self.vllm_config = vllm_config
61 - tokenizer_group = init_tokenizer_from_configs(
62 - model_config=vllm_config.model_config,
63 - scheduler_config=vllm_config.scheduler_config,
64 - lora_config=vllm_config.lora_config) # type: ignore[arg-type]
65 - self.vllm_config = vllm_config
66 - self.vocab_size = vllm_config.model_config.get_vocab_size()
67 -
60 + def __post_init__(self):
61 self.disable_any_whitespace = \
69 - vllm_config.decoding_config.disable_any_whitespace
62 + self.vllm_config.decoding_config.disable_any_whitespace
63 self.disable_additional_properties = \
71 - vllm_config.decoding_config.disable_additional_properties
64 + self.vllm_config.decoding_config.disable_additional_properties
65
73 - tokenizer = tokenizer_group.get_lora_tokenizer(None)
66 self.ll_tokenizer = llguidance_hf.from_tokenizer(
75 - tokenizer, self.vocab_size)
67 + self.tokenizer, self.vocab_size)
68
69 def compile_grammar(self, request_type: StructuredOutputOptions,
70 grammar_spec: str) -> StructuredOutputGrammar:
vllm/v1/structured_output/backend_types.py
+15 -2
@@ -1,9 +1,17 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 +from __future__ import annotations
4 +
5 import enum
6 from abc import ABC, abstractmethod
7 +from dataclasses import dataclass
8 +from typing import TYPE_CHECKING
9 +
10 +if TYPE_CHECKING:
11 + import torch
12
6 -import torch
13 + from vllm.config import VllmConfig
14 + from vllm.transformers_utils.tokenizer import AnyTokenizer
15
16
17 class StructuredOutputOptions(enum.Enum):
@@ -85,9 +93,14 @@ class StructuredOutputGrammar(ABC):
93 """
94
95
96 +@dataclass
97 class StructuredOutputBackend(ABC):
98 """Engine-level backend for structured output requests."""
99
100 + vllm_config: VllmConfig
101 + tokenizer: AnyTokenizer
102 + vocab_size: int
103 +
104 @abstractmethod
105 def compile_grammar(self, request_type: StructuredOutputOptions,
106 grammar_spec: str) -> StructuredOutputGrammar:
@@ -104,7 +117,7 @@ class StructuredOutputBackend(ABC):
117 """
118
119 @abstractmethod
107 - def allocate_token_bitmask(self, max_num_seqs: int):
120 + def allocate_token_bitmask(self, max_num_seqs: int) -> torch.Tensor:
121 """
122 Allocates a token bitmask for the specified maximum number of sequences.
123
vllm/v1/structured_output/backend_xgrammar.py
+21 -28
@@ -1,5 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 +from __future__ import annotations
4 +
5 import json
6 from dataclasses import dataclass, field
7 from typing import TYPE_CHECKING, Any
@@ -7,10 +9,8 @@ from typing import TYPE_CHECKING, Any
9 import torch
10
11 import vllm.envs
10 -from vllm.config import VllmConfig
12 from vllm.logger import init_logger
13 from vllm.sampling_params import SamplingParams
13 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
14 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
15 from vllm.utils import LazyLoader
16 from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
@@ -28,61 +28,49 @@ else:
28 logger = init_logger(__name__)
29
30
31 +@dataclass
32 class XgrammarBackend(StructuredOutputBackend):
33
33 - def __init__(self, vllm_config: VllmConfig):
34 - self.vllm_config = vllm_config
35 - tokenizer_group = init_tokenizer_from_configs(
36 - model_config=vllm_config.model_config,
37 - scheduler_config=vllm_config.scheduler_config,
38 - lora_config=vllm_config.lora_config) # type: ignore[arg-type]
39 -
34 + def __post_init__(self):
35 self.disable_any_whitespace = \
41 - vllm_config.decoding_config.disable_any_whitespace
36 + self.vllm_config.decoding_config.disable_any_whitespace
37
43 - self.num_speculative_tokens = 0
44 - if self.vllm_config.speculative_config is not None:
45 - self.num_speculative_tokens = \
46 - self.vllm_config.speculative_config.num_speculative_tokens
47 -
48 - tokenizer = tokenizer_group.get_lora_tokenizer(None)
49 - self.vocab_size = vllm_config.model_config.get_vocab_size()
50 - if isinstance(tokenizer, MistralTokenizer):
38 + if isinstance(self.tokenizer, MistralTokenizer):
39 # NOTE: ideally, xgrammar should handle this accordingly.
40 # refer to https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98
41 try:
54 - if tokenizer.is_tekken:
55 - encoded_vocab = tokenizer._vocab
42 + if self.tokenizer.is_tekken:
43 + encoded_vocab = self.tokenizer._vocab
44 else:
45 encoded_vocab = [
46 token for token, _ in sorted(
59 - tokenizer.get_vocab().items(),
47 + self.tokenizer.get_vocab().items(),
48 key=lambda x: x[1],
49 )
50 ]
51 stop_token_ids = None
64 - if hasattr(
65 - tokenizer,
52 + if (hasattr(
53 + self.tokenizer,
54 "eos_token_id",
67 - ) and tokenizer.eos_token_id is not None:
68 - stop_token_ids = [tokenizer.eos_token_id]
55 + ) and self.tokenizer.eos_token_id is not None):
56 + stop_token_ids = [self.tokenizer.eos_token_id]
57 except AttributeError as e:
58 raise ValueError(
59 f"Cannot get the vocabulary of the tokenizer "
72 - f"{type(tokenizer)}. The tokenizer should have a "
60 + f"{type(self.tokenizer)}. The tokenizer should have a "
61 "get_vocab method.") from e
62 tokenizer_info = xgr.TokenizerInfo( # type: ignore
63 encoded_vocab=encoded_vocab,
64 # NOTE: https://github.com/mlc-ai/xgrammar/blob/5e141f6ff1ca02bc31f9e512e68b61f2a8ae88e5/tests/python/test_tokenizer_info.py#L43 # noqa: E501
65 vocab_type=xgr.VocabType.RAW
78 - if tokenizer.is_tekken else xgr.VocabType.BYTE_FALLBACK,
66 + if self.tokenizer.is_tekken else xgr.VocabType.BYTE_FALLBACK,
67 vocab_size=self.vocab_size,
68 stop_token_ids=stop_token_ids,
69 add_prefix_space=True,
70 )
71 else:
72 tokenizer_info = xgr.TokenizerInfo.from_huggingface(
85 - tokenizer,
73 + self.tokenizer,
74 vocab_size=self.vocab_size,
75 )
76 self.compiler = xgr.GrammarCompiler(
@@ -92,6 +80,11 @@ class XgrammarBackend(StructuredOutputBackend):
80 cache_limit_bytes=vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024,
81 )
82
83 + self.num_speculative_tokens = 0
84 + if self.vllm_config.speculative_config is not None:
85 + self.num_speculative_tokens = \
86 + self.vllm_config.speculative_config.num_speculative_tokens
87 +
88 def compile_grammar(self, request_type: StructuredOutputOptions,
89 grammar_spec: str) -> StructuredOutputGrammar:
90 if request_type == StructuredOutputOptions.JSON:
vllm/v1/structured_output/request.py
+1
@@ -20,6 +20,7 @@ class StructuredOutputRequest:
20 sampling_params: SamplingParams
21 _grammar: Optional[Union[Future[StructuredOutputGrammar],
22 StructuredOutputGrammar]] = None
23 + reasoning_ended: bool = False
24
25 def _check_grammar_completion(self) -> bool:
26 # NOTE: We have to lazy import to gate circular imports