A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[V1][Structured Output] Minor modification to `_validate_structured_output()` (#16748)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
Apr 18, 2025 at 13:12 UTC
30ed81b7cafa79decb97fc6793eed516c6924543
1 file changed
+10
-10
vllm/v1/engine/processor.py
+10
-10
@@ -149,6 +149,7 @@ class Processor:
149
"xgrammar", "xgrammar:disable-any-whitespace", "guidance",
150
"guidance:disable-any-whitespace", "auto"
151
]
152
+
153
engine_level_backend = self.decoding_config.guided_decoding_backend
154
if engine_level_backend not in supported_backends:
155
raise ValueError(f"Only {supported_backends} structured output is "
@@ -169,8 +170,15 @@ class Processor:
170
if engine_level_backend.startswith("xgrammar"):
171
# xgrammar with no fallback
172
validate_xgrammar_grammar(params)
172
- params.guided_decoding.backend = engine_level_backend
173
- elif engine_level_backend == "auto":
173
+ elif engine_level_backend.startswith("guidance"):
174
+ # TODO: ideally we would have the LLTokenizer here as Lark syntax
175
+ # allows <|special_token|> and similar, see
176
+ # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
177
+ # Without tokenizer these are disallowed in grammars.
178
+ validate_guidance_grammar(params, tokenizer=None)
179
+ else:
180
+ # NOTE: engine_level_backend must be "auto" here, because we have
181
+ # checked supported_backends above.
182
# "auto" is an opt-in to opinionated behavior where we try to
183
# choose a backend based on request contents. This is not the
184
# default as it is less predictable and subject to change
@@ -183,14 +191,6 @@ class Processor:
191
# are not supported in xgrammar. Fall back to guidance.
192
params.guided_decoding.backend = "guidance"
193
186
- if engine_level_backend.startswith("guidance"):
187
- # TODO ideally we would have the LLTokenizer here as Lark syntax
188
- # allows <|special_token|> and similar, see
189
- # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
190
- # Without tokenizer these are disallowed in grammars.
191
- validate_guidance_grammar(params, tokenizer=None)
192
- params.guided_decoding.backend = engine_level_backend
193
-
194
def process_inputs(
195
self,
196
request_id: str,