A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[V1] Add regex structured output support with xgrammar (#14590)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Mar 11, 2025 at 11:03 UTC
4bf82d4b90b7452fe4c3371526ae6aee714a0c60
4 files changed
+25
-19
requirements/common.txt
+2
-2
@@ -19,7 +19,7 @@ tiktoken >= 0.6.0 # Required for DBRX tokenizer
19
lm-format-enforcer >= 0.10.11, < 0.11
20
outlines == 0.1.11
21
lark == 1.2.2
22
-xgrammar == 0.1.11; platform_machine == "x86_64"
22
+xgrammar == 0.1.15; platform_machine == "x86_64" or platform_machine == "aarch64"
23
typing_extensions >= 4.10
24
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
25
partial-json-parser # used for parsing partial JSON outputs
@@ -37,4 +37,4 @@ depyf==0.18.0 # required for profiling and debugging with compilation config
37
cloudpickle # allows pickling lambda functions in model_executor/models/registry.py
38
watchfiles # required for http server to monitor the updates of TLS files
39
python-json-logger # Used by logging as per examples/other/logging_configuration.md
40
-scipy # Required for phi-4-multimodal-instruct
\ No newline at end of file
40
+scipy # Required for phi-4-multimodal-instruct
tests/v1/entrypoints/llm/test_struct_output_generate.py
+16
-16
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
+import re
5
6
import jsonschema
7
import pytest
@@ -219,25 +220,24 @@ def test_guided_regex(monkeypatch, sample_regex, guided_decoding_backend: str):
220
guided_decoding=GuidedDecodingParams(
221
regex=sample_regex,
222
backend=guided_decoding_backend))
222
- with pytest.raises(ValueError,
223
- match="Regex guided decoding is not supported."):
224
- llm.generate(prompts=[
223
+ outputs = llm.generate(
224
+ prompts=[
225
f"Give an example IPv4 address with this regex: {sample_regex}"
226
] * 2,
227
- sampling_params=sampling_params,
228
- use_tqdm=True)
227
+ sampling_params=sampling_params,
228
+ use_tqdm=True,
229
+ )
230
230
- # Once regex is supported --
231
- #assert outputs is not None
232
- #for output in outputs:
233
- # assert output is not None
234
- # assert isinstance(output, RequestOutput)
235
- # prompt = output.prompt
236
- # generated_text = output.outputs[0].text
237
- # print(generated_text)
238
- # assert generated_text is not None
239
- # assert re.fullmatch(sample_regex, generated_text) is not None
240
- # print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
231
+ assert outputs is not None
232
+ for output in outputs:
233
+ assert output is not None
234
+ assert isinstance(output, RequestOutput)
235
+ prompt = output.prompt
236
+ generated_text = output.outputs[0].text
237
+ print(generated_text)
238
+ assert generated_text is not None
239
+ assert re.fullmatch(sample_regex, generated_text) is not None
240
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
241
242
243
@pytest.mark.skip_global_cleanup
vllm/v1/structured_output/__init__.py
+2
@@ -112,6 +112,8 @@ class StructuredOutputManager:
112
ctx = self.compiler.compile_builtin_json_grammar()
113
elif request_type == StructuredOutputOptions.GRAMMAR:
114
ctx = self.compiler.compile_grammar(grammar_spec)
115
+ elif request_type == StructuredOutputOptions.REGEX:
116
+ ctx = self.compiler.compile_regex(grammar_spec)
117
else:
118
logger.error("Validation should have already occurred. "
119
"Please file an issue.")
vllm/v1/structured_output/utils.py
+5
-1
@@ -251,7 +251,11 @@ def validate_structured_output_request(
251
gd_params = sampling_params.guided_decoding
252
253
if gd_params.regex:
254
- raise ValueError("Regex structured output is not supported.")
254
+ try:
255
+ xgr.Grammar.from_regex(gd_params.regex)
256
+ except Exception as err:
257
+ raise ValueError("Failed to transform regex into a grammar: "
258
+ f"{err}") from err
259
260
if gd_params.choice:
261
choice_grammar = choice_as_grammar(gd_params.choice)