A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Core][V0] Enable regex support with xgrammar (#13228)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Apr 13, 2025 at 22:13 UTC
dc1b4a6f1300003ae27f033afbdff5e2683721ce
3 files changed
+25
-9
tests/entrypoints/llm/test_guided_generate.py
+13
-2
@@ -286,15 +286,26 @@ def test_validation_against_both_guided_decoding_options(sample_regex, llm):
286
287
@pytest.mark.skip_global_cleanup
288
def test_disable_guided_decoding_fallback(sample_regex, llm):
289
+ # see has_xgrammar_unsupported_json_features()
290
+ unsupported_json = {
291
+ "type": "object",
292
+ "properties": {
293
+ "example": {
294
+ "type": "string",
295
+ "minLength": 5 # unsupported by xgrammar
296
+ }
297
+ }
298
+ }
299
sampling_params = SamplingParams(temperature=0.8,
300
top_p=0.95,
301
guided_decoding=GuidedDecodingParams(
292
- regex=sample_regex,
302
+ json=unsupported_json,
303
backend="xgrammar:no-fallback"))
304
305
with pytest.raises(
306
ValueError,
297
- match="xgrammar does not support regex guided decoding"):
307
+ match="xgrammar does not support advanced JSON schema features "
308
+ "like enums, patterns or numeric ranges."):
309
llm.generate(prompts="This should fail",
310
sampling_params=sampling_params,
311
use_tqdm=True)
vllm/model_executor/guided_decoding/__init__.py
+2
-7
@@ -59,14 +59,9 @@ def maybe_backend_fallback(
59
from vllm.model_executor.guided_decoding.xgrammar_decoding import (
60
xgr_installed)
61
62
- # xgrammar doesn't support regex, fallback to outlines
63
- if guided_params.regex is not None:
64
- fallback_or_error(
65
- guided_params,
66
- "xgrammar does not support regex guided decoding.", "outlines")
62
# xgrammar doesn't support some JSON schema features
68
- elif (guided_params.json is not None
69
- and has_xgrammar_unsupported_json_features(guided_params.json)):
63
+ if (guided_params.json is not None and
64
+ has_xgrammar_unsupported_json_features(guided_params.json)):
65
fallback_or_error(
66
guided_params,
67
"xgrammar does not support advanced JSON schema features like "
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+10
@@ -152,6 +152,7 @@ class GrammarConfig:
152
grammar_str: str | None = None
153
json_object: bool | None = None
154
any_whitespace: bool = True
155
+ regex_str: str | None = None
156
max_threads: int = 8
157
158
@classmethod
@@ -255,6 +256,13 @@ class GrammarConfig:
256
max_threads=max_threads,
257
tokenizer_data=tokenizer_data,
258
)
259
+ elif guided_params.regex:
260
+ return cls(
261
+ regex_str=guided_params.regex,
262
+ tokenizer_hash=tokenizer_hash,
263
+ max_threads=max_threads,
264
+ tokenizer_data=tokenizer_data,
265
+ )
266
else:
267
raise ValueError(
268
"Currently only support JSON and EBNF grammar mode for xgrammar"
@@ -330,6 +338,8 @@ class XGrammarLogitsProcessor:
338
self.ctx = compiler\
339
.compile_json_schema('{"type": "object"}',
340
any_whitespace=any_whitespace)
341
+ elif self.config.regex_str:
342
+ self.ctx = compiler.compile_regex(self.config.regex_str)
343
else:
344
raise ValueError(
345
"Invalid configuration for xgrammar logits processor")