[Structured Outputs] Parse Lark grammars natively in the xgrammar backend (#58321)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Sep 25, 2026 at 12:35 UTC
1f0bc49ee61be3406d7cb32355980c6f90865c5e
5 files changed
+171
-145
tests/entrypoints/llm/test_struct_output_generate.py
+1
-1
@@ -455,7 +455,7 @@ def test_structured_output(
455
structured_outputs=StructuredOutputsParams(grammar="not a grammar"),
456
)
457
with pytest.raises(
458
- VLLMValidationError, match="Failed to convert the grammar "
458
+ VLLMValidationError, match="Invalid grammar specification"
459
):
460
runner.llm.generate(
461
(
tests/v1/structured_output/test_backend_xgrammar_lark.py
new
+143
@@ -0,0 +1,143 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Lark grammars in the xgrammar backend.
4
+
5
+`structured_outputs.grammar` accepts either EBNF or Lark. The xgrammar backend
6
+parses Lark natively, both when validating a request and when compiling it, so
7
+the request's grammar is never rewritten and frontends that skip Python
8
+request validation get the same behavior.
9
+"""
10
+
11
+import pytest
12
+from mistral_common.protocol.instruct.tool_calls import Tool, ToolChoiceEnum
13
+from transformers import AutoTokenizer
14
+
15
+from vllm.config import StructuredOutputsConfig, VllmConfig
16
+from vllm.exceptions import VLLMValidationError
17
+from vllm.sampling_params import SamplingParams, StructuredOutputsParams
18
+from vllm.tokenizers.mistral import MistralTokenizer
19
+from vllm.v1.structured_output.backend_types import StructuredOutputOptions
20
+from vllm.v1.structured_output.backend_xgrammar import (
21
+ XgrammarBackend,
22
+ validate_xgrammar_grammar,
23
+)
24
+
25
+pytestmark = pytest.mark.cpu_test
26
+
27
+TOKENIZER = "openai-community/gpt2"
28
+VOCAB_SIZE = 50257
29
+MISTRAL_TOKENIZER = "mistralai/Ministral-3-3B-Instruct-2512"
30
+
31
+# Uses Lark features beyond plain rules: a regex terminal, `%import common`,
32
+# and a rule alias. The Codex `apply_patch` tool grammar has the same shape.
33
+LARK_GRAMMAR = """
34
+start: "id=" NUMBER tag? LF?
35
+tag: " #" /[a-z]+/ -> label
36
+NUMBER: /[0-9]+/
37
+%import common.LF
38
+"""
39
+
40
+EBNF_GRAMMAR = 'root ::= "id=" [0-9]+'
41
+
42
+
43
+def _validate(
44
+ grammar: str, backend: str = "xgrammar", tokenizer: object | None = None
45
+) -> StructuredOutputsParams:
46
+ params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar))
47
+ params._validate_structured_outputs(
48
+ _StubModelConfig(),
49
+ StructuredOutputsConfig(backend=backend),
50
+ tokenizer=tokenizer or object(),
51
+ )
52
+ assert params.structured_outputs is not None
53
+ return params.structured_outputs
54
+
55
+
56
+class _StubModelConfig:
57
+ is_diffusion = False
58
+
59
+
60
+GRAMMARS = [
61
+ pytest.param(LARK_GRAMMAR, id="lark"),
62
+ pytest.param(EBNF_GRAMMAR, id="ebnf"),
63
+]
64
+
65
+
66
+@pytest.mark.parametrize("grammar", GRAMMARS)
67
+def test_validation_accepts_grammar_without_rewriting_it(grammar: str):
68
+ assert _validate(grammar).grammar == grammar
69
+
70
+
71
+def test_auto_backend_selects_xgrammar_for_lark():
72
+ assert _validate(LARK_GRAMMAR, backend="auto")._backend == "xgrammar"
73
+
74
+
75
+@pytest.mark.parametrize("mode", [ToolChoiceEnum.required, ToolChoiceEnum.none])
76
+def test_auto_backend_selects_guidance_for_mistral_lark(mode: ToolChoiceEnum):
77
+ """The Mistral tool parser generates Lark grammars for llguidance, which
78
+ keeps special tokens such as `[TOOL_CALLS]` out of regex matches."""
79
+ tokenizer = MistralTokenizer.from_pretrained(MISTRAL_TOKENIZER)
80
+ factory = tokenizer.grammar_factory
81
+ tool = {
82
+ "type": "function",
83
+ "function": {
84
+ "name": "get_weather",
85
+ "parameters": {"type": "object", "properties": {}},
86
+ },
87
+ }
88
+ grammar = factory.get_lark_from_jinja(
89
+ template=factory.select_jinja_template(),
90
+ mode=mode,
91
+ tools=[Tool.from_openai(tool)],
92
+ json_schema=None,
93
+ parallel_tool_calls=True,
94
+ json_only=False,
95
+ )
96
+ assert _validate(grammar, "auto", tokenizer)._backend == "guidance"
97
+
98
+
99
+@pytest.mark.parametrize(
100
+ "grammar",
101
+ [
102
+ pytest.param('rule: "hello"', id="missing-start-rule"),
103
+ pytest.param("start: 'hello'", id="single-quoted-literal"),
104
+ ],
105
+)
106
+def test_validation_rejects_invalid_lark(grammar: str):
107
+ params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar))
108
+ with pytest.raises(VLLMValidationError, match="Invalid grammar specification"):
109
+ validate_xgrammar_grammar(params)
110
+
111
+
112
+@pytest.fixture(scope="module")
113
+def backend() -> XgrammarBackend:
114
+ vllm_config = VllmConfig(
115
+ structured_outputs_config=StructuredOutputsConfig(backend="xgrammar")
116
+ )
117
+ return XgrammarBackend(
118
+ vllm_config,
119
+ tokenizer=AutoTokenizer.from_pretrained(TOKENIZER),
120
+ vocab_size=VOCAB_SIZE,
121
+ )
122
+
123
+
124
+@pytest.mark.parametrize("grammar", GRAMMARS)
125
+@pytest.mark.parametrize(
126
+ "text, accepted",
127
+ [("id=42", True), ("id=x", False)],
128
+)
129
+def test_compile_grammar_matches_lark_and_ebnf(
130
+ backend: XgrammarBackend, grammar: str, text: str, accepted: bool
131
+):
132
+ compiled = backend.compile_grammar(StructuredOutputOptions.GRAMMAR, grammar)
133
+ token_ids = backend.tokenizer.encode(text, add_special_tokens=False)
134
+ assert compiled.accept_tokens("req", token_ids) is accepted
135
+
136
+
137
+def test_compile_grammar_uses_lark_semantics(backend: XgrammarBackend):
138
+ compiled = backend.compile_grammar(StructuredOutputOptions.GRAMMAR, LARK_GRAMMAR)
139
+ token_ids = backend.tokenizer.encode("id=7 #ok\n", add_special_tokens=False)
140
+ assert compiled.accept_tokens("req", token_ids)
141
+ assert compiled.is_terminated() or compiled.accept_tokens(
142
+ "req", [backend.tokenizer.eos_token_id]
143
+ )
vllm/sampling_params.py
+17
@@ -1178,6 +1178,7 @@ class SamplingParams(
1178
validate_structured_output_request_outlines,
1179
)
1180
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
1181
+ from vllm.v1.structured_output.utils import grammar_is_likely_lark
1182
1183
if backend.startswith("xgrammar"):
1184
# xgrammar with no fallback
@@ -1210,6 +1211,22 @@ class SamplingParams(
1211
"backends or tokenizer_mode='hf' instead."
1212
)
1213
validate_structured_output_request_lm_format_enforcer(self)
1214
+ elif (
1215
+ backend == "auto"
1216
+ and is_mistral_tokenizer(tokenizer)
1217
+ and tokenizer.is_tekken
1218
+ and self.structured_outputs.grammar
1219
+ and grammar_is_likely_lark(self.structured_outputs.grammar)
1220
+ ):
1221
+ # Lark grammars for Tekken Mistral tokenizers, including the ones
1222
+ # the Mistral tool parser generates, go to guidance. llguidance
1223
+ # takes the tokenizer from mistral-common and never lets a regex
1224
+ # match a special token. The xgrammar backend declares no special
1225
+ # tokens for Mistral tokenizers, so a regex like `.` can match
1226
+ # `[TOOL_CALLS]` or `[INST]` as plain text.
1227
+ validate_guidance_grammar(self, tokenizer=_get_llg_tokenizer(tokenizer))
1228
+ self.structured_outputs._backend = "guidance"
1229
+ self.structured_outputs._backend_was_auto = True
1230
else:
1231
# NOTE: backend must be "auto" here, because we have
1232
# checked supported_backends above.
vllm/v1/structured_output/backend_xgrammar.py
+10
-14
@@ -21,7 +21,6 @@ from vllm.v1.structured_output.backend_types import (
21
from vllm.v1.structured_output.utils import (
22
choice_as_grammar,
23
compile_regex_with_timeout,
24
- convert_lark_to_ebnf,
24
grammar_is_likely_lark,
25
)
26
@@ -91,7 +90,10 @@ class XgrammarBackend(StructuredOutputBackend):
90
'{"type": "object"}', any_whitespace=not self.disable_any_whitespace
91
)
92
elif request_type == StructuredOutputOptions.GRAMMAR:
94
- ctx = self.compiler.compile_grammar(grammar_spec)
93
+ if grammar_is_likely_lark(grammar_spec):
94
+ ctx = self.compiler.compile_lark(grammar_spec)
95
+ else:
96
+ ctx = self.compiler.compile_grammar(grammar_spec)
97
elif request_type == StructuredOutputOptions.REGEX:
98
ctx = compile_regex_with_timeout(
99
self.compiler.compile_regex,
@@ -404,19 +406,13 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
406
return
407
408
if so_params.grammar:
407
- if grammar_is_likely_lark(so_params.grammar):
408
- # xgrammar supports EBNF grammars only
409
- try:
410
- so_params.grammar = convert_lark_to_ebnf(so_params.grammar)
411
- except ValueError as e:
412
- raise VLLMValidationError(
413
- "Failed to convert the grammar from Lark to EBNF. "
414
- ) from e
415
-
416
- # Test parsing EBNF grammar, possibly already converted from Lark
409
+ # Parse the grammar with the same syntax `compile_grammar` will use,
410
+ # but don't compile it. The grammar is passed on unchanged.
411
try:
418
- # parse the grammar, but we aren't compiling it.
419
- xgr.Grammar.from_ebnf(so_params.grammar)
412
+ if grammar_is_likely_lark(so_params.grammar):
413
+ xgr.Grammar.from_lark(so_params.grammar)
414
+ else:
415
+ xgr.Grammar.from_ebnf(so_params.grammar)
416
except Exception as e:
417
raise VLLMValidationError("Invalid grammar specification.") from e
418
return
vllm/v1/structured_output/utils.py
-130
@@ -435,136 +435,6 @@ def grammar_is_likely_lark(grammar_str: str) -> bool:
435
return True
436
437
438
-def convert_lark_to_ebnf(grammar_str: str) -> str:
439
- """Convert a Lark grammar string to EBNF format.
440
-
441
- EBNF reference:
442
- https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
443
- Lark grammar reference:
444
- https://lark-parser.readthedocs.io/en/latest/grammar.html
445
-
446
- Args:
447
- grammar_str: Input grammar in Lark format
448
-
449
- Returns:
450
- str: Converted grammar in EBNF format
451
-
452
- Examples:
453
- >>> print(convert_lark_to_ebnf("rule: 'hello'"))
454
- root ::= rule
455
- rule ::= "hello"
456
-
457
- """
458
- if not isinstance(grammar_str, str):
459
- raise ValueError(f"Grammar must be a string, got {type(grammar_str)}")
460
- if not grammar_str.strip():
461
- raise ValueError("Grammar string cannot be empty")
462
-
463
- defined_rules = set()
464
- referenced_rules = set()
465
- output_lines = []
466
-
467
- def clean_line(line: str) -> str:
468
- """Remove comments and whitespace from line."""
469
- return re.sub(r"(#|//).*$", "", line).strip()
470
-
471
- def check_quotes(text: str, rule_name: str, line_num: int) -> None:
472
- """Validate quote matching in text."""
473
- if text.count("'") % 2 != 0 or text.count('"') % 2 != 0:
474
- raise ValueError(f"Mismatched quotes in {rule_name} on line {line_num}")
475
-
476
- def extract_references(text: str) -> set[str]:
477
- """Extract rule references from text."""
478
- # Remove quoted strings and special characters
479
- text = re.sub(r'"[^"]*"', "", text)
480
- text = re.sub(r"[+*?()|\[\]{}]", " ", text)
481
- return set(re.findall(r"\b[a-zA-Z_][a-zA-Z0-9_]*\b", text))
482
-
483
- # First pass: Find root rule and validate rule definitions
484
- lines = [clean_line(line) for line in grammar_str.split("\n")]
485
- first_rule = None
486
-
487
- for line_num, line in enumerate(lines, 1):
488
- if not line or line.startswith("|"):
489
- continue
490
-
491
- if ":" in line:
492
- try:
493
- name = line.split(":", 1)[0].strip().strip("?")
494
- defined_rules.add(name)
495
- if first_rule is None:
496
- first_rule = name
497
- if name == "start":
498
- first_rule = "start"
499
- except IndexError as e:
500
- raise ValueError(
501
- f"Invalid rule format on line {line_num}. "
502
- "Expected 'rule_name: definition'"
503
- ) from e
504
-
505
- if not defined_rules:
506
- raise ValueError("No valid rules found in grammar")
507
-
508
- # Add root rule
509
- output_lines.append(f"root ::= {first_rule}")
510
-
511
- # Second pass: Process rule definitions and alternatives
512
- current_rule = None
513
- current_definition = []
514
-
515
- for line_num, line in enumerate(lines, 1):
516
- if not line:
517
- continue
518
-
519
- try:
520
- if ":" in line and not line.startswith("|"):
521
- # Save previous rule if exists
522
- if current_rule:
523
- output_lines.append(
524
- f"{current_rule} ::= {' | '.join(current_definition)}"
525
- )
526
-
527
- # Process new rule
528
- name, definition = line.split(":", 1)
529
- current_rule = name.strip().strip("?")
530
-
531
- check_quotes(definition, f"rule '{current_rule}'", line_num)
532
- definition = re.sub(r"'([^']*)'", r'"\1"', definition)
533
- referenced_rules.update(extract_references(definition))
534
- current_definition = [definition.strip()]
535
-
536
- elif line.startswith("|"):
537
- if not current_rule:
538
- raise ValueError(
539
- f"Alternative '|' on line {line_num} "
540
- "without a preceding rule definition"
541
- )
542
-
543
- alt_def = line[1:].strip()
544
- check_quotes(
545
- alt_def, f"alternative for rule '{current_rule}'", line_num
546
- )
547
- alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def)
548
- referenced_rules.update(extract_references(alt_def))
549
- current_definition.append(alt_def)
550
-
551
- except ValueError as e:
552
- raise ValueError(f"Error on line {line_num}: {str(e)}") from e
553
-
554
- # Add final rule if exists
555
- if current_rule:
556
- output_lines.append(f"{current_rule} ::= {' | '.join(current_definition)}")
557
-
558
- # Validate all rules are defined
559
- undefined_rules = referenced_rules - defined_rules - {"root"}
560
- if undefined_rules:
561
- raise ValueError(
562
- f"Referenced rules are not defined: {', '.join(sorted(undefined_rules))}"
563
- )
564
-
565
- return "\n".join(output_lines)
566
-
567
-
438
def choice_as_grammar(choice: list[str]) -> str:
439
def escape_ebnf_string(s: str) -> str:
440
"""Escape EBNF literals, including raw LF, CR, and NUL terminators."""