A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Core] Support Lark grammars for XGrammar (#10870)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Dec 6, 2024 at 10:34 UTC
8b5963185512eb7799f12240570e0ac7e7462a88
3 files changed
+178
-9
vllm/model_executor/guided_decoding/__init__.py
-8
@@ -73,14 +73,6 @@ def maybe_backend_fallback(
73
"Falling back to use outlines instead.")
74
guided_params.backend = "outlines"
75
76
- # xgrammar only supports EBNF grammars and uses the GBNF format
77
- # https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
78
- elif (guided_params.grammar is not None
79
- and "::=" not in guided_params.grammar):
80
- logger.warning("xgrammar only supports EBNF grammars. "
81
- "Falling back to use outlines instead.")
82
- guided_params.backend = "outlines"
83
-
76
# xgrammar doesn't support some JSON schema features
77
elif (guided_params.json is not None
78
and has_xgrammar_unsupported_json_features(guided_params.json)):
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+16
-1
@@ -14,6 +14,9 @@ try:
14
except ImportError:
15
pass
16
17
+from vllm.model_executor.guided_decoding.xgrammar_utils import (
18
+ convert_lark_to_gbnf, grammar_is_likely_lark)
19
+
20
if TYPE_CHECKING:
21
from transformers import PreTrainedTokenizer
22
@@ -152,7 +155,19 @@ class GrammarConfig:
155
tokenizer_hash=tokenizer_hash,
156
max_threads=max_threads)
157
elif guided_params.grammar:
155
- return cls(grammar_str=guided_params.grammar,
158
+ # XGrammar only supports GBNF grammars, so we must convert Lark
159
+ if grammar_is_likely_lark(guided_params.grammar):
160
+ try:
161
+ grammar_str = convert_lark_to_gbnf(guided_params.grammar)
162
+ except ValueError as e:
163
+ raise ValueError(
164
+ "Failed to convert the grammar from Lark to GBNF. "
165
+ "Please either use GBNF grammar directly or specify"
166
+ " --guided-decoding-backend=outlines.\n"
167
+ f"Conversion error: {str(e)}") from e
168
+ else:
169
+ grammar_str = guided_params.grammar
170
+ return cls(grammar_str=grammar_str,
171
vocab_size=model_config.hf_config.vocab_size,
172
encoded_vocab=encoded_vocab,
173
stop_token_ids=stop_token_ids,
vllm/model_executor/guided_decoding/xgrammar_utils.py
new
+162
@@ -0,0 +1,162 @@
1
+import re
2
+
3
+
4
+def grammar_is_likely_lark(grammar_str: str) -> bool:
5
+ """
6
+ Check if grammar appears to use Lark syntax.
7
+
8
+ Args:
9
+ grammar_str: Input grammar string
10
+
11
+ Returns:
12
+ bool: True if grammar appears to be in Lark format, False otherwise
13
+
14
+ Examples:
15
+ >>> grammar_is_likely_lark("rule: 'abc'")
16
+ True
17
+ >>> grammar_is_likely_lark("rule ::= 'abc'")
18
+ False
19
+ """
20
+ if not grammar_str or not isinstance(grammar_str, str):
21
+ return False
22
+
23
+ for line in grammar_str.split('\n'):
24
+ # Remove both comment styles
25
+ line = re.sub(r'(#|//).*$', '', line).strip()
26
+ if not line:
27
+ continue
28
+
29
+ # Look for Lark-style rule definitions
30
+ if ':' in line and '::=' not in line:
31
+ return True
32
+
33
+ # Look for Lark-specific features
34
+ if any(pattern in line for pattern in ['?start:', '|', '~']):
35
+ return True
36
+
37
+ return False
38
+
39
+
40
+def convert_lark_to_gbnf(grammar_str: str) -> str:
41
+ """
42
+ Convert a Lark grammar string to GBNF format.
43
+
44
+ GBNF reference:
45
+ https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
46
+ Lark grammar reference:
47
+ https://lark-parser.readthedocs.io/en/latest/grammar.html
48
+
49
+ Args:
50
+ grammar_str: Input grammar in Lark format
51
+
52
+ Returns:
53
+ str: Converted grammar in GBNF format
54
+
55
+ Examples:
56
+ >>> print(convert_lark_to_gbnf("rule: 'hello'"))
57
+ root ::= rule
58
+ rule ::= "hello"
59
+ """
60
+ if not isinstance(grammar_str, str):
61
+ raise ValueError(f"Grammar must be a string, got {type(grammar_str)}")
62
+ if not grammar_str.strip():
63
+ raise ValueError("Grammar string cannot be empty")
64
+
65
+ defined_rules = set()
66
+ referenced_rules = set()
67
+ output_lines = []
68
+
69
+ def clean_line(line: str) -> str:
70
+ """Remove comments and whitespace from line."""
71
+ return re.sub(r'(#|//).*$', '', line).strip()
72
+
73
+ def check_quotes(text: str, rule_name: str, line_num: int) -> None:
74
+ """Validate quote matching in text."""
75
+ if text.count("'") % 2 != 0 or text.count('"') % 2 != 0:
76
+ raise ValueError(
77
+ f"Mismatched quotes in {rule_name} on line {line_num}")
78
+
79
+ def extract_references(text: str) -> set:
80
+ """Extract rule references from text."""
81
+ # Remove quoted strings and special characters
82
+ text = re.sub(r'"[^"]*"', '', text)
83
+ text = re.sub(r'[+*?()|\[\]{}]', ' ', text)
84
+ return set(re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b', text))
85
+
86
+ # First pass: Find root rule and validate rule definitions
87
+ lines = [clean_line(line) for line in grammar_str.split('\n')]
88
+ first_rule = None
89
+
90
+ for line_num, line in enumerate(lines, 1):
91
+ if not line or line.startswith('|'):
92
+ continue
93
+
94
+ if ':' in line:
95
+ try:
96
+ name = line.split(':', 1)[0].strip().strip('?')
97
+ defined_rules.add(name)
98
+ if first_rule is None:
99
+ first_rule = name
100
+ if name == 'start':
101
+ first_rule = 'start'
102
+ except IndexError as e:
103
+ raise ValueError(f"Invalid rule format on line {line_num}. "
104
+ "Expected 'rule_name: definition'") from e
105
+
106
+ if not defined_rules:
107
+ raise ValueError("No valid rules found in grammar")
108
+
109
+ # Add root rule
110
+ output_lines.append(f"root ::= {first_rule}")
111
+
112
+ # Second pass: Process rule definitions and alternatives
113
+ current_rule = None
114
+ current_definition = []
115
+
116
+ for line_num, line in enumerate(lines, 1):
117
+ if not line:
118
+ continue
119
+
120
+ try:
121
+ if ':' in line and not line.startswith('|'):
122
+ # Save previous rule if exists
123
+ if current_rule:
124
+ output_lines.append(
125
+ f"{current_rule} ::= {' | '.join(current_definition)}")
126
+
127
+ # Process new rule
128
+ name, definition = line.split(':', 1)
129
+ current_rule = name.strip().strip('?')
130
+
131
+ check_quotes(definition, f"rule '{current_rule}'", line_num)
132
+ definition = re.sub(r"'([^']*)'", r'"\1"', definition)
133
+ referenced_rules.update(extract_references(definition))
134
+ current_definition = [definition.strip()]
135
+
136
+ elif line.startswith('|'):
137
+ if not current_rule:
138
+ raise ValueError(f"Alternative '|' on line {line_num} "
139
+ "without a preceding rule definition")
140
+
141
+ alt_def = line[1:].strip()
142
+ check_quotes(alt_def, f"alternative for rule '{current_rule}'",
143
+ line_num)
144
+ alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def)
145
+ referenced_rules.update(extract_references(alt_def))
146
+ current_definition.append(alt_def)
147
+
148
+ except ValueError as e:
149
+ raise ValueError(f"Error on line {line_num}: {str(e)}") from e
150
+
151
+ # Add final rule if exists
152
+ if current_rule:
153
+ output_lines.append(
154
+ f"{current_rule} ::= {' | '.join(current_definition)}")
155
+
156
+ # Validate all rules are defined
157
+ undefined_rules = referenced_rules - defined_rules - {'root'}
158
+ if undefined_rules:
159
+ raise ValueError("Referenced rules are not defined: "
160
+ f"{', '.join(sorted(undefined_rules))}")
161
+
162
+ return '\n'.join(output_lines)