@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8b59631855
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Core] Support Lark grammars for XGrammar (#10870)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Dec 6, 2024 at 10:34 UTC 8b5963185512eb7799f12240570e0ac7e7462a88
3 files changed +178 -9
vllm/model_executor/guided_decoding/__init__.py
-8
@@ -73,14 +73,6 @@ def maybe_backend_fallback(
73 "Falling back to use outlines instead.")
74 guided_params.backend = "outlines"
75
76 - # xgrammar only supports EBNF grammars and uses the GBNF format
77 - # https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
78 - elif (guided_params.grammar is not None
79 - and "::=" not in guided_params.grammar):
80 - logger.warning("xgrammar only supports EBNF grammars. "
81 - "Falling back to use outlines instead.")
82 - guided_params.backend = "outlines"
83 -
76 # xgrammar doesn't support some JSON schema features
77 elif (guided_params.json is not None
78 and has_xgrammar_unsupported_json_features(guided_params.json)):
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+16 -1
@@ -14,6 +14,9 @@ try:
14 except ImportError:
15 pass
16
17 +from vllm.model_executor.guided_decoding.xgrammar_utils import (
18 + convert_lark_to_gbnf, grammar_is_likely_lark)
19 +
20 if TYPE_CHECKING:
21 from transformers import PreTrainedTokenizer
22
@@ -152,7 +155,19 @@ class GrammarConfig:
155 tokenizer_hash=tokenizer_hash,
156 max_threads=max_threads)
157 elif guided_params.grammar:
155 - return cls(grammar_str=guided_params.grammar,
158 + # XGrammar only supports GBNF grammars, so we must convert Lark
159 + if grammar_is_likely_lark(guided_params.grammar):
160 + try:
161 + grammar_str = convert_lark_to_gbnf(guided_params.grammar)
162 + except ValueError as e:
163 + raise ValueError(
164 + "Failed to convert the grammar from Lark to GBNF. "
165 + "Please either use GBNF grammar directly or specify"
166 + " --guided-decoding-backend=outlines.\n"
167 + f"Conversion error: {str(e)}") from e
168 + else:
169 + grammar_str = guided_params.grammar
170 + return cls(grammar_str=grammar_str,
171 vocab_size=model_config.hf_config.vocab_size,
172 encoded_vocab=encoded_vocab,
173 stop_token_ids=stop_token_ids,
vllm/model_executor/guided_decoding/xgrammar_utils.py new
+162
@@ -0,0 +1,162 @@
1 +import re
2 +
3 +
4 +def grammar_is_likely_lark(grammar_str: str) -> bool:
5 + """
6 + Check if grammar appears to use Lark syntax.
7 +
8 + Args:
9 + grammar_str: Input grammar string
10 +
11 + Returns:
12 + bool: True if grammar appears to be in Lark format, False otherwise
13 +
14 + Examples:
15 + >>> grammar_is_likely_lark("rule: 'abc'")
16 + True
17 + >>> grammar_is_likely_lark("rule ::= 'abc'")
18 + False
19 + """
20 + if not grammar_str or not isinstance(grammar_str, str):
21 + return False
22 +
23 + for line in grammar_str.split('\n'):
24 + # Remove both comment styles
25 + line = re.sub(r'(#|//).*$', '', line).strip()
26 + if not line:
27 + continue
28 +
29 + # Look for Lark-style rule definitions
30 + if ':' in line and '::=' not in line:
31 + return True
32 +
33 + # Look for Lark-specific features
34 + if any(pattern in line for pattern in ['?start:', '|', '~']):
35 + return True
36 +
37 + return False
38 +
39 +
40 +def convert_lark_to_gbnf(grammar_str: str) -> str:
41 + """
42 + Convert a Lark grammar string to GBNF format.
43 +
44 + GBNF reference:
45 + https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
46 + Lark grammar reference:
47 + https://lark-parser.readthedocs.io/en/latest/grammar.html
48 +
49 + Args:
50 + grammar_str: Input grammar in Lark format
51 +
52 + Returns:
53 + str: Converted grammar in GBNF format
54 +
55 + Examples:
56 + >>> print(convert_lark_to_gbnf("rule: 'hello'"))
57 + root ::= rule
58 + rule ::= "hello"
59 + """
60 + if not isinstance(grammar_str, str):
61 + raise ValueError(f"Grammar must be a string, got {type(grammar_str)}")
62 + if not grammar_str.strip():
63 + raise ValueError("Grammar string cannot be empty")
64 +
65 + defined_rules = set()
66 + referenced_rules = set()
67 + output_lines = []
68 +
69 + def clean_line(line: str) -> str:
70 + """Remove comments and whitespace from line."""
71 + return re.sub(r'(#|//).*$', '', line).strip()
72 +
73 + def check_quotes(text: str, rule_name: str, line_num: int) -> None:
74 + """Validate quote matching in text."""
75 + if text.count("'") % 2 != 0 or text.count('"') % 2 != 0:
76 + raise ValueError(
77 + f"Mismatched quotes in {rule_name} on line {line_num}")
78 +
79 + def extract_references(text: str) -> set:
80 + """Extract rule references from text."""
81 + # Remove quoted strings and special characters
82 + text = re.sub(r'"[^"]*"', '', text)
83 + text = re.sub(r'[+*?()|\[\]{}]', ' ', text)
84 + return set(re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b', text))
85 +
86 + # First pass: Find root rule and validate rule definitions
87 + lines = [clean_line(line) for line in grammar_str.split('\n')]
88 + first_rule = None
89 +
90 + for line_num, line in enumerate(lines, 1):
91 + if not line or line.startswith('|'):
92 + continue
93 +
94 + if ':' in line:
95 + try:
96 + name = line.split(':', 1)[0].strip().strip('?')
97 + defined_rules.add(name)
98 + if first_rule is None:
99 + first_rule = name
100 + if name == 'start':
101 + first_rule = 'start'
102 + except IndexError as e:
103 + raise ValueError(f"Invalid rule format on line {line_num}. "
104 + "Expected 'rule_name: definition'") from e
105 +
106 + if not defined_rules:
107 + raise ValueError("No valid rules found in grammar")
108 +
109 + # Add root rule
110 + output_lines.append(f"root ::= {first_rule}")
111 +
112 + # Second pass: Process rule definitions and alternatives
113 + current_rule = None
114 + current_definition = []
115 +
116 + for line_num, line in enumerate(lines, 1):
117 + if not line:
118 + continue
119 +
120 + try:
121 + if ':' in line and not line.startswith('|'):
122 + # Save previous rule if exists
123 + if current_rule:
124 + output_lines.append(
125 + f"{current_rule} ::= {' | '.join(current_definition)}")
126 +
127 + # Process new rule
128 + name, definition = line.split(':', 1)
129 + current_rule = name.strip().strip('?')
130 +
131 + check_quotes(definition, f"rule '{current_rule}'", line_num)
132 + definition = re.sub(r"'([^']*)'", r'"\1"', definition)
133 + referenced_rules.update(extract_references(definition))
134 + current_definition = [definition.strip()]
135 +
136 + elif line.startswith('|'):
137 + if not current_rule:
138 + raise ValueError(f"Alternative '|' on line {line_num} "
139 + "without a preceding rule definition")
140 +
141 + alt_def = line[1:].strip()
142 + check_quotes(alt_def, f"alternative for rule '{current_rule}'",
143 + line_num)
144 + alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def)
145 + referenced_rules.update(extract_references(alt_def))
146 + current_definition.append(alt_def)
147 +
148 + except ValueError as e:
149 + raise ValueError(f"Error on line {line_num}: {str(e)}") from e
150 +
151 + # Add final rule if exists
152 + if current_rule:
153 + output_lines.append(
154 + f"{current_rule} ::= {' | '.join(current_definition)}")
155 +
156 + # Validate all rules are defined
157 + undefined_rules = referenced_rules - defined_rules - {'root'}
158 + if undefined_rules:
159 + raise ValueError("Referenced rules are not defined: "
160 + f"{', '.join(sorted(undefined_rules))}")
161 +
162 + return '\n'.join(output_lines)