A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Validate grammar and fail early (#11119)
Cody Yu committed
Dec 12, 2024 at 10:34 UTC
2c97eca1fff5297089794d2bd8ebd0bf98e12641
2 files changed
+26
-18
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+22
-10
@@ -131,22 +131,25 @@ class GrammarConfig:
131
max_threads: int = 8) -> GrammarConfig:
132
133
tokenizer_hash = hash(tokenizer)
134
- # Only get tokenizer data if not already cached
135
- if tokenizer_hash in TokenizerDataCache._cache:
136
- encoded_vocab = None
137
- stop_token_ids = None
138
- backend_str = None
139
- else:
140
- tokenizer_data = TokenizerDataCache.get_tokenizer_data(tokenizer)
141
- encoded_vocab = tokenizer_data.encoded_vocab
142
- stop_token_ids = tokenizer_data.stop_token_ids
143
- backend_str = tokenizer_data.backend_str
134
+ tokenizer_data = TokenizerDataCache.get_tokenizer_data(tokenizer)
135
+ encoded_vocab = tokenizer_data.encoded_vocab
136
+ stop_token_ids = tokenizer_data.stop_token_ids
137
+ backend_str = tokenizer_data.backend_str
138
139
if guided_params.json:
140
if not isinstance(guided_params.json, str):
141
json_str = json.dumps(guided_params.json)
142
else:
143
json_str = guided_params.json
144
+
145
+ # Validate the schema and raise ValueError here if it is invalid.
146
+ # This is to avoid exceptions in model execution, which will crash
147
+ # the engine worker process.
148
+ try:
149
+ xgr.Grammar.from_json_schema(json_str)
150
+ except RuntimeError as err:
151
+ raise ValueError(str(err)) from err
152
+
153
return cls(json_str=json_str,
154
vocab_size=model_config.hf_text_config.vocab_size,
155
encoded_vocab=encoded_vocab,
@@ -167,6 +170,15 @@ class GrammarConfig:
170
f"Conversion error: {str(e)}") from e
171
else:
172
grammar_str = guided_params.grammar
173
+
174
+ # Validate the grammar and raise ValueError here if it is invalid.
175
+ # This is to avoid exceptions in model execution, which will crash
176
+ # the engine worker process.
177
+ try:
178
+ xgr.Grammar.from_ebnf(grammar_str)
179
+ except RuntimeError as err:
180
+ raise ValueError(str(err)) from err
181
+
182
return cls(grammar_str=grammar_str,
183
vocab_size=model_config.hf_text_config.vocab_size,
184
encoded_vocab=encoded_vocab,
vllm/model_executor/guided_decoding/xgrammar_utils.py
+4
-8
@@ -26,15 +26,11 @@ def grammar_is_likely_lark(grammar_str: str) -> bool:
26
if not line:
27
continue
28
29
- # Look for Lark-style rule definitions
30
- if ':' in line and '::=' not in line:
31
- return True
29
+ # Look for GBNF rule definition
30
+ if '::=' in line:
31
+ return False
32
33
- # Look for Lark-specific features
34
- if any(pattern in line for pattern in ['?start:', '|', '~']):
35
- return True
36
-
37
- return False
33
+ return True
34
35
36
def convert_lark_to_gbnf(grammar_str: str) -> str: