@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2c97eca1ff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Validate grammar and fail early (#11119)

Cody Yu committed Dec 12, 2024 at 10:34 UTC 2c97eca1fff5297089794d2bd8ebd0bf98e12641
2 files changed +26 -18
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+22 -10
@@ -131,22 +131,25 @@ class GrammarConfig:
131 max_threads: int = 8) -> GrammarConfig:
132
133 tokenizer_hash = hash(tokenizer)
134 - # Only get tokenizer data if not already cached
135 - if tokenizer_hash in TokenizerDataCache._cache:
136 - encoded_vocab = None
137 - stop_token_ids = None
138 - backend_str = None
139 - else:
140 - tokenizer_data = TokenizerDataCache.get_tokenizer_data(tokenizer)
141 - encoded_vocab = tokenizer_data.encoded_vocab
142 - stop_token_ids = tokenizer_data.stop_token_ids
143 - backend_str = tokenizer_data.backend_str
134 + tokenizer_data = TokenizerDataCache.get_tokenizer_data(tokenizer)
135 + encoded_vocab = tokenizer_data.encoded_vocab
136 + stop_token_ids = tokenizer_data.stop_token_ids
137 + backend_str = tokenizer_data.backend_str
138
139 if guided_params.json:
140 if not isinstance(guided_params.json, str):
141 json_str = json.dumps(guided_params.json)
142 else:
143 json_str = guided_params.json
144 +
145 + # Validate the schema and raise ValueError here if it is invalid.
146 + # This is to avoid exceptions in model execution, which will crash
147 + # the engine worker process.
148 + try:
149 + xgr.Grammar.from_json_schema(json_str)
150 + except RuntimeError as err:
151 + raise ValueError(str(err)) from err
152 +
153 return cls(json_str=json_str,
154 vocab_size=model_config.hf_text_config.vocab_size,
155 encoded_vocab=encoded_vocab,
@@ -167,6 +170,15 @@ class GrammarConfig:
170 f"Conversion error: {str(e)}") from e
171 else:
172 grammar_str = guided_params.grammar
173 +
174 + # Validate the grammar and raise ValueError here if it is invalid.
175 + # This is to avoid exceptions in model execution, which will crash
176 + # the engine worker process.
177 + try:
178 + xgr.Grammar.from_ebnf(grammar_str)
179 + except RuntimeError as err:
180 + raise ValueError(str(err)) from err
181 +
182 return cls(grammar_str=grammar_str,
183 vocab_size=model_config.hf_text_config.vocab_size,
184 encoded_vocab=encoded_vocab,
vllm/model_executor/guided_decoding/xgrammar_utils.py
+4 -8
@@ -26,15 +26,11 @@ def grammar_is_likely_lark(grammar_str: str) -> bool:
26 if not line:
27 continue
28
29 - # Look for Lark-style rule definitions
30 - if ':' in line and '::=' not in line:
31 - return True
29 + # Look for GBNF rule definition
30 + if '::=' in line:
31 + return False
32
33 - # Look for Lark-specific features
34 - if any(pattern in line for pattern in ['?start:', '|', '~']):
35 - return True
36 -
37 - return False
33 + return True
34
35
36 def convert_lark_to_gbnf(grammar_str: str) -> str: