A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix xgrammar failing to read a vocab_size from LlavaConfig on PixtralHF. (#11043)
Jeff Cook committed
Dec 9, 2024 at 23:54 UTC
e35879c27601b09aab49e054786ce2a459f7a384
1 file changed
+3
-3
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+3
-3
@@ -148,7 +148,7 @@ class GrammarConfig:
148
else:
149
json_str = guided_params.json
150
return cls(json_str=json_str,
151
- vocab_size=model_config.hf_config.vocab_size,
151
+ vocab_size=model_config.hf_text_config.vocab_size,
152
encoded_vocab=encoded_vocab,
153
stop_token_ids=stop_token_ids,
154
backend_str=backend_str,
@@ -168,7 +168,7 @@ class GrammarConfig:
168
else:
169
grammar_str = guided_params.grammar
170
return cls(grammar_str=grammar_str,
171
- vocab_size=model_config.hf_config.vocab_size,
171
+ vocab_size=model_config.hf_text_config.vocab_size,
172
encoded_vocab=encoded_vocab,
173
stop_token_ids=stop_token_ids,
174
backend_str=backend_str,
@@ -176,7 +176,7 @@ class GrammarConfig:
176
max_threads=max_threads)
177
elif guided_params.json_object:
178
return cls(json_object=True,
179
- vocab_size=model_config.hf_config.vocab_size,
179
+ vocab_size=model_config.hf_text_config.vocab_size,
180
encoded_vocab=encoded_vocab,
181
stop_token_ids=stop_token_ids,
182
backend_str=backend_str,