@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e35879c276
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix xgrammar failing to read a vocab_size from LlavaConfig on PixtralHF. (#11043)

Jeff Cook committed Dec 9, 2024 at 23:54 UTC e35879c27601b09aab49e054786ce2a459f7a384
1 file changed +3 -3
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+3 -3
@@ -148,7 +148,7 @@ class GrammarConfig:
148 else:
149 json_str = guided_params.json
150 return cls(json_str=json_str,
151 - vocab_size=model_config.hf_config.vocab_size,
151 + vocab_size=model_config.hf_text_config.vocab_size,
152 encoded_vocab=encoded_vocab,
153 stop_token_ids=stop_token_ids,
154 backend_str=backend_str,
@@ -168,7 +168,7 @@ class GrammarConfig:
168 else:
169 grammar_str = guided_params.grammar
170 return cls(grammar_str=grammar_str,
171 - vocab_size=model_config.hf_config.vocab_size,
171 + vocab_size=model_config.hf_text_config.vocab_size,
172 encoded_vocab=encoded_vocab,
173 stop_token_ids=stop_token_ids,
174 backend_str=backend_str,
@@ -176,7 +176,7 @@ class GrammarConfig:
176 max_threads=max_threads)
177 elif guided_params.json_object:
178 return cls(json_object=True,
179 - vocab_size=model_config.hf_config.vocab_size,
179 + vocab_size=model_config.hf_text_config.vocab_size,
180 encoded_vocab=encoded_vocab,
181 stop_token_ids=stop_token_ids,
182 backend_str=backend_str,