A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix] Fix guidance backend for Qwen models (#16210)
Signed-off-by: Benjamin Chislett <benjamin.chislett@centml.ai>
Benjamin Chislett committed
Apr 7, 2025 at 18:15 UTC
3147586ebdb36ceae653e9dceec8cf9922fe2c28
1 file changed
+2
-1
vllm/v1/structured_output/backend_guidance.py
+2
-1
@@ -46,7 +46,8 @@ class GuidanceBackend(StructuredOutputBackend):
46
in vllm_config.decoding_config.guided_decoding_backend)
47
48
tokenizer = tokenizer_group.get_lora_tokenizer(None)
49
- self.ll_tokenizer = llguidance_hf.from_tokenizer(tokenizer, None)
49
+ self.ll_tokenizer = llguidance_hf.from_tokenizer(
50
+ tokenizer, self.vocab_size)
51
52
def compile_grammar(self, request_type: StructuredOutputOptions,
53
grammar_spec: str) -> StructuredOutputGrammar: