A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[V1][Core] using cached vocab_size for Structured Outputs (#14630)
Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
Aaron Pham committed
Mar 13, 2025 at 14:39 UTC
8a4a2efc6fc32cdc30e4e35ba3f8c64dcd0aa1d0
1 file changed
+1
-1
vllm/v1/structured_output/__init__.py
+1
-1
@@ -27,7 +27,6 @@ logger = init_logger(__name__)
27
class StructuredOutputManager:
28
29
def __init__(self, vllm_config: VllmConfig):
30
- self.vocab_size = vllm_config.model_config.get_vocab_size()
30
self.vllm_config = vllm_config
31
self.init_complete = False
32
@@ -41,6 +40,7 @@ class StructuredOutputManager:
40
tokenizer_group.ping()
41
42
tokenizer = tokenizer_group.get_lora_tokenizer(None)
43
+ self.vocab_size = tokenizer.max_token_id
44
if isinstance(tokenizer, MistralTokenizer):
45
# NOTE: ideally, xgrammar should handle this accordingly.
46
# refer to https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98