@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 865732342b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc][Log] Add log for tokenizer length not equal to vocabulary size (#3500)

Roy committed Mar 21, 2024 at 18:07 UTC 865732342b4e3b8a4ef38f28a2a5bdb87cf3f970
2 files changed +16
vllm/engine/llm_engine.py
+8
@@ -169,6 +169,14 @@ class LLMEngine:
169 self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
170 self.parallel_config.tokenizer_pool_config, **init_kwargs)
171
172 + if len(self.get_tokenizer()) != self.model_config.get_vocab_size():
173 + logger.warning(
174 + f"The tokenizer's vocabulary size {len(self.get_tokenizer())}"
175 + f" does not match the model's vocabulary size "
176 + f"{self.model_config.get_vocab_size()}. This might "
177 + f"cause an error in decoding. Please change config.json "
178 + "to match the tokenizer's vocabulary size.")
179 +
180 def _verify_args(self) -> None:
181 self.model_config.verify_with_parallel_config(self.parallel_config)
182 self.cache_config.verify_with_parallel_config(self.parallel_config)
vllm/entrypoints/openai/serving_engine.py
+8
@@ -68,6 +68,14 @@ class OpenAIServing:
68 tokenizer_mode=engine_model_config.tokenizer_mode,
69 trust_remote_code=engine_model_config.trust_remote_code)
70
71 + if len(self.tokenizer) != engine_model_config.get_vocab_size():
72 + logger.warning(
73 + f"The tokenizer's vocabulary size {len(self.tokenizer)}"
74 + f" does not match the model's vocabulary size "
75 + f"{engine_model_config.get_vocab_size()}. This might "
76 + f"cause an error in decoding. Please change config.json "
77 + "to match the tokenizer's vocabulary size.")
78 +
79 async def show_available_models(self) -> ModelList:
80 """Show available models. Right now we only have one model."""
81 model_cards = [