A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Misc][Log] Add log for tokenizer length not equal to vocabulary size (#3500)
Roy committed
Mar 21, 2024 at 18:07 UTC
865732342b4e3b8a4ef38f28a2a5bdb87cf3f970
2 files changed
+16
vllm/engine/llm_engine.py
+8
@@ -169,6 +169,14 @@ class LLMEngine:
169
self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
170
self.parallel_config.tokenizer_pool_config, **init_kwargs)
171
172
+ if len(self.get_tokenizer()) != self.model_config.get_vocab_size():
173
+ logger.warning(
174
+ f"The tokenizer's vocabulary size {len(self.get_tokenizer())}"
175
+ f" does not match the model's vocabulary size "
176
+ f"{self.model_config.get_vocab_size()}. This might "
177
+ f"cause an error in decoding. Please change config.json "
178
+ "to match the tokenizer's vocabulary size.")
179
+
180
def _verify_args(self) -> None:
181
self.model_config.verify_with_parallel_config(self.parallel_config)
182
self.cache_config.verify_with_parallel_config(self.parallel_config)
vllm/entrypoints/openai/serving_engine.py
+8
@@ -68,6 +68,14 @@ class OpenAIServing:
68
tokenizer_mode=engine_model_config.tokenizer_mode,
69
trust_remote_code=engine_model_config.trust_remote_code)
70
71
+ if len(self.tokenizer) != engine_model_config.get_vocab_size():
72
+ logger.warning(
73
+ f"The tokenizer's vocabulary size {len(self.tokenizer)}"
74
+ f" does not match the model's vocabulary size "
75
+ f"{engine_model_config.get_vocab_size()}. This might "
76
+ f"cause an error in decoding. Please change config.json "
77
+ "to match the tokenizer's vocabulary size.")
78
+
79
async def show_available_models(self) -> ModelList:
80
"""Show available models. Right now we only have one model."""
81
model_cards = [