@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6110c39dc8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[BugFix] Fix tokenizer out of vocab size (#3685)

Roy committed Mar 29, 2024 at 23:18 UTC 6110c39dc8f7ab177a48d35a4426a0f6e811d538
4 files changed +14 -19
tests/tokenization/test_detokenize.py
+7
@@ -83,6 +83,13 @@ def test_decode_streaming(tokenizer_id, truth, with_prompt,
83
84 assert decoded_text == generated
85
86 + decoded_text = _run_incremental_decode(
87 + tokenizer, [len(tokenizer)],
88 + skip_special_tokens=skip_special_tokens,
89 + starting_index=starting_index)
90 +
91 + assert decoded_text == ''
92 +
93
94 @pytest.fixture
95 def detokenizer(tokenizer_name: str) -> Detokenizer:
vllm/engine/llm_engine.py
-8
@@ -222,14 +222,6 @@ class LLMEngine:
222 self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
223 self.parallel_config.tokenizer_pool_config, **init_kwargs)
224
225 - if len(self.get_tokenizer()) != self.model_config.get_vocab_size():
226 - logger.warning(
227 - f"The tokenizer's vocabulary size {len(self.get_tokenizer())}"
228 - f" does not match the model's vocabulary size "
229 - f"{self.model_config.get_vocab_size()}. This might "
230 - f"cause an error in decoding. Please change config.json "
231 - "to match the tokenizer's vocabulary size.")
232 -
225 def _verify_args(self) -> None:
226 self.model_config.verify_with_parallel_config(self.parallel_config)
227 self.cache_config.verify_with_parallel_config(self.parallel_config)
vllm/entrypoints/openai/serving_engine.py
-8
@@ -68,14 +68,6 @@ class OpenAIServing:
68 tokenizer_mode=engine_model_config.tokenizer_mode,
69 trust_remote_code=engine_model_config.trust_remote_code)
70
71 - if len(self.tokenizer) != engine_model_config.get_vocab_size():
72 - logger.warning(
73 - f"The tokenizer's vocabulary size {len(self.tokenizer)}"
74 - f" does not match the model's vocabulary size "
75 - f"{engine_model_config.get_vocab_size()}. This might "
76 - f"cause an error in decoding. Please change config.json "
77 - "to match the tokenizer's vocabulary size.")
78 -
71 async def show_available_models(self) -> ModelList:
72 """Show available models. Right now we only have one model."""
73 model_cards = [
vllm/transformers_utils/tokenizer.py
+7 -3
@@ -232,9 +232,13 @@ def detokenize_incrementally(
232 all_input_ids[:-1],
233 skip_special_tokens=skip_special_tokens)
234
235 - # Put new_token_id in a list so skip_special_tokens is respected
236 - new_tokens = tokenizer.convert_ids_to_tokens(
237 - [new_token_id], skip_special_tokens=skip_special_tokens)
235 + # If the new token id is out of bounds, return an empty string.
236 + if new_token_id >= len(tokenizer):
237 + new_tokens = [""]
238 + else:
239 + # Put new_token_id in a list so skip_special_tokens is respected
240 + new_tokens = tokenizer.convert_ids_to_tokens(
241 + [new_token_id], skip_special_tokens=skip_special_tokens)
242 output_tokens = prev_tokens + new_tokens
243
244 # If this is the first iteration, return all tokens.