A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[BugFix] Fix tokenizer out of vocab size (#3685)
Roy committed
Mar 29, 2024 at 23:18 UTC
6110c39dc8f7ab177a48d35a4426a0f6e811d538
4 files changed
+14
-19
tests/tokenization/test_detokenize.py
+7
@@ -83,6 +83,13 @@ def test_decode_streaming(tokenizer_id, truth, with_prompt,
83
84
assert decoded_text == generated
85
86
+ decoded_text = _run_incremental_decode(
87
+ tokenizer, [len(tokenizer)],
88
+ skip_special_tokens=skip_special_tokens,
89
+ starting_index=starting_index)
90
+
91
+ assert decoded_text == ''
92
+
93
94
@pytest.fixture
95
def detokenizer(tokenizer_name: str) -> Detokenizer:
vllm/engine/llm_engine.py
-8
@@ -222,14 +222,6 @@ class LLMEngine:
222
self.tokenizer: BaseTokenizerGroup = get_tokenizer_group(
223
self.parallel_config.tokenizer_pool_config, **init_kwargs)
224
225
- if len(self.get_tokenizer()) != self.model_config.get_vocab_size():
226
- logger.warning(
227
- f"The tokenizer's vocabulary size {len(self.get_tokenizer())}"
228
- f" does not match the model's vocabulary size "
229
- f"{self.model_config.get_vocab_size()}. This might "
230
- f"cause an error in decoding. Please change config.json "
231
- "to match the tokenizer's vocabulary size.")
232
-
225
def _verify_args(self) -> None:
226
self.model_config.verify_with_parallel_config(self.parallel_config)
227
self.cache_config.verify_with_parallel_config(self.parallel_config)
vllm/entrypoints/openai/serving_engine.py
-8
@@ -68,14 +68,6 @@ class OpenAIServing:
68
tokenizer_mode=engine_model_config.tokenizer_mode,
69
trust_remote_code=engine_model_config.trust_remote_code)
70
71
- if len(self.tokenizer) != engine_model_config.get_vocab_size():
72
- logger.warning(
73
- f"The tokenizer's vocabulary size {len(self.tokenizer)}"
74
- f" does not match the model's vocabulary size "
75
- f"{engine_model_config.get_vocab_size()}. This might "
76
- f"cause an error in decoding. Please change config.json "
77
- "to match the tokenizer's vocabulary size.")
78
-
71
async def show_available_models(self) -> ModelList:
72
"""Show available models. Right now we only have one model."""
73
model_cards = [
vllm/transformers_utils/tokenizer.py
+7
-3
@@ -232,9 +232,13 @@ def detokenize_incrementally(
232
all_input_ids[:-1],
233
skip_special_tokens=skip_special_tokens)
234
235
- # Put new_token_id in a list so skip_special_tokens is respected
236
- new_tokens = tokenizer.convert_ids_to_tokens(
237
- [new_token_id], skip_special_tokens=skip_special_tokens)
235
+ # If the new token id is out of bounds, return an empty string.
236
+ if new_token_id >= len(tokenizer):
237
+ new_tokens = [""]
238
+ else:
239
+ # Put new_token_id in a list so skip_special_tokens is respected
240
+ new_tokens = tokenizer.convert_ids_to_tokens(
241
+ [new_token_id], skip_special_tokens=skip_special_tokens)
242
output_tokens = prev_tokens + new_tokens
243
244
# If this is the first iteration, return all tokens.