@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2bcbae704c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Fix edge-case crash when using chat with the Mistral Tekken Tokenizer (#10051)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Nov 5, 2024 at 21:28 UTC 2bcbae704c0d52913c6a2887260fc6bde6c20361
2 files changed +12 -5
tests/models/decoder_only/language/test_mistral.py
+6 -3
@@ -10,19 +10,22 @@ from ...utils import check_logprobs_close
10
11 MODELS = [
12 "mistralai/Mistral-7B-Instruct-v0.1",
13 - "mistralai/Mistral-7B-Instruct-v0.3",
14 - # Mistral-Nemo is to big for CI, but passes locally
15 - # "mistralai/Mistral-Nemo-Instruct-2407"
13 ]
14
15 MISTRAL_FORMAT_MODELS = [
16 "mistralai/Mistral-7B-Instruct-v0.3",
17 + # uses the v3-Tekken tokenizer
18 + "mistralai/Ministral-8B-Instruct-2410",
19 + # Mistral-Nemo is to big for CI, but passes locally
20 + # "mistralai/Mistral-Nemo-Instruct-2407"
21 ]
22
23 SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
24 SYMBOLIC_LANG_PROMPTS = [
25 "勇敢な船乗りについての詩を書く", # japanese
26 "寫一首關於勇敢的水手的詩", # chinese
27 + "ပုံပြင်လေးပြောပြပါ်:\n", # burmese
28 + "Repeat the phrase 'URGENCY🌶️':\nURGENCY🌶️\nURGENCY🌶️\n", # see https://github.com/vllm-project/vllm/pull/9625
29 ]
30
31 # for function calling
vllm/transformers_utils/tokenizers/mistral.py
+6 -2
@@ -254,7 +254,7 @@ class MistralTokenizer:
254 skip_special_tokens: bool = True) -> str:
255 assert (
256 skip_special_tokens
257 - ), "Skipping special tokens is not supported for Mistral tokenizers."
257 + ), "skip_special_tokens=False is not supported for Mistral tokenizers."
258
259 if isinstance(ids, int):
260 ids = [ids]
@@ -268,12 +268,16 @@ class MistralTokenizer:
268 # TODO(Patrick) - potentially allow special tokens to not be skipped
269 assert (
270 skip_special_tokens
271 - ), "Skipping special tokens is not supported for Mistral tokenizers."
271 + ), "skip_special_tokens=False is not supported for Mistral tokenizers."
272
273 assert isinstance(self.tokenizer,
274 (Tekkenizer, SentencePieceTokenizer)), type(
275 self.tokenizer)
276
277 + if isinstance(self.tokenizer, Tekkenizer):
278 + # skip special tokens
279 + ids = [i for i in ids if i > self.tokenizer.num_special_tokens]
280 +
281 tokens = [self.tokenizer.id_to_piece(id) for id in ids]
282
283 if any("�" in t for t in tokens):