@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b4e4eda92e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix][Core] Fix tekken edge case for mistral tokenizer (#8640)

Patrick von Platen committed Sep 20, 2024 at 23:33 UTC b4e4eda92e1d3a013fc4007db64b69d8604264ff
2 files changed +54 -4
tests/models/decoder_only/language/test_mistral.py
+25 -1
@@ -4,7 +4,7 @@ Run `pytest tests/models/test_mistral.py`.
4 """
5 import pytest
6
7 -from vllm import SamplingParams
7 +from vllm import LLM, SamplingParams
8
9 from ...utils import check_logprobs_close
10
@@ -16,6 +16,10 @@ MODELS = [
16 ]
17
18 SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
19 +SYMBOLIC_LANG_PROMPTS = [
20 + "勇敢な船乗りについての詩を書く", # japanese
21 + "寫一首關於勇敢的水手的詩", # chinese
22 +]
23
24 # for function calling
25 TOOLS = [{
@@ -131,6 +135,26 @@ def test_mistral_format(
135 )
136
137
138 +@pytest.mark.parametrize("model", MODELS[1:])
139 +@pytest.mark.parametrize("dtype", ["bfloat16"])
140 +@pytest.mark.parametrize("prompt", SYMBOLIC_LANG_PROMPTS)
141 +def test_mistral_symbolic_languages(
142 + model: str,
143 + dtype: str,
144 + prompt: str,
145 +) -> None:
146 + prompt = "hi"
147 + msg = {"role": "user", "content": prompt}
148 + llm = LLM(model=model,
149 + dtype=dtype,
150 + max_model_len=8192,
151 + tokenizer_mode="mistral",
152 + config_format="mistral",
153 + load_format="mistral")
154 + outputs = llm.chat([msg], sampling_params=SAMPLING_PARAMS)
155 + assert "�" not in outputs[0].outputs[0].text.strip()
156 +
157 +
158 @pytest.mark.parametrize("dtype", ["bfloat16"])
159 @pytest.mark.parametrize("model", MODELS[1:]) # v1 can't do func calling
160 def test_mistral_function_calling(
vllm/transformers_utils/tokenizers/mistral.py
+29 -3
@@ -175,10 +175,29 @@ class MistralTokenizer:
175
176 def convert_tokens_to_string(self, tokens: List[str]) -> str:
177 if isinstance(self.tokenizer, Tekkenizer):
178 - return "".join(t for t in tokens
179 - if t not in self.tokenizer._all_special_tokens)
178 + tokens = [
179 + t for t in tokens
180 + if t not in self.tokenizer._all_special_tokens
181 + ]
182 +
183 + if any(isinstance(t, bytes) for t in tokens):
184 + # we need to encode and decode all tokens again
185 + shift = self.tokenizer.num_special_tokens
186 + byte_tokens = [
187 + t.encode("utf-8") if not isinstance(t, bytes) else t
188 + for t in tokens
189 + ]
190 + ids = [
191 + self.tokenizer._tekken_token2id_nospecial[t] + shift
192 + for t in byte_tokens
193 + ]
194 + decoded = self.tokenizer.decode(ids)
195 + else:
196 + decoded = "".join(tokens)
197 else:
181 - return self.tokenizer.decode(tokens) # type: ignore[arg-type]
198 + decoded = self.tokenizer.decode(tokens) # type: ignore[arg-type]
199 +
200 + return decoded
201
202 def decode(self, ids: Union[List[int], int]) -> str:
203 if isinstance(ids, int):
@@ -200,4 +219,11 @@ class MistralTokenizer:
219 self.tokenizer)
220
221 tokens = [self.tokenizer.id_to_piece(id) for id in ids]
222 +
223 + if any(t.strip() == "�" for t in tokens):
224 + # if any stripped decoded token is undefined
225 + # because it's invalid unicode then pass bytes
226 + # See: https://github.com/vllm-project/vllm/pull/8640
227 + tokens = [self.tokenizer.id_to_byte_piece(id) for id in ids]
228 +
229 return tokens