A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix][Core] Fix tekken edge case for mistral tokenizer (#8640)
Patrick von Platen committed
Sep 20, 2024 at 23:33 UTC
b4e4eda92e1d3a013fc4007db64b69d8604264ff
2 files changed
+54
-4
tests/models/decoder_only/language/test_mistral.py
+25
-1
@@ -4,7 +4,7 @@ Run `pytest tests/models/test_mistral.py`.
4
"""
5
import pytest
6
7
-from vllm import SamplingParams
7
+from vllm import LLM, SamplingParams
8
9
from ...utils import check_logprobs_close
10
@@ -16,6 +16,10 @@ MODELS = [
16
]
17
18
SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
19
+SYMBOLIC_LANG_PROMPTS = [
20
+ "勇敢な船乗りについての詩を書く", # japanese
21
+ "寫一首關於勇敢的水手的詩", # chinese
22
+]
23
24
# for function calling
25
TOOLS = [{
@@ -131,6 +135,26 @@ def test_mistral_format(
135
)
136
137
138
+@pytest.mark.parametrize("model", MODELS[1:])
139
+@pytest.mark.parametrize("dtype", ["bfloat16"])
140
+@pytest.mark.parametrize("prompt", SYMBOLIC_LANG_PROMPTS)
141
+def test_mistral_symbolic_languages(
142
+ model: str,
143
+ dtype: str,
144
+ prompt: str,
145
+) -> None:
146
+ prompt = "hi"
147
+ msg = {"role": "user", "content": prompt}
148
+ llm = LLM(model=model,
149
+ dtype=dtype,
150
+ max_model_len=8192,
151
+ tokenizer_mode="mistral",
152
+ config_format="mistral",
153
+ load_format="mistral")
154
+ outputs = llm.chat([msg], sampling_params=SAMPLING_PARAMS)
155
+ assert "�" not in outputs[0].outputs[0].text.strip()
156
+
157
+
158
@pytest.mark.parametrize("dtype", ["bfloat16"])
159
@pytest.mark.parametrize("model", MODELS[1:]) # v1 can't do func calling
160
def test_mistral_function_calling(
vllm/transformers_utils/tokenizers/mistral.py
+29
-3
@@ -175,10 +175,29 @@ class MistralTokenizer:
175
176
def convert_tokens_to_string(self, tokens: List[str]) -> str:
177
if isinstance(self.tokenizer, Tekkenizer):
178
- return "".join(t for t in tokens
179
- if t not in self.tokenizer._all_special_tokens)
178
+ tokens = [
179
+ t for t in tokens
180
+ if t not in self.tokenizer._all_special_tokens
181
+ ]
182
+
183
+ if any(isinstance(t, bytes) for t in tokens):
184
+ # we need to encode and decode all tokens again
185
+ shift = self.tokenizer.num_special_tokens
186
+ byte_tokens = [
187
+ t.encode("utf-8") if not isinstance(t, bytes) else t
188
+ for t in tokens
189
+ ]
190
+ ids = [
191
+ self.tokenizer._tekken_token2id_nospecial[t] + shift
192
+ for t in byte_tokens
193
+ ]
194
+ decoded = self.tokenizer.decode(ids)
195
+ else:
196
+ decoded = "".join(tokens)
197
else:
181
- return self.tokenizer.decode(tokens) # type: ignore[arg-type]
198
+ decoded = self.tokenizer.decode(tokens) # type: ignore[arg-type]
199
+
200
+ return decoded
201
202
def decode(self, ids: Union[List[int], int]) -> str:
203
if isinstance(ids, int):
@@ -200,4 +219,11 @@ class MistralTokenizer:
219
self.tokenizer)
220
221
tokens = [self.tokenizer.id_to_piece(id) for id in ids]
222
+
223
+ if any(t.strip() == "�" for t in tokens):
224
+ # if any stripped decoded token is undefined
225
+ # because it's invalid unicode then pass bytes
226
+ # See: https://github.com/vllm-project/vllm/pull/8640
227
+ tokens = [self.tokenizer.id_to_byte_piece(id) for id in ids]
228
+
229
return tokens