A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix edge cases for MistralTokenizer (#9625)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>
Travis Johnson committed
Nov 1, 2024 at 11:33 UTC
1dd4cb2935fc3fff9c156b5772d18e0a0d1861f0
2 files changed
+105
-39
tests/tokenization/test_detokenize.py
+63
-17
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Dict, Generator, List, Optional
2
3
import pytest
4
from transformers import AutoTokenizer
@@ -7,11 +7,17 @@ from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
7
from vllm.transformers_utils.detokenizer import (Detokenizer,
8
detokenize_incrementally)
9
from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
10
+from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
11
12
TRUTH = [
13
"Hello here, this is a simple test",
14
"vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving", # noqa
14
- "我很感谢你的热情"
15
+ "我很感谢你的热情",
16
+ # Burmese text triggers an edge-case for Mistral's V3-Tekken tokenizer (eg.
17
+ # for mistralai/Pixtral-12B-2409) where tokens may map to bytes with
18
+ # incomplete UTF-8 characters
19
+ # see https://github.com/vllm-project/vllm/pull/9625
20
+ "ပုံပြင်လေးပြောပြပါ်",
21
]
22
TOKENIZERS = [
23
"facebook/opt-125m",
@@ -24,6 +30,7 @@ TOKENIZERS = [
30
"tiiuae/falcon-7b",
31
"meta-llama/Llama-2-7b-hf",
32
"codellama/CodeLlama-7b-hf",
33
+ "mistralai/Pixtral-12B-2409",
34
]
35
36
@@ -49,15 +56,55 @@ def _run_incremental_decode(tokenizer, all_input_ids,
56
return decoded_text
57
58
59
+@pytest.fixture
60
+def tokenizer(tokenizer_name):
61
+ return (MistralTokenizer.from_pretrained(tokenizer_name)
62
+ if "mistral" in tokenizer_name else
63
+ AutoTokenizer.from_pretrained(tokenizer_name))
64
+
65
+
66
+@pytest.mark.parametrize("tokenizer_name", ["mistralai/Pixtral-12B-2409"])
67
+@pytest.mark.parametrize(
68
+ "truth",
69
+ [
70
+ # Burmese text triggers an edge-case where tokens may map to bytes with
71
+ # incomplete UTF-8 characters
72
+ "ပုံပြင်လေးပြောပြပါ",
73
+ # Using "URGENCY" since "CY" has token id 130282
74
+ "URGENCY🌶️",
75
+ ])
76
+def test_mistral_edge_case(tokenizer, truth):
77
+ """Test for a specific edge cases with V3-Tekken MistralTokenizer.
78
+
79
+ See https://github.com/vllm-project/vllm/pull/9625
80
+ """
81
+ starting_index = 0
82
+ all_input_ids = tokenizer(truth, add_special_tokens=False).input_ids
83
+
84
+ decoded_text = _run_incremental_decode(tokenizer,
85
+ all_input_ids,
86
+ skip_special_tokens=True,
87
+ starting_index=starting_index)
88
+ assert decoded_text == truth
89
+
90
+
91
+@pytest.fixture
92
+def skip_special_tokens(request, tokenizer_name) -> Generator[bool, Any, None]:
93
+ if "mistral" in tokenizer_name:
94
+ yield (
95
+ bool(True) if request.param else
96
+ pytest.skip("mistral doesn't support skip_special_tokens=False"))
97
+ else:
98
+ yield bool(True) if request.param else bool(False)
99
+
100
+
101
@pytest.mark.parametrize("truth", TRUTH)
102
@pytest.mark.parametrize("with_prompt", [True, False])
54
-@pytest.mark.parametrize("tokenizer_id", TOKENIZERS)
55
-@pytest.mark.parametrize("skip_special_tokens", (True, False))
56
-def test_decode_streaming(tokenizer_id, truth, with_prompt,
57
- skip_special_tokens):
58
- tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
103
+@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
104
+@pytest.mark.parametrize("skip_special_tokens", (True, False), indirect=True)
105
+def test_decode_streaming(tokenizer, truth, with_prompt, skip_special_tokens):
106
if with_prompt:
60
- truth_tokens = tokenizer(truth, add_special_tokens=False)["input_ids"]
107
+ truth_tokens = tokenizer(truth, add_special_tokens=False).input_ids
108
prompt_input_ids = truth_tokens[:len(truth) // 2]
109
generated_input_ids = truth_tokens[len(truth) // 2:]
110
all_input_ids = prompt_input_ids + generated_input_ids
@@ -68,7 +115,7 @@ def test_decode_streaming(tokenizer_id, truth, with_prompt,
115
else:
116
generated = truth
117
starting_index = 0
71
- all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
118
+ all_input_ids = tokenizer(truth, add_special_tokens=False).input_ids
119
if skip_special_tokens:
120
if tokenizer.bos_token_id is not None:
121
all_input_ids = [tokenizer.bos_token_id] + all_input_ids
@@ -98,7 +145,7 @@ def detokenizer(tokenizer_name: str) -> Detokenizer:
145
enable_lora=False,
146
max_num_seqs=100,
147
max_input_length=None,
101
- tokenizer_mode="auto",
148
+ tokenizer_mode="mistral" if "mistral" in tokenizer_name else "auto",
149
trust_remote_code=False,
150
revision=None,
151
)
@@ -113,9 +160,8 @@ def detokenizer(tokenizer_name: str) -> Detokenizer:
160
161
@pytest.fixture(name="complete_sequence_token_ids")
162
def create_complete_sequence_token_ids(complete_sequence: str,
116
- tokenizer_name: str) -> List[int]:
117
- tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
118
- complete_sequence_token_ids = tokenizer(complete_sequence)["input_ids"]
163
+ tokenizer) -> List[int]:
164
+ complete_sequence_token_ids = tokenizer(complete_sequence).input_ids
165
return complete_sequence_token_ids
166
167
@@ -150,7 +196,7 @@ def create_dummy_prompt_logprobs(
196
197
@pytest.mark.parametrize("complete_sequence", TRUTH)
198
@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
153
-@pytest.mark.parametrize("skip_special_tokens", [True, False])
199
+@pytest.mark.parametrize("skip_special_tokens", [True, False], indirect=True)
200
def test_decode_sequence_logprobs(complete_sequence: str,
201
complete_sequence_token_ids: List[int],
202
detokenizer: Detokenizer,
@@ -208,9 +254,9 @@ def test_decode_prompt_logprobs(complete_sequence_token_ids: List[int],
254
255
# decoded_prompt_logprobs doesn't contain the first token.
256
token_ids = complete_sequence_token_ids
211
- tokenzier = detokenizer.get_tokenizer_for_seq(seq)
212
- text_full = tokenzier.decode(token_ids, skip_special_tokens=True)
213
- text_first = tokenzier.decode(token_ids[0], skip_special_tokens=True)
257
+ tokenizer = detokenizer.get_tokenizer_for_seq(seq)
258
+ text_full = tokenizer.decode(token_ids, skip_special_tokens=True)
259
+ text_first = tokenizer.decode(token_ids[0], skip_special_tokens=True)
260
text = text_full[len(text_first):]
261
262
# Text for logprobs for the chosen token should be the same as the
vllm/transformers_utils/tokenizers/mistral.py
+42
-22
@@ -16,9 +16,13 @@ from mistral_common.tokens.tokenizers.sentencepiece import (
16
from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
17
Tekkenizer)
18
19
+from vllm.logger import init_logger
20
+
21
if TYPE_CHECKING:
22
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
23
24
+logger = init_logger(__name__)
25
+
26
27
@dataclass
28
class Encoding:
@@ -72,20 +76,21 @@ class MistralTokenizer:
76
# Make sure special tokens will not raise
77
tokenizer_.special_token_policy = SpecialTokenPolicy.IGNORE
78
75
- self._vocab = {
76
- token: idx
77
- for idx, token in enumerate(tokenizer_.vocab())
78
- }
79
elif isinstance(tokenizer_, SentencePieceTokenizer):
80
- self._vocab = {
81
- token: idx
82
- for idx, token in enumerate(tokenizer_.vocab())
83
- }
80
+ pass
81
else:
82
raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
83
84
+ self._vocab = tokenizer_.vocab()
85
+ # Convert to a Dict[str, int] to match protocol, but this is a lossy
86
+ # conversion. There may be multiple token ids that decode to the same
87
+ # string due to partial UTF-8 byte sequences being converted to �
88
+ self._vocab_dict = {
89
+ token: idx
90
+ for idx, token in enumerate(self._vocab)
91
+ }
92
self.tokenizer = tokenizer_
88
- self._max_token_id = max(self._vocab.values())
93
+ self._max_token_id = self.vocab_size - 1
94
95
@classmethod
96
def from_pretrained(cls,
@@ -182,7 +187,9 @@ class MistralTokenizer:
187
return Encoding(input_ids=input_ids)
188
189
def get_vocab(self) -> Dict[str, int]:
185
- return self._vocab
190
+ # NB: the dictionary form of the vocabulary collapses token ids that map
191
+ # to the same string but have different bytes
192
+ return self._vocab_dict
193
194
def get_added_vocab(self) -> Dict[str, int]:
195
# Mistral tokenizers have no added vocabulary
@@ -220,14 +227,20 @@ class MistralTokenizer:
227
if any(isinstance(t, bytes) for t in tokens):
228
# we need to encode and decode all tokens again
229
shift = self.tokenizer.num_special_tokens
223
- byte_tokens = [
224
- t.encode("utf-8") if not isinstance(t, bytes) else t
225
- for t in tokens
226
- ]
227
- ids = [
228
- self.tokenizer._tekken_token2id_nospecial[t] + shift
229
- for t in byte_tokens
230
- ]
230
+
231
+ def _token_to_id(t: str):
232
+ t_bytes = t.encode("utf-8") \
233
+ if not isinstance(t, bytes) else t
234
+ try:
235
+ return shift + \
236
+ self.tokenizer._tekken_token2id_nospecial[t_bytes]
237
+ except KeyError:
238
+ logger.warning(
239
+ "Failed to convert token %s to id,"
240
+ " replacing with <unk>", t_bytes)
241
+ return self.tokenizer.unk_id
242
+
243
+ ids = [_token_to_id(t) for t in tokens]
244
decoded = self.tokenizer.decode(ids)
245
else:
246
decoded = "".join(tokens)
@@ -236,7 +249,13 @@ class MistralTokenizer:
249
250
return decoded
251
239
- def decode(self, ids: Union[List[int], int]) -> str:
252
+ def decode(self,
253
+ ids: Union[List[int], int],
254
+ skip_special_tokens: bool = True) -> str:
255
+ assert (
256
+ skip_special_tokens
257
+ ), "Skipping special tokens is not supported for Mistral tokenizers."
258
+
259
if isinstance(ids, int):
260
ids = [ids]
261
return self.tokenizer.decode(ids)
@@ -257,10 +276,11 @@ class MistralTokenizer:
276
277
tokens = [self.tokenizer.id_to_piece(id) for id in ids]
278
260
- if any(t.strip() == "�" for t in tokens):
261
- # if any stripped decoded token is undefined
262
- # because it's invalid unicode then pass bytes
279
+ if any("�" in t for t in tokens):
280
+ # if a decoded token contains the replacement character, then the
281
+ # token has an incomplete UTF-8 character so we must use bytes
282
# See: https://github.com/vllm-project/vllm/pull/8640
283
+ # https://github.com/vllm-project/vllm/pull/9625
284
tokens = [self.tokenizer.id_to_byte_piece(id) for id in ids]
285
286
return tokens