@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1dd4cb2935
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix edge cases for MistralTokenizer (#9625)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Prashant Gupta <prashantgupta@us.ibm.com> Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>

Travis Johnson committed Nov 1, 2024 at 11:33 UTC 1dd4cb2935fc3fff9c156b5772d18e0a0d1861f0
2 files changed +105 -39
tests/tokenization/test_detokenize.py
+63 -17
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Dict, Generator, List, Optional
2
3 import pytest
4 from transformers import AutoTokenizer
@@ -7,11 +7,17 @@ from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
7 from vllm.transformers_utils.detokenizer import (Detokenizer,
8 detokenize_incrementally)
9 from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
10 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
11
12 TRUTH = [
13 "Hello here, this is a simple test",
14 "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving", # noqa
14 - "我很感谢你的热情"
15 + "我很感谢你的热情",
16 + # Burmese text triggers an edge-case for Mistral's V3-Tekken tokenizer (eg.
17 + # for mistralai/Pixtral-12B-2409) where tokens may map to bytes with
18 + # incomplete UTF-8 characters
19 + # see https://github.com/vllm-project/vllm/pull/9625
20 + "ပုံပြင်လေးပြောပြပါ်",
21 ]
22 TOKENIZERS = [
23 "facebook/opt-125m",
@@ -24,6 +30,7 @@ TOKENIZERS = [
30 "tiiuae/falcon-7b",
31 "meta-llama/Llama-2-7b-hf",
32 "codellama/CodeLlama-7b-hf",
33 + "mistralai/Pixtral-12B-2409",
34 ]
35
36
@@ -49,15 +56,55 @@ def _run_incremental_decode(tokenizer, all_input_ids,
56 return decoded_text
57
58
59 +@pytest.fixture
60 +def tokenizer(tokenizer_name):
61 + return (MistralTokenizer.from_pretrained(tokenizer_name)
62 + if "mistral" in tokenizer_name else
63 + AutoTokenizer.from_pretrained(tokenizer_name))
64 +
65 +
66 +@pytest.mark.parametrize("tokenizer_name", ["mistralai/Pixtral-12B-2409"])
67 +@pytest.mark.parametrize(
68 + "truth",
69 + [
70 + # Burmese text triggers an edge-case where tokens may map to bytes with
71 + # incomplete UTF-8 characters
72 + "ပုံပြင်လေးပြောပြပါ",
73 + # Using "URGENCY" since "CY" has token id 130282
74 + "URGENCY🌶️",
75 + ])
76 +def test_mistral_edge_case(tokenizer, truth):
77 + """Test for a specific edge cases with V3-Tekken MistralTokenizer.
78 +
79 + See https://github.com/vllm-project/vllm/pull/9625
80 + """
81 + starting_index = 0
82 + all_input_ids = tokenizer(truth, add_special_tokens=False).input_ids
83 +
84 + decoded_text = _run_incremental_decode(tokenizer,
85 + all_input_ids,
86 + skip_special_tokens=True,
87 + starting_index=starting_index)
88 + assert decoded_text == truth
89 +
90 +
91 +@pytest.fixture
92 +def skip_special_tokens(request, tokenizer_name) -> Generator[bool, Any, None]:
93 + if "mistral" in tokenizer_name:
94 + yield (
95 + bool(True) if request.param else
96 + pytest.skip("mistral doesn't support skip_special_tokens=False"))
97 + else:
98 + yield bool(True) if request.param else bool(False)
99 +
100 +
101 @pytest.mark.parametrize("truth", TRUTH)
102 @pytest.mark.parametrize("with_prompt", [True, False])
54 -@pytest.mark.parametrize("tokenizer_id", TOKENIZERS)
55 -@pytest.mark.parametrize("skip_special_tokens", (True, False))
56 -def test_decode_streaming(tokenizer_id, truth, with_prompt,
57 - skip_special_tokens):
58 - tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
103 +@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
104 +@pytest.mark.parametrize("skip_special_tokens", (True, False), indirect=True)
105 +def test_decode_streaming(tokenizer, truth, with_prompt, skip_special_tokens):
106 if with_prompt:
60 - truth_tokens = tokenizer(truth, add_special_tokens=False)["input_ids"]
107 + truth_tokens = tokenizer(truth, add_special_tokens=False).input_ids
108 prompt_input_ids = truth_tokens[:len(truth) // 2]
109 generated_input_ids = truth_tokens[len(truth) // 2:]
110 all_input_ids = prompt_input_ids + generated_input_ids
@@ -68,7 +115,7 @@ def test_decode_streaming(tokenizer_id, truth, with_prompt,
115 else:
116 generated = truth
117 starting_index = 0
71 - all_input_ids = tokenizer(truth, add_special_tokens=False)["input_ids"]
118 + all_input_ids = tokenizer(truth, add_special_tokens=False).input_ids
119 if skip_special_tokens:
120 if tokenizer.bos_token_id is not None:
121 all_input_ids = [tokenizer.bos_token_id] + all_input_ids
@@ -98,7 +145,7 @@ def detokenizer(tokenizer_name: str) -> Detokenizer:
145 enable_lora=False,
146 max_num_seqs=100,
147 max_input_length=None,
101 - tokenizer_mode="auto",
148 + tokenizer_mode="mistral" if "mistral" in tokenizer_name else "auto",
149 trust_remote_code=False,
150 revision=None,
151 )
@@ -113,9 +160,8 @@ def detokenizer(tokenizer_name: str) -> Detokenizer:
160
161 @pytest.fixture(name="complete_sequence_token_ids")
162 def create_complete_sequence_token_ids(complete_sequence: str,
116 - tokenizer_name: str) -> List[int]:
117 - tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
118 - complete_sequence_token_ids = tokenizer(complete_sequence)["input_ids"]
163 + tokenizer) -> List[int]:
164 + complete_sequence_token_ids = tokenizer(complete_sequence).input_ids
165 return complete_sequence_token_ids
166
167
@@ -150,7 +196,7 @@ def create_dummy_prompt_logprobs(
196
197 @pytest.mark.parametrize("complete_sequence", TRUTH)
198 @pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
153 -@pytest.mark.parametrize("skip_special_tokens", [True, False])
199 +@pytest.mark.parametrize("skip_special_tokens", [True, False], indirect=True)
200 def test_decode_sequence_logprobs(complete_sequence: str,
201 complete_sequence_token_ids: List[int],
202 detokenizer: Detokenizer,
@@ -208,9 +254,9 @@ def test_decode_prompt_logprobs(complete_sequence_token_ids: List[int],
254
255 # decoded_prompt_logprobs doesn't contain the first token.
256 token_ids = complete_sequence_token_ids
211 - tokenzier = detokenizer.get_tokenizer_for_seq(seq)
212 - text_full = tokenzier.decode(token_ids, skip_special_tokens=True)
213 - text_first = tokenzier.decode(token_ids[0], skip_special_tokens=True)
257 + tokenizer = detokenizer.get_tokenizer_for_seq(seq)
258 + text_full = tokenizer.decode(token_ids, skip_special_tokens=True)
259 + text_first = tokenizer.decode(token_ids[0], skip_special_tokens=True)
260 text = text_full[len(text_first):]
261
262 # Text for logprobs for the chosen token should be the same as the
vllm/transformers_utils/tokenizers/mistral.py
+42 -22
@@ -16,9 +16,13 @@ from mistral_common.tokens.tokenizers.sentencepiece import (
16 from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
17 Tekkenizer)
18
19 +from vllm.logger import init_logger
20 +
21 if TYPE_CHECKING:
22 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
23
24 +logger = init_logger(__name__)
25 +
26
27 @dataclass
28 class Encoding:
@@ -72,20 +76,21 @@ class MistralTokenizer:
76 # Make sure special tokens will not raise
77 tokenizer_.special_token_policy = SpecialTokenPolicy.IGNORE
78
75 - self._vocab = {
76 - token: idx
77 - for idx, token in enumerate(tokenizer_.vocab())
78 - }
79 elif isinstance(tokenizer_, SentencePieceTokenizer):
80 - self._vocab = {
81 - token: idx
82 - for idx, token in enumerate(tokenizer_.vocab())
83 - }
80 + pass
81 else:
82 raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
83
84 + self._vocab = tokenizer_.vocab()
85 + # Convert to a Dict[str, int] to match protocol, but this is a lossy
86 + # conversion. There may be multiple token ids that decode to the same
87 + # string due to partial UTF-8 byte sequences being converted to �
88 + self._vocab_dict = {
89 + token: idx
90 + for idx, token in enumerate(self._vocab)
91 + }
92 self.tokenizer = tokenizer_
88 - self._max_token_id = max(self._vocab.values())
93 + self._max_token_id = self.vocab_size - 1
94
95 @classmethod
96 def from_pretrained(cls,
@@ -182,7 +187,9 @@ class MistralTokenizer:
187 return Encoding(input_ids=input_ids)
188
189 def get_vocab(self) -> Dict[str, int]:
185 - return self._vocab
190 + # NB: the dictionary form of the vocabulary collapses token ids that map
191 + # to the same string but have different bytes
192 + return self._vocab_dict
193
194 def get_added_vocab(self) -> Dict[str, int]:
195 # Mistral tokenizers have no added vocabulary
@@ -220,14 +227,20 @@ class MistralTokenizer:
227 if any(isinstance(t, bytes) for t in tokens):
228 # we need to encode and decode all tokens again
229 shift = self.tokenizer.num_special_tokens
223 - byte_tokens = [
224 - t.encode("utf-8") if not isinstance(t, bytes) else t
225 - for t in tokens
226 - ]
227 - ids = [
228 - self.tokenizer._tekken_token2id_nospecial[t] + shift
229 - for t in byte_tokens
230 - ]
230 +
231 + def _token_to_id(t: str):
232 + t_bytes = t.encode("utf-8") \
233 + if not isinstance(t, bytes) else t
234 + try:
235 + return shift + \
236 + self.tokenizer._tekken_token2id_nospecial[t_bytes]
237 + except KeyError:
238 + logger.warning(
239 + "Failed to convert token %s to id,"
240 + " replacing with <unk>", t_bytes)
241 + return self.tokenizer.unk_id
242 +
243 + ids = [_token_to_id(t) for t in tokens]
244 decoded = self.tokenizer.decode(ids)
245 else:
246 decoded = "".join(tokens)
@@ -236,7 +249,13 @@ class MistralTokenizer:
249
250 return decoded
251
239 - def decode(self, ids: Union[List[int], int]) -> str:
252 + def decode(self,
253 + ids: Union[List[int], int],
254 + skip_special_tokens: bool = True) -> str:
255 + assert (
256 + skip_special_tokens
257 + ), "Skipping special tokens is not supported for Mistral tokenizers."
258 +
259 if isinstance(ids, int):
260 ids = [ids]
261 return self.tokenizer.decode(ids)
@@ -257,10 +276,11 @@ class MistralTokenizer:
276
277 tokens = [self.tokenizer.id_to_piece(id) for id in ids]
278
260 - if any(t.strip() == "�" for t in tokens):
261 - # if any stripped decoded token is undefined
262 - # because it's invalid unicode then pass bytes
279 + if any("�" in t for t in tokens):
280 + # if a decoded token contains the replacement character, then the
281 + # token has an incomplete UTF-8 character so we must use bytes
282 # See: https://github.com/vllm-project/vllm/pull/8640
283 + # https://github.com/vllm-project/vllm/pull/9625
284 tokens = [self.tokenizer.id_to_byte_piece(id) for id in ids]
285
286 return tokens