A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Model] Add Mistral Tokenization to improve robustness and chat encoding (#7739)
Patrick von Platen committed
Aug 27, 2024 at 14:40 UTC
6fc4e6e07a55559c3744212b4d562e20d024e661
12 files changed
+275
-60
docs/requirements-docs.txt
+1
@@ -11,4 +11,5 @@ pydantic >= 2.8
11
torch
12
py-cpuinfo
13
transformers
14
+mistral_common >= 1.3.4
15
openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
requirements-common.txt
+1
@@ -26,3 +26,4 @@ librosa # Required for audio processing
26
soundfile # Required for audio processing
27
gguf == 0.9.1
28
importlib_metadata
29
+mistral_common >= 1.3.4
tests/models/test_mistral.py
+3
-1
@@ -30,9 +30,11 @@ def test_models(
30
hf_outputs = hf_model.generate_greedy_logprobs_limit(
31
example_prompts, max_tokens, num_logprobs)
32
33
- with vllm_runner(model, dtype=dtype) as vllm_model:
33
+ with vllm_runner(model, dtype=dtype,
34
+ tokenizer_mode="mistral") as vllm_model:
35
vllm_outputs = vllm_model.generate_greedy_logprobs(
36
example_prompts, max_tokens, num_logprobs)
37
+
38
check_logprobs_close(
39
outputs_0_lst=hf_outputs,
40
outputs_1_lst=vllm_outputs,
vllm/config.py
+4
-3
@@ -61,7 +61,8 @@ class ModelConfig:
61
output when `served_model_name` is not specified.
62
tokenizer: Name or path of the huggingface tokenizer to use.
63
tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
64
- available, and "slow" will always use the slow tokenizer.
64
+ available, "slow" will always use the slow tokenizer, and
65
+ "mistral" will always use the tokenizer from `mistral_common`.
66
trust_remote_code: Trust remote code (e.g., from HuggingFace) when
67
downloading the model and tokenizer.
68
dtype: Data type for model weights and activations. The "auto" option
@@ -246,10 +247,10 @@ class ModelConfig:
247
248
def _verify_tokenizer_mode(self) -> None:
249
tokenizer_mode = self.tokenizer_mode.lower()
249
- if tokenizer_mode not in ["auto", "slow"]:
250
+ if tokenizer_mode not in ["auto", "slow", "mistral"]:
251
raise ValueError(
252
f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
252
- "either 'auto' or 'slow'.")
253
+ "either 'auto', 'slow' or 'mistral'.")
254
self.tokenizer_mode = tokenizer_mode
255
256
def _verify_embedding_mode(self) -> None:
vllm/engine/arg_utils.py
+3
-2
@@ -198,10 +198,11 @@ class EngineArgs:
198
'--tokenizer-mode',
199
type=str,
200
default=EngineArgs.tokenizer_mode,
201
- choices=['auto', 'slow'],
201
+ choices=['auto', 'slow', 'mistral'],
202
help='The tokenizer mode.\n\n* "auto" will use the '
203
'fast tokenizer if available.\n* "slow" will '
204
- 'always use the slow tokenizer.')
204
+ 'always use the slow tokenizer. \n* '
205
+ '"mistral" will always use the `mistral_common` tokenizer.')
206
parser.add_argument('--trust-remote-code',
207
action='store_true',
208
help='Trust remote code from huggingface.')
vllm/entrypoints/chat_utils.py
+1
-3
@@ -267,7 +267,7 @@ def apply_chat_template(
267
*,
268
tokenize: bool = False, # Different from HF's default
269
**kwargs: Any,
270
-) -> str:
270
+) -> Union[str, List[int]]:
271
if chat_template is None and tokenizer.chat_template is None:
272
raise ValueError(
273
"As of transformers v4.44, default chat template is no longer "
@@ -280,6 +280,4 @@ def apply_chat_template(
280
tokenize=tokenize,
281
**kwargs,
282
)
283
- assert isinstance(prompt, str)
284
-
283
return prompt
vllm/entrypoints/llm.py
+9
-3
@@ -390,15 +390,21 @@ class LLM:
390
conversations, _ = parse_chat_messages(messages, model_config,
391
tokenizer)
392
393
- prompts = apply_chat_template(
393
+ prompt = apply_chat_template(
394
tokenizer,
395
conversations,
396
chat_template=chat_template,
397
add_generation_prompt=add_generation_prompt)
398
399
+ inputs: PromptInputs
400
+ if isinstance(prompt, list) and isinstance(prompt[0], int):
401
+ inputs = TokensPrompt(prompt_token_ids=prompt)
402
+ else:
403
+ inputs = TextPrompt(prompt=prompt)
404
+
405
return self.generate(
400
- prompts,
401
- sampling_params,
406
+ inputs,
407
+ sampling_params=sampling_params,
408
use_tqdm=use_tqdm,
409
lora_request=lora_request,
410
)
vllm/entrypoints/openai/serving_chat.py
+18
-8
@@ -22,7 +22,8 @@ from vllm.entrypoints.openai.protocol import (
22
FunctionCall, ToolCall, UsageInfo)
23
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
24
OpenAIServing,
25
- PromptAdapterPath)
25
+ PromptAdapterPath,
26
+ TextTokensPrompt)
27
from vllm.inputs import TokensPrompt
28
from vllm.logger import init_logger
29
from vllm.multimodal import MultiModalDataDict
@@ -130,13 +131,22 @@ class OpenAIServingChat(OpenAIServing):
131
guided_decode_logits_processor = (
132
await self._guided_decode_logits_processor(request, tokenizer))
133
133
- prompt_inputs = self._tokenize_prompt_input(
134
- request,
135
- tokenizer,
136
- prompt,
137
- truncate_prompt_tokens=request.truncate_prompt_tokens,
138
- add_special_tokens=request.add_special_tokens,
139
- )
134
+ if isinstance(prompt, str):
135
+ prompt_inputs = self._tokenize_prompt_input(
136
+ request,
137
+ tokenizer,
138
+ prompt,
139
+ truncate_prompt_tokens=request.truncate_prompt_tokens,
140
+ add_special_tokens=request.add_special_tokens,
141
+ )
142
+ else:
143
+ assert isinstance(prompt, list) and isinstance(
144
+ prompt[0], int
145
+ ), "Prompt has to be either a string or a list of token ids"
146
+ prompt_inputs = TextTokensPrompt(
147
+ prompt=tokenizer.decode(prompt), prompt_token_ids=prompt)
148
+
149
+ assert prompt_inputs is not None
150
151
sampling_params = request.to_sampling_params(
152
tokenizer,
vllm/transformers_utils/detokenizer.py
+1
-1
@@ -230,7 +230,7 @@ def convert_prompt_ids_to_tokens(
230
prefix_offset = max(
231
read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
232
# This is required to guard against out-of-vocab prompt token ids
233
- _replace_none_with_empty(new_tokens)
233
+ _replace_none_with_empty(new_tokens) # type: ignore[arg-type]
234
return new_tokens, prefix_offset, read_offset
235
236
vllm/transformers_utils/tokenizer.py
+58
-36
@@ -1,4 +1,5 @@
1
import os
2
+import warnings
3
from pathlib import Path
4
from typing import Optional, Union
5
@@ -9,12 +10,14 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
10
from vllm.envs import VLLM_USE_MODELSCOPE
11
from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
12
-from vllm.transformers_utils.tokenizers import BaichuanTokenizer
13
+from vllm.transformers_utils.tokenizers import (BaichuanTokenizer,
14
+ MistralTokenizer)
15
from vllm.utils import make_async
16
17
logger = init_logger(__name__)
18
17
-AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
19
+AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
20
+ MistralTokenizer]
21
22
23
def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
@@ -99,45 +102,64 @@ def get_tokenizer(
102
kwargs["gguf_file"] = Path(tokenizer_name).name
103
tokenizer_name = Path(tokenizer_name).parent
104
102
- try:
103
- tokenizer = AutoTokenizer.from_pretrained(
104
- tokenizer_name,
105
- *args,
106
- trust_remote_code=trust_remote_code,
107
- revision=revision,
108
- **kwargs)
109
- except ValueError as e:
110
- # If the error pertains to the tokenizer class not existing or not
111
- # currently being imported, suggest using the --trust-remote-code flag.
112
- if (not trust_remote_code and
113
- ("does not exist or is not currently imported." in str(e)
114
- or "requires you to execute the tokenizer file" in str(e))):
115
- err_msg = (
116
- "Failed to load the tokenizer. If the tokenizer is a custom "
117
- "tokenizer not yet available in the HuggingFace transformers "
118
- "library, consider setting `trust_remote_code=True` in LLM "
119
- "or using the `--trust-remote-code` flag in the CLI.")
120
- raise RuntimeError(err_msg) from e
121
- else:
122
- raise e
123
- except AttributeError as e:
124
- if "BaichuanTokenizer" in str(e):
125
- # This is for the error "'BaichuanTokenizer' object has no
126
- # attribute 'sp_model'".
127
- tokenizer = BaichuanTokenizer.from_pretrained(
105
+ # if tokenizer is from official mistral org
106
+ is_from_mistral_org = str(tokenizer_name).split("/")[0] == "mistralai"
107
+ if is_from_mistral_org and tokenizer_mode != "mistral":
108
+ warnings.warn(
109
+ 'It is strongly recommended to run mistral models with '
110
+ '`--tokenizer_mode "mistral"` to ensure correct '
111
+ 'encoding and decoding.',
112
+ FutureWarning,
113
+ stacklevel=2)
114
+
115
+ if tokenizer_mode == "mistral":
116
+ tokenizer = MistralTokenizer.from_pretrained(str(tokenizer_name),
117
+ revision=revision)
118
+ else:
119
+ try:
120
+ tokenizer = AutoTokenizer.from_pretrained(
121
tokenizer_name,
122
*args,
123
trust_remote_code=trust_remote_code,
124
revision=revision,
132
- **kwargs)
133
- else:
134
- raise e
125
+ **kwargs,
126
+ )
127
+ except ValueError as e:
128
+ # If the error pertains to the tokenizer class not existing or not
129
+ # currently being imported,
130
+ # suggest using the --trust-remote-code flag.
131
+ if not trust_remote_code and (
132
+ "does not exist or is not currently imported." in str(e)
133
+ or "requires you to execute the tokenizer file" in str(e)):
134
+ err_msg = ("Failed to load the tokenizer. If the tokenizer "
135
+ "is a custom tokenizer not yet available in the "
136
+ "HuggingFace transformers library, consider "
137
+ "setting `trust_remote_code=True` in LLM or using "
138
+ "the `--trust-remote-code` flag in the CLI.")
139
+ raise RuntimeError(err_msg) from e
140
+ else:
141
+ raise e
142
+ except AttributeError as e:
143
+ if "BaichuanTokenizer" in str(e):
144
+ # This is for the error "'BaichuanTokenizer' object has no
145
+ # attribute 'sp_model'".
146
+ tokenizer = BaichuanTokenizer.from_pretrained(
147
+ tokenizer_name,
148
+ *args,
149
+ trust_remote_code=trust_remote_code,
150
+ revision=revision,
151
+ **kwargs,
152
+ )
153
+ else:
154
+ raise e
155
+
156
+ if not isinstance(tokenizer, PreTrainedTokenizerFast):
157
+ logger.warning(
158
+ "Using a slow tokenizer. This might cause a significant "
159
+ "slowdown. Consider using a fast tokenizer instead.")
160
+ tokenizer = get_cached_tokenizer(tokenizer)
161
136
- if not isinstance(tokenizer, PreTrainedTokenizerFast):
137
- logger.warning(
138
- "Using a slow tokenizer. This might cause a significant "
139
- "slowdown. Consider using a fast tokenizer instead.")
140
- return get_cached_tokenizer(tokenizer)
162
+ return tokenizer
163
164
165
def get_lora_tokenizer(lora_request: LoRARequest, *args,
vllm/transformers_utils/tokenizers/__init__.py
+2
-3
@@ -1,5 +1,4 @@
1
from vllm.transformers_utils.tokenizers.baichuan import BaichuanTokenizer
2
+from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
3
3
-__all__ = [
4
- "BaichuanTokenizer",
5
-]
4
+__all__ = ["BaichuanTokenizer", "MistralTokenizer"]
vllm/transformers_utils/tokenizers/mistral.py
new
+174
@@ -0,0 +1,174 @@
1
+import os
2
+import re
3
+from dataclasses import dataclass
4
+from pathlib import Path
5
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
6
+
7
+from huggingface_hub import HfApi, hf_hub_download
8
+# yapf: disable
9
+from mistral_common.tokens.tokenizers.mistral import ChatCompletionRequest
10
+from mistral_common.tokens.tokenizers.mistral import (
11
+ MistralTokenizer as PublicMistralTokenizer)
12
+# yapf: enable
13
+from mistral_common.tokens.tokenizers.sentencepiece import (
14
+ SentencePieceTokenizer)
15
+from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
16
+ Tekkenizer)
17
+
18
+if TYPE_CHECKING:
19
+ from vllm.entrypoints.chat_utils import ConversationMessage
20
+
21
+
22
+@dataclass
23
+class Encoding:
24
+ input_ids: List[int]
25
+
26
+
27
+def find_tokenizer_file(files: List[str]):
28
+ file_pattern = re.compile(r"^tokenizer\.model\.v.*$|^tekken\.json$")
29
+
30
+ matched_files = [file for file in files if file_pattern.match(file)]
31
+ if len(matched_files) > 1:
32
+ raise OSError(f"Found {len(matched_files)} files matching the "
33
+ "pattern: {matched_files}. Make sure only one Mistral "
34
+ "tokenizer is present in {tokenizer_name}.")
35
+ elif len(matched_files) == 0:
36
+ raise OSError(f"Found {len(matched_files)} files matching the "
37
+ "pattern: {matched_files}. Make sure that a Mistral "
38
+ "tokenizer is present in {tokenizer_name}.")
39
+
40
+ return matched_files[0]
41
+
42
+
43
+class MistralTokenizer:
44
+
45
+ def __init__(self, tokenizer: PublicMistralTokenizer) -> None:
46
+ self.mistral = tokenizer
47
+ self.instruct = tokenizer.instruct_tokenizer
48
+ self.tokenizer = tokenizer.instruct_tokenizer.tokenizer
49
+
50
+ self.vocab_size = len(self.tokenizer.vocab())
51
+
52
+ assert isinstance(self.tokenizer,
53
+ (Tekkenizer, SentencePieceTokenizer)), type(
54
+ self.tokenizer)
55
+ self._is_tekken = isinstance(self.tokenizer, Tekkenizer)
56
+
57
+ if self._is_tekken:
58
+ # Make sure special tokens will not raise
59
+ self.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE
60
+
61
+ # the following attributes are set to fit VLLM's design
62
+ self.is_fast = True
63
+ self.chat_template = True
64
+ self.all_special_ids: List[Any] = []
65
+ self.all_special_tokens: List[Any] = []
66
+ self.all_special_tokens_extended: List[Any] = []
67
+
68
+ @classmethod
69
+ def from_pretrained(cls,
70
+ path_or_repo_id: str,
71
+ *,
72
+ revision: Optional[str] = None) -> "MistralTokenizer":
73
+ if not Path(path_or_repo_id).exists():
74
+ assert len(path_or_repo_id.split("/")) == 2, (
75
+ "You have either provided a non-existent path: "
76
+ "{path_or_repo_id} or an invalid HF Hub repo id.")
77
+ tokenizer_file = cls._download_mistral_tokenizer_from_hf(
78
+ path_or_repo_id, revision)
79
+ elif Path(path_or_repo_id).is_dir():
80
+ tokenizer_file_name = find_tokenizer_file(
81
+ os.listdir(path_or_repo_id))
82
+ tokenizer_file = str(Path(path_or_repo_id) / tokenizer_file_name)
83
+ else:
84
+ assert Path(
85
+ path_or_repo_id).is_file(), f"Invalid path: {path_or_repo_id}"
86
+
87
+ mistral_tokenizer = PublicMistralTokenizer.from_file(tokenizer_file)
88
+ return cls(mistral_tokenizer)
89
+
90
+ @staticmethod
91
+ def _download_mistral_tokenizer_from_hf(tokenizer_name: str,
92
+ revision: Optional[str]) -> str:
93
+ api = HfApi()
94
+ repo_info = api.model_info(tokenizer_name)
95
+ files = [s.rfilename for s in repo_info.siblings]
96
+
97
+ filename = find_tokenizer_file(files)
98
+
99
+ tokenizer_file = hf_hub_download(tokenizer_name,
100
+ filename=filename,
101
+ revision=revision)
102
+ return tokenizer_file
103
+
104
+ def __call__(
105
+ self,
106
+ prompt: str,
107
+ add_special_tokens: bool = False,
108
+ truncation: bool = False,
109
+ max_length: Optional[int] = None,
110
+ ):
111
+ # Mistral Tokenizers should not add special tokens
112
+ input_ids = self.encode(prompt)
113
+
114
+ if truncation:
115
+ input_ids = input_ids[:max_length]
116
+
117
+ return Encoding(input_ids=input_ids)
118
+
119
+ def get_added_vocab(self) -> List[str]:
120
+ # Mistral tokenizers have no added vocabulary
121
+ return []
122
+
123
+ def encode(self, prompt: str) -> List[int]:
124
+ # `encode ` should only be used for prompt completion
125
+ # it should never be used for chat_completion.
126
+ # For chat completion use `apply_chat_template`
127
+ return self.tokenizer.encode(prompt, bos=True, eos=False)
128
+
129
+ def apply_chat_template(self,
130
+ conversation: List["ConversationMessage"],
131
+ tools: Optional[Dict[str, Any]] = None,
132
+ **kwargs) -> List[int]:
133
+ assert tools is None, "`tools` are not yet supported."
134
+
135
+ request = ChatCompletionRequest(
136
+ messages=conversation) # type: ignore[type-var]
137
+ encoded = self.mistral.encode_chat_completion(request)
138
+
139
+ # encode-decode to get clean prompt
140
+ return encoded.tokens
141
+
142
+ def convert_tokens_to_string(self, tokens: List[str]) -> str:
143
+ if self._is_tekken:
144
+ return "".join(tokens)
145
+ else:
146
+ return self.tokenizer.decode(tokens) # type: ignore[arg-type]
147
+
148
+ def decode(self, ids: Union[List[int], int]) -> str:
149
+ if isinstance(ids, int):
150
+ ids = [ids]
151
+ return self.tokenizer.decode(ids)
152
+
153
+ @property
154
+ def eos_token_id(self):
155
+ return self.tokenizer.eos_id
156
+
157
+ def convert_ids_to_tokens(
158
+ self,
159
+ ids: List[int],
160
+ skip_special_tokens: Optional[bool] = True) -> List[str]:
161
+ # TODO(Patrick) - potentially allow special tokens to not be skipped
162
+ assert (
163
+ skip_special_tokens
164
+ ), "Skipping special tokens is not supported for Mistral tokenizers."
165
+
166
+ assert isinstance(self.tokenizer,
167
+ (Tekkenizer, SentencePieceTokenizer)), type(
168
+ self.tokenizer)
169
+
170
+ tokens = [self.tokenizer.id_to_piece(id) for id in ids]
171
+ return tokens
172
+
173
+ def __len__(self):
174
+ return self.vocab_size