@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6fc4e6e07a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Model] Add Mistral Tokenization to improve robustness and chat encoding (#7739)

Patrick von Platen committed Aug 27, 2024 at 14:40 UTC 6fc4e6e07a55559c3744212b4d562e20d024e661
12 files changed +275 -60
docs/requirements-docs.txt
+1
@@ -11,4 +11,5 @@ pydantic >= 2.8
11 torch
12 py-cpuinfo
13 transformers
14 +mistral_common >= 1.3.4
15 openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
requirements-common.txt
+1
@@ -26,3 +26,4 @@ librosa # Required for audio processing
26 soundfile # Required for audio processing
27 gguf == 0.9.1
28 importlib_metadata
29 +mistral_common >= 1.3.4
tests/models/test_mistral.py
+3 -1
@@ -30,9 +30,11 @@ def test_models(
30 hf_outputs = hf_model.generate_greedy_logprobs_limit(
31 example_prompts, max_tokens, num_logprobs)
32
33 - with vllm_runner(model, dtype=dtype) as vllm_model:
33 + with vllm_runner(model, dtype=dtype,
34 + tokenizer_mode="mistral") as vllm_model:
35 vllm_outputs = vllm_model.generate_greedy_logprobs(
36 example_prompts, max_tokens, num_logprobs)
37 +
38 check_logprobs_close(
39 outputs_0_lst=hf_outputs,
40 outputs_1_lst=vllm_outputs,
vllm/config.py
+4 -3
@@ -61,7 +61,8 @@ class ModelConfig:
61 output when `served_model_name` is not specified.
62 tokenizer: Name or path of the huggingface tokenizer to use.
63 tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
64 - available, and "slow" will always use the slow tokenizer.
64 + available, "slow" will always use the slow tokenizer, and
65 + "mistral" will always use the tokenizer from `mistral_common`.
66 trust_remote_code: Trust remote code (e.g., from HuggingFace) when
67 downloading the model and tokenizer.
68 dtype: Data type for model weights and activations. The "auto" option
@@ -246,10 +247,10 @@ class ModelConfig:
247
248 def _verify_tokenizer_mode(self) -> None:
249 tokenizer_mode = self.tokenizer_mode.lower()
249 - if tokenizer_mode not in ["auto", "slow"]:
250 + if tokenizer_mode not in ["auto", "slow", "mistral"]:
251 raise ValueError(
252 f"Unknown tokenizer mode: {self.tokenizer_mode}. Must be "
252 - "either 'auto' or 'slow'.")
253 + "either 'auto', 'slow' or 'mistral'.")
254 self.tokenizer_mode = tokenizer_mode
255
256 def _verify_embedding_mode(self) -> None:
vllm/engine/arg_utils.py
+3 -2
@@ -198,10 +198,11 @@ class EngineArgs:
198 '--tokenizer-mode',
199 type=str,
200 default=EngineArgs.tokenizer_mode,
201 - choices=['auto', 'slow'],
201 + choices=['auto', 'slow', 'mistral'],
202 help='The tokenizer mode.\n\n* "auto" will use the '
203 'fast tokenizer if available.\n* "slow" will '
204 - 'always use the slow tokenizer.')
204 + 'always use the slow tokenizer. \n* '
205 + '"mistral" will always use the `mistral_common` tokenizer.')
206 parser.add_argument('--trust-remote-code',
207 action='store_true',
208 help='Trust remote code from huggingface.')
vllm/entrypoints/chat_utils.py
+1 -3
@@ -267,7 +267,7 @@ def apply_chat_template(
267 *,
268 tokenize: bool = False, # Different from HF's default
269 **kwargs: Any,
270 -) -> str:
270 +) -> Union[str, List[int]]:
271 if chat_template is None and tokenizer.chat_template is None:
272 raise ValueError(
273 "As of transformers v4.44, default chat template is no longer "
@@ -280,6 +280,4 @@ def apply_chat_template(
280 tokenize=tokenize,
281 **kwargs,
282 )
283 - assert isinstance(prompt, str)
284 -
283 return prompt
vllm/entrypoints/llm.py
+9 -3
@@ -390,15 +390,21 @@ class LLM:
390 conversations, _ = parse_chat_messages(messages, model_config,
391 tokenizer)
392
393 - prompts = apply_chat_template(
393 + prompt = apply_chat_template(
394 tokenizer,
395 conversations,
396 chat_template=chat_template,
397 add_generation_prompt=add_generation_prompt)
398
399 + inputs: PromptInputs
400 + if isinstance(prompt, list) and isinstance(prompt[0], int):
401 + inputs = TokensPrompt(prompt_token_ids=prompt)
402 + else:
403 + inputs = TextPrompt(prompt=prompt)
404 +
405 return self.generate(
400 - prompts,
401 - sampling_params,
406 + inputs,
407 + sampling_params=sampling_params,
408 use_tqdm=use_tqdm,
409 lora_request=lora_request,
410 )
vllm/entrypoints/openai/serving_chat.py
+18 -8
@@ -22,7 +22,8 @@ from vllm.entrypoints.openai.protocol import (
22 FunctionCall, ToolCall, UsageInfo)
23 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
24 OpenAIServing,
25 - PromptAdapterPath)
25 + PromptAdapterPath,
26 + TextTokensPrompt)
27 from vllm.inputs import TokensPrompt
28 from vllm.logger import init_logger
29 from vllm.multimodal import MultiModalDataDict
@@ -130,13 +131,22 @@ class OpenAIServingChat(OpenAIServing):
131 guided_decode_logits_processor = (
132 await self._guided_decode_logits_processor(request, tokenizer))
133
133 - prompt_inputs = self._tokenize_prompt_input(
134 - request,
135 - tokenizer,
136 - prompt,
137 - truncate_prompt_tokens=request.truncate_prompt_tokens,
138 - add_special_tokens=request.add_special_tokens,
139 - )
134 + if isinstance(prompt, str):
135 + prompt_inputs = self._tokenize_prompt_input(
136 + request,
137 + tokenizer,
138 + prompt,
139 + truncate_prompt_tokens=request.truncate_prompt_tokens,
140 + add_special_tokens=request.add_special_tokens,
141 + )
142 + else:
143 + assert isinstance(prompt, list) and isinstance(
144 + prompt[0], int
145 + ), "Prompt has to be either a string or a list of token ids"
146 + prompt_inputs = TextTokensPrompt(
147 + prompt=tokenizer.decode(prompt), prompt_token_ids=prompt)
148 +
149 + assert prompt_inputs is not None
150
151 sampling_params = request.to_sampling_params(
152 tokenizer,
vllm/transformers_utils/detokenizer.py
+1 -1
@@ -230,7 +230,7 @@ def convert_prompt_ids_to_tokens(
230 prefix_offset = max(
231 read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
232 # This is required to guard against out-of-vocab prompt token ids
233 - _replace_none_with_empty(new_tokens)
233 + _replace_none_with_empty(new_tokens) # type: ignore[arg-type]
234 return new_tokens, prefix_offset, read_offset
235
236
vllm/transformers_utils/tokenizer.py
+58 -36
@@ -1,4 +1,5 @@
1 import os
2 +import warnings
3 from pathlib import Path
4 from typing import Optional, Union
5
@@ -9,12 +10,14 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
10 from vllm.envs import VLLM_USE_MODELSCOPE
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
12 -from vllm.transformers_utils.tokenizers import BaichuanTokenizer
13 +from vllm.transformers_utils.tokenizers import (BaichuanTokenizer,
14 + MistralTokenizer)
15 from vllm.utils import make_async
16
17 logger = init_logger(__name__)
18
17 -AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
19 +AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
20 + MistralTokenizer]
21
22
23 def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
@@ -99,45 +102,64 @@ def get_tokenizer(
102 kwargs["gguf_file"] = Path(tokenizer_name).name
103 tokenizer_name = Path(tokenizer_name).parent
104
102 - try:
103 - tokenizer = AutoTokenizer.from_pretrained(
104 - tokenizer_name,
105 - *args,
106 - trust_remote_code=trust_remote_code,
107 - revision=revision,
108 - **kwargs)
109 - except ValueError as e:
110 - # If the error pertains to the tokenizer class not existing or not
111 - # currently being imported, suggest using the --trust-remote-code flag.
112 - if (not trust_remote_code and
113 - ("does not exist or is not currently imported." in str(e)
114 - or "requires you to execute the tokenizer file" in str(e))):
115 - err_msg = (
116 - "Failed to load the tokenizer. If the tokenizer is a custom "
117 - "tokenizer not yet available in the HuggingFace transformers "
118 - "library, consider setting `trust_remote_code=True` in LLM "
119 - "or using the `--trust-remote-code` flag in the CLI.")
120 - raise RuntimeError(err_msg) from e
121 - else:
122 - raise e
123 - except AttributeError as e:
124 - if "BaichuanTokenizer" in str(e):
125 - # This is for the error "'BaichuanTokenizer' object has no
126 - # attribute 'sp_model'".
127 - tokenizer = BaichuanTokenizer.from_pretrained(
105 + # if tokenizer is from official mistral org
106 + is_from_mistral_org = str(tokenizer_name).split("/")[0] == "mistralai"
107 + if is_from_mistral_org and tokenizer_mode != "mistral":
108 + warnings.warn(
109 + 'It is strongly recommended to run mistral models with '
110 + '`--tokenizer_mode "mistral"` to ensure correct '
111 + 'encoding and decoding.',
112 + FutureWarning,
113 + stacklevel=2)
114 +
115 + if tokenizer_mode == "mistral":
116 + tokenizer = MistralTokenizer.from_pretrained(str(tokenizer_name),
117 + revision=revision)
118 + else:
119 + try:
120 + tokenizer = AutoTokenizer.from_pretrained(
121 tokenizer_name,
122 *args,
123 trust_remote_code=trust_remote_code,
124 revision=revision,
132 - **kwargs)
133 - else:
134 - raise e
125 + **kwargs,
126 + )
127 + except ValueError as e:
128 + # If the error pertains to the tokenizer class not existing or not
129 + # currently being imported,
130 + # suggest using the --trust-remote-code flag.
131 + if not trust_remote_code and (
132 + "does not exist or is not currently imported." in str(e)
133 + or "requires you to execute the tokenizer file" in str(e)):
134 + err_msg = ("Failed to load the tokenizer. If the tokenizer "
135 + "is a custom tokenizer not yet available in the "
136 + "HuggingFace transformers library, consider "
137 + "setting `trust_remote_code=True` in LLM or using "
138 + "the `--trust-remote-code` flag in the CLI.")
139 + raise RuntimeError(err_msg) from e
140 + else:
141 + raise e
142 + except AttributeError as e:
143 + if "BaichuanTokenizer" in str(e):
144 + # This is for the error "'BaichuanTokenizer' object has no
145 + # attribute 'sp_model'".
146 + tokenizer = BaichuanTokenizer.from_pretrained(
147 + tokenizer_name,
148 + *args,
149 + trust_remote_code=trust_remote_code,
150 + revision=revision,
151 + **kwargs,
152 + )
153 + else:
154 + raise e
155 +
156 + if not isinstance(tokenizer, PreTrainedTokenizerFast):
157 + logger.warning(
158 + "Using a slow tokenizer. This might cause a significant "
159 + "slowdown. Consider using a fast tokenizer instead.")
160 + tokenizer = get_cached_tokenizer(tokenizer)
161
136 - if not isinstance(tokenizer, PreTrainedTokenizerFast):
137 - logger.warning(
138 - "Using a slow tokenizer. This might cause a significant "
139 - "slowdown. Consider using a fast tokenizer instead.")
140 - return get_cached_tokenizer(tokenizer)
162 + return tokenizer
163
164
165 def get_lora_tokenizer(lora_request: LoRARequest, *args,
vllm/transformers_utils/tokenizers/__init__.py
+2 -3
@@ -1,5 +1,4 @@
1 from vllm.transformers_utils.tokenizers.baichuan import BaichuanTokenizer
2 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
3
3 -__all__ = [
4 - "BaichuanTokenizer",
5 -]
4 +__all__ = ["BaichuanTokenizer", "MistralTokenizer"]
vllm/transformers_utils/tokenizers/mistral.py new
+174
@@ -0,0 +1,174 @@
1 +import os
2 +import re
3 +from dataclasses import dataclass
4 +from pathlib import Path
5 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
6 +
7 +from huggingface_hub import HfApi, hf_hub_download
8 +# yapf: disable
9 +from mistral_common.tokens.tokenizers.mistral import ChatCompletionRequest
10 +from mistral_common.tokens.tokenizers.mistral import (
11 + MistralTokenizer as PublicMistralTokenizer)
12 +# yapf: enable
13 +from mistral_common.tokens.tokenizers.sentencepiece import (
14 + SentencePieceTokenizer)
15 +from mistral_common.tokens.tokenizers.tekken import (SpecialTokenPolicy,
16 + Tekkenizer)
17 +
18 +if TYPE_CHECKING:
19 + from vllm.entrypoints.chat_utils import ConversationMessage
20 +
21 +
22 +@dataclass
23 +class Encoding:
24 + input_ids: List[int]
25 +
26 +
27 +def find_tokenizer_file(files: List[str]):
28 + file_pattern = re.compile(r"^tokenizer\.model\.v.*$|^tekken\.json$")
29 +
30 + matched_files = [file for file in files if file_pattern.match(file)]
31 + if len(matched_files) > 1:
32 + raise OSError(f"Found {len(matched_files)} files matching the "
33 + "pattern: {matched_files}. Make sure only one Mistral "
34 + "tokenizer is present in {tokenizer_name}.")
35 + elif len(matched_files) == 0:
36 + raise OSError(f"Found {len(matched_files)} files matching the "
37 + "pattern: {matched_files}. Make sure that a Mistral "
38 + "tokenizer is present in {tokenizer_name}.")
39 +
40 + return matched_files[0]
41 +
42 +
43 +class MistralTokenizer:
44 +
45 + def __init__(self, tokenizer: PublicMistralTokenizer) -> None:
46 + self.mistral = tokenizer
47 + self.instruct = tokenizer.instruct_tokenizer
48 + self.tokenizer = tokenizer.instruct_tokenizer.tokenizer
49 +
50 + self.vocab_size = len(self.tokenizer.vocab())
51 +
52 + assert isinstance(self.tokenizer,
53 + (Tekkenizer, SentencePieceTokenizer)), type(
54 + self.tokenizer)
55 + self._is_tekken = isinstance(self.tokenizer, Tekkenizer)
56 +
57 + if self._is_tekken:
58 + # Make sure special tokens will not raise
59 + self.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE
60 +
61 + # the following attributes are set to fit VLLM's design
62 + self.is_fast = True
63 + self.chat_template = True
64 + self.all_special_ids: List[Any] = []
65 + self.all_special_tokens: List[Any] = []
66 + self.all_special_tokens_extended: List[Any] = []
67 +
68 + @classmethod
69 + def from_pretrained(cls,
70 + path_or_repo_id: str,
71 + *,
72 + revision: Optional[str] = None) -> "MistralTokenizer":
73 + if not Path(path_or_repo_id).exists():
74 + assert len(path_or_repo_id.split("/")) == 2, (
75 + "You have either provided a non-existent path: "
76 + "{path_or_repo_id} or an invalid HF Hub repo id.")
77 + tokenizer_file = cls._download_mistral_tokenizer_from_hf(
78 + path_or_repo_id, revision)
79 + elif Path(path_or_repo_id).is_dir():
80 + tokenizer_file_name = find_tokenizer_file(
81 + os.listdir(path_or_repo_id))
82 + tokenizer_file = str(Path(path_or_repo_id) / tokenizer_file_name)
83 + else:
84 + assert Path(
85 + path_or_repo_id).is_file(), f"Invalid path: {path_or_repo_id}"
86 +
87 + mistral_tokenizer = PublicMistralTokenizer.from_file(tokenizer_file)
88 + return cls(mistral_tokenizer)
89 +
90 + @staticmethod
91 + def _download_mistral_tokenizer_from_hf(tokenizer_name: str,
92 + revision: Optional[str]) -> str:
93 + api = HfApi()
94 + repo_info = api.model_info(tokenizer_name)
95 + files = [s.rfilename for s in repo_info.siblings]
96 +
97 + filename = find_tokenizer_file(files)
98 +
99 + tokenizer_file = hf_hub_download(tokenizer_name,
100 + filename=filename,
101 + revision=revision)
102 + return tokenizer_file
103 +
104 + def __call__(
105 + self,
106 + prompt: str,
107 + add_special_tokens: bool = False,
108 + truncation: bool = False,
109 + max_length: Optional[int] = None,
110 + ):
111 + # Mistral Tokenizers should not add special tokens
112 + input_ids = self.encode(prompt)
113 +
114 + if truncation:
115 + input_ids = input_ids[:max_length]
116 +
117 + return Encoding(input_ids=input_ids)
118 +
119 + def get_added_vocab(self) -> List[str]:
120 + # Mistral tokenizers have no added vocabulary
121 + return []
122 +
123 + def encode(self, prompt: str) -> List[int]:
124 + # `encode ` should only be used for prompt completion
125 + # it should never be used for chat_completion.
126 + # For chat completion use `apply_chat_template`
127 + return self.tokenizer.encode(prompt, bos=True, eos=False)
128 +
129 + def apply_chat_template(self,
130 + conversation: List["ConversationMessage"],
131 + tools: Optional[Dict[str, Any]] = None,
132 + **kwargs) -> List[int]:
133 + assert tools is None, "`tools` are not yet supported."
134 +
135 + request = ChatCompletionRequest(
136 + messages=conversation) # type: ignore[type-var]
137 + encoded = self.mistral.encode_chat_completion(request)
138 +
139 + # encode-decode to get clean prompt
140 + return encoded.tokens
141 +
142 + def convert_tokens_to_string(self, tokens: List[str]) -> str:
143 + if self._is_tekken:
144 + return "".join(tokens)
145 + else:
146 + return self.tokenizer.decode(tokens) # type: ignore[arg-type]
147 +
148 + def decode(self, ids: Union[List[int], int]) -> str:
149 + if isinstance(ids, int):
150 + ids = [ids]
151 + return self.tokenizer.decode(ids)
152 +
153 + @property
154 + def eos_token_id(self):
155 + return self.tokenizer.eos_id
156 +
157 + def convert_ids_to_tokens(
158 + self,
159 + ids: List[int],
160 + skip_special_tokens: Optional[bool] = True) -> List[str]:
161 + # TODO(Patrick) - potentially allow special tokens to not be skipped
162 + assert (
163 + skip_special_tokens
164 + ), "Skipping special tokens is not supported for Mistral tokenizers."
165 +
166 + assert isinstance(self.tokenizer,
167 + (Tekkenizer, SentencePieceTokenizer)), type(
168 + self.tokenizer)
169 +
170 + tokens = [self.tokenizer.id_to_piece(id) for id in ids]
171 + return tokens
172 +
173 + def __len__(self):
174 + return self.vocab_size