A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Ensure special tokens are properly filtered out for guided structured output with MistralTokenizer (#10363)
Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>
Guillaume Calmettes committed
Nov 15, 2024 at 15:50 UTC
691a3ec0475ba1fe4255bc975d02cc7a4392bf2c
2 files changed
+17
-6
requirements-common.txt
+2
-2
@@ -17,7 +17,7 @@ pillow # Required for image processing
17
prometheus_client >= 0.18.0
18
prometheus-fastapi-instrumentator >= 7.0.0
19
tiktoken >= 0.6.0 # Required for DBRX tokenizer
20
-lm-format-enforcer == 0.10.6
20
+lm-format-enforcer >= 0.10.9, < 0.11
21
outlines >= 0.0.43, < 0.1
22
typing_extensions >= 4.10
23
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
@@ -31,4 +31,4 @@ pyyaml
31
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
32
setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
33
einops # Required for Qwen2-VL.
34
-compressed-tensors == 0.8.0 # required for compressed-tensors
\ No newline at end of file
34
+compressed-tensors == 0.8.0 # required for compressed-tensors
vllm/transformers_utils/tokenizers/mistral.py
+15
-4
@@ -174,18 +174,29 @@ class MistralTokenizer:
174
revision=revision)
175
return tokenizer_file
176
177
- # the following attributes are set to fit VLLM's design
177
+ # the following attributes are set to fit VLLM's design and are used
178
+ # by the guided structured output backends.
179
@property
180
def all_special_tokens_extended(self) -> List[str]:
180
- return []
181
+ # tekken defines its own extended special tokens list
182
+ if hasattr(self.tokenizer, "SPECIAL_TOKENS"):
183
+ special_tokens = self.tokenizer.SPECIAL_TOKENS
184
+ else:
185
+ special_tokens = list(SpecialTokens)
186
+ return [
187
+ s.value if isinstance(s, SpecialTokens) else s
188
+ for s in special_tokens
189
+ ]
190
191
@property
192
def all_special_tokens(self) -> List[str]:
184
- return []
193
+ return self.all_special_tokens_extended
194
195
@property
196
def all_special_ids(self) -> List[int]:
188
- return []
197
+ return [
198
+ self.all_special_tokens.index(t) for t in self.all_special_tokens
199
+ ]
200
201
@property
202
def bos_token_id(self) -> int: