@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 691a3ec047
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Ensure special tokens are properly filtered out for guided structured output with MistralTokenizer (#10363)

Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>

Guillaume Calmettes committed Nov 15, 2024 at 15:50 UTC 691a3ec0475ba1fe4255bc975d02cc7a4392bf2c
2 files changed +17 -6
requirements-common.txt
+2 -2
@@ -17,7 +17,7 @@ pillow # Required for image processing
17 prometheus_client >= 0.18.0
18 prometheus-fastapi-instrumentator >= 7.0.0
19 tiktoken >= 0.6.0 # Required for DBRX tokenizer
20 -lm-format-enforcer == 0.10.6
20 +lm-format-enforcer >= 0.10.9, < 0.11
21 outlines >= 0.0.43, < 0.1
22 typing_extensions >= 4.10
23 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
@@ -31,4 +31,4 @@ pyyaml
31 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
32 setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
33 einops # Required for Qwen2-VL.
34 -compressed-tensors == 0.8.0 # required for compressed-tensors
\ No newline at end of file
34 +compressed-tensors == 0.8.0 # required for compressed-tensors
vllm/transformers_utils/tokenizers/mistral.py
+15 -4
@@ -174,18 +174,29 @@ class MistralTokenizer:
174 revision=revision)
175 return tokenizer_file
176
177 - # the following attributes are set to fit VLLM's design
177 + # the following attributes are set to fit VLLM's design and are used
178 + # by the guided structured output backends.
179 @property
180 def all_special_tokens_extended(self) -> List[str]:
180 - return []
181 + # tekken defines its own extended special tokens list
182 + if hasattr(self.tokenizer, "SPECIAL_TOKENS"):
183 + special_tokens = self.tokenizer.SPECIAL_TOKENS
184 + else:
185 + special_tokens = list(SpecialTokens)
186 + return [
187 + s.value if isinstance(s, SpecialTokens) else s
188 + for s in special_tokens
189 + ]
190
191 @property
192 def all_special_tokens(self) -> List[str]:
184 - return []
193 + return self.all_special_tokens_extended
194
195 @property
196 def all_special_ids(self) -> List[int]:
188 - return []
197 + return [
198 + self.all_special_tokens.index(t) for t in self.all_special_tokens
199 + ]
200
201 @property
202 def bos_token_id(self) -> int: