A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Bugfix] Fix bad words for Mistral models (#17753)
Signed-off-by: Qiong Zhou Huang <qiong@phonic.co>
Qiong Zhou Huang committed
May 7, 2025 at 23:32 UTC
39956efb3f2a3457d29725916e0ba9cbb69841a3
2 files changed
+9
-18
vllm/logits_process.py
+7
-10
@@ -4,11 +4,12 @@ from typing import Callable, Union
4
5
import torch
6
7
-from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
7
+from vllm.transformers_utils.tokenizer import AnyTokenizer
8
9
-LogitsProcessor = Union[Callable[[list[int], torch.Tensor], torch.Tensor],
10
- Callable[[list[int], list[int], torch.Tensor],
11
- torch.Tensor]]
9
+LogitsProcessor = Union[
10
+ Callable[[list[int], torch.Tensor], torch.Tensor],
11
+ Callable[[list[int], list[int], torch.Tensor], torch.Tensor],
12
+]
13
"""LogitsProcessor is a function that takes a list
14
of previously generated tokens, the logits tensor
15
for the next token and, optionally, prompt tokens as a
@@ -29,12 +30,8 @@ def get_bad_words_logits_processors(
30
prefix = " " if add_prefix_space else ""
31
prompt = prefix + bad_word.lstrip()
32
32
- if isinstance(tokenizer, MistralTokenizer):
33
- # Mistral tokenizers should not add special tokens
34
- prompt_token_ids = tokenizer.encode(text=prompt)
35
- else:
36
- prompt_token_ids = tokenizer.encode(text=prompt,
37
- add_special_tokens=False)
33
+ prompt_token_ids = tokenizer.encode(text=prompt,
34
+ add_special_tokens=False)
35
36
# If no space at the beginning
37
# or if prefix space produces a new word token
vllm/sampling_params.py
+2
-8
@@ -13,7 +13,6 @@ from typing_extensions import deprecated
13
from vllm.logger import init_logger
14
from vllm.logits_process import LogitsProcessor
15
from vllm.transformers_utils.tokenizer import AnyTokenizer
16
-from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
16
17
logger = init_logger(__name__)
18
@@ -491,13 +490,8 @@ class SamplingParams(
490
for add_prefix_space in [False, True]:
491
prefix = " " if add_prefix_space else ""
492
prompt = prefix + bad_word.lstrip()
494
-
495
- if isinstance(tokenizer, MistralTokenizer):
496
- # Mistral tokenizers should not add special tokens
497
- prompt_token_ids = tokenizer.encode(text=prompt)
498
- else:
499
- prompt_token_ids = tokenizer.encode(
500
- text=prompt, add_special_tokens=False)
493
+ prompt_token_ids = tokenizer.encode(text=prompt,
494
+ add_special_tokens=False)
495
496
# If no space at the beginning
497
# or if prefix space produces a new word token