A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Bad words sampling parameter (#9717)
Signed-off-by: Vasily Alexeev <alvasian@yandex.ru>
Vasiliy Alekseev committed
Oct 26, 2024 at 19:29 UTC
07e981fdf43bb7a7186c782a5ad6b99b36c2fc19
6 files changed
+339
-16
tests/samplers/test_no_bad_words.py
new
+185
@@ -0,0 +1,185 @@
1
+"""Make sure bad_words works.
2
+
3
+Run `pytest tests/samplers/test_no_bad_words.py`.
4
+
5
+"""
6
+from typing import List, Optional
7
+
8
+from transformers import AutoTokenizer
9
+
10
+from vllm import LLM, SamplingParams
11
+
12
+
13
+def _generate(
14
+ model: LLM,
15
+ prompt: str,
16
+ num_prompt_tokens: int,
17
+ temperature: float = 0,
18
+ bad_words: Optional[List[str]] = None,
19
+) -> List[int]:
20
+ sampling_params = SamplingParams(
21
+ temperature=temperature,
22
+ bad_words=bad_words,
23
+ )
24
+
25
+ # [([output_token_ids, ], [output_text, ]), ]
26
+ output = model.generate([prompt], sampling_params=sampling_params)
27
+
28
+ output_token_ids = output[0][0][0][num_prompt_tokens:]
29
+ # [0] first (and only) request output
30
+ # [0] token_ids (not text)
31
+ # [0] first (and only) output completion
32
+
33
+ return output_token_ids
34
+
35
+
36
+class TestOneTokenBadWord:
37
+ MODEL = "TheBloke/Llama-2-7B-fp16"
38
+
39
+ PROMPT = "Hi! How are"
40
+ TARGET_TOKEN = "you"
41
+
42
+ def setup_method(self, method):
43
+ self.tokenizer = AutoTokenizer.from_pretrained(self.MODEL,
44
+ add_prefix_space=True)
45
+
46
+ self.num_prompt_tokens = len(self._encode(self.PROMPT))
47
+ self.target_token_id = self._encode(self.TARGET_TOKEN,
48
+ add_special_tokens=False)[0]
49
+
50
+ def test_one_token_bad_word(self, vllm_runner):
51
+ with vllm_runner(self.MODEL) as llm:
52
+ output_token_ids = self._generate(llm)
53
+ assert output_token_ids[0] == self.target_token_id
54
+
55
+ output_token_ids = self._generate(llm,
56
+ bad_words=[self.TARGET_TOKEN])
57
+ assert self.target_token_id not in output_token_ids
58
+
59
+ def _generate(self,
60
+ model: LLM,
61
+ bad_words: Optional[List[str]] = None) -> List[int]:
62
+ return _generate(
63
+ model=model,
64
+ prompt=self.PROMPT,
65
+ num_prompt_tokens=self.num_prompt_tokens,
66
+ bad_words=bad_words,
67
+ )
68
+
69
+ def _encode(self,
70
+ prompt: str,
71
+ add_special_tokens: bool = True) -> List[int]:
72
+ return self.tokenizer(prompt,
73
+ add_special_tokens=add_special_tokens).input_ids
74
+
75
+
76
+class TestTwoTokenBadWord:
77
+ # Another model (with a different tokenizer behaviour)
78
+ MODEL = "openai-community/gpt2"
79
+
80
+ PROMPT = "How old are you? I am 10"
81
+ TARGET_TOKEN1 = "years"
82
+ TARGET_TOKEN2 = "old"
83
+ NEIGHBOUR_TOKEN2 = "older"
84
+
85
+ def setup_method(self, method):
86
+ self.tokenizer = AutoTokenizer.from_pretrained(self.MODEL,
87
+ add_prefix_space=True)
88
+
89
+ self.num_prompt_tokens = len(self._encode(self.PROMPT))
90
+ self.target_token_id1 = self._encode(self.TARGET_TOKEN1,
91
+ add_special_tokens=False)[0]
92
+ self.target_token_id2 = self._encode(self.TARGET_TOKEN2,
93
+ add_special_tokens=False)[0]
94
+ self.neighbour_token_id2 = self._encode(self.NEIGHBOUR_TOKEN2,
95
+ add_special_tokens=False)[0]
96
+
97
+ def test_two_token_bad_word(self, vllm_runner):
98
+ with vllm_runner(self.MODEL) as llm:
99
+ output_token_ids = self._generate(llm)
100
+ assert output_token_ids[:2] == [
101
+ self.target_token_id1, self.target_token_id2
102
+ ]
103
+
104
+ output_token_ids = self._generate(llm,
105
+ bad_words=[self.TARGET_TOKEN1])
106
+ assert self.target_token_id1 not in output_token_ids
107
+
108
+ output_token_ids = self._generate(llm,
109
+ bad_words=[self.TARGET_TOKEN2])
110
+ assert output_token_ids[0] == self.target_token_id1
111
+ assert self.target_token_id2 not in output_token_ids
112
+
113
+ output_token_ids = self._generate(
114
+ llm, bad_words=[f'{self.TARGET_TOKEN1} {self.TARGET_TOKEN2}'])
115
+ assert output_token_ids[0] == self.target_token_id1
116
+ assert output_token_ids[:2] != [
117
+ self.target_token_id1, self.target_token_id2
118
+ ]
119
+ assert not self._contains(
120
+ output_token_ids,
121
+ [self.target_token_id1, self.target_token_id2])
122
+ # Model dependent behaviour
123
+ assert output_token_ids[:2] == [
124
+ self.target_token_id1, self.neighbour_token_id2
125
+ ]
126
+
127
+ output_token_ids = self._generate(
128
+ llm,
129
+ bad_words=[
130
+ f'{self.TARGET_TOKEN1} {self.TARGET_TOKEN2}',
131
+ f'{self.TARGET_TOKEN1} {self.NEIGHBOUR_TOKEN2}'
132
+ ])
133
+ assert output_token_ids[0] == self.target_token_id1
134
+ assert output_token_ids[:2] != [
135
+ self.target_token_id1, self.target_token_id2
136
+ ]
137
+ assert not self._contains(
138
+ output_token_ids,
139
+ [self.target_token_id1, self.target_token_id2])
140
+ assert output_token_ids[:2] != [
141
+ self.target_token_id1, self.neighbour_token_id2
142
+ ]
143
+ assert not self._contains(
144
+ output_token_ids,
145
+ [self.target_token_id1, self.neighbour_token_id2])
146
+ assert ((self.target_token_id2 in output_token_ids)
147
+ or (self.neighbour_token_id2 in output_token_ids))
148
+
149
+ def _generate(self,
150
+ model: LLM,
151
+ bad_words: Optional[List[str]] = None) -> List[int]:
152
+ return _generate(
153
+ model=model,
154
+ prompt=self.PROMPT,
155
+ num_prompt_tokens=self.num_prompt_tokens,
156
+ bad_words=bad_words,
157
+ )
158
+
159
+ @staticmethod
160
+ def _contains(sequence: List[int], subsequence: List[int]) -> bool:
161
+ searched = False
162
+
163
+ for start in range(len(sequence)):
164
+ end = start + len(subsequence)
165
+ current_subsequence = sequence[start:end]
166
+
167
+ if len(current_subsequence) < len(subsequence):
168
+ continue
169
+
170
+ searched = True
171
+
172
+ assert len(current_subsequence) == len(subsequence)
173
+
174
+ if current_subsequence == subsequence:
175
+ return True
176
+
177
+ assert searched, "All subsequences did not match in length..."
178
+
179
+ return False
180
+
181
+ def _encode(self,
182
+ prompt: str,
183
+ add_special_tokens: bool = True) -> List[int]:
184
+ return self.tokenizer(prompt,
185
+ add_special_tokens=add_special_tokens).input_ids
vllm/engine/llm_engine.py
+11
-2
@@ -26,7 +26,8 @@ from vllm.engine.output_processor.interfaces import (
26
SequenceGroupOutputProcessor)
27
from vllm.engine.output_processor.stop_checker import StopChecker
28
from vllm.engine.output_processor.util import create_output_by_sequence_group
29
-from vllm.entrypoints.openai.logits_processors import get_logits_processors
29
+from vllm.entrypoints.openai.logits_processors import (
30
+ get_logits_processors as get_openai_logits_processors)
31
from vllm.executor.executor_base import ExecutorBase
32
from vllm.executor.gpu_executor import GPUExecutor
33
from vllm.executor.ray_utils import initialize_ray_cluster
@@ -34,6 +35,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
35
EncoderDecoderInputs, InputRegistry, PromptType)
36
from vllm.inputs.preprocess import InputPreprocessor
37
from vllm.logger import init_logger
38
+from vllm.logits_process import get_bad_words_logits_processors
39
from vllm.lora.request import LoRARequest
40
from vllm.model_executor.guided_decoding import (
41
get_local_guided_decoding_logits_processor)
@@ -1963,6 +1965,7 @@ class LLMEngine:
1965
logits_processors field. Returns the modified sampling params."""
1966
1967
logits_processors = []
1968
+
1969
if (guided_decoding := sampling_params.guided_decoding) is not None:
1970
1971
logger.debug(
@@ -1984,7 +1987,7 @@ class LLMEngine:
1987
if (sampling_params.logit_bias or sampling_params.allowed_token_ids):
1988
tokenizer = self.get_tokenizer(lora_request=lora_request)
1989
1987
- processors = get_logits_processors(
1990
+ processors = get_openai_logits_processors(
1991
logit_bias=sampling_params.logit_bias,
1992
allowed_token_ids=sampling_params.allowed_token_ids,
1993
tokenizer=tokenizer)
@@ -1994,6 +1997,12 @@ class LLMEngine:
1997
sampling_params.logit_bias = None
1998
sampling_params.allowed_token_ids = None
1999
2000
+ if len(sampling_params.bad_words) > 0:
2001
+ tokenizer = self.get_tokenizer(lora_request)
2002
+ processors = get_bad_words_logits_processors(
2003
+ bad_words=sampling_params.bad_words, tokenizer=tokenizer)
2004
+ logits_processors.extend(processors)
2005
+
2006
if logits_processors:
2007
if sampling_params.logits_processors is None:
2008
sampling_params.logits_processors = logits_processors
vllm/logits_process.py
new
+119
@@ -0,0 +1,119 @@
1
+from typing import Callable, List, Tuple, Union
2
+
3
+import torch
4
+
5
+from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
6
+
7
+LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],
8
+ Callable[[List[int], List[int], torch.Tensor],
9
+ torch.Tensor]]
10
+"""LogitsProcessor is a function that takes a list
11
+of previously generated tokens, the logits tensor
12
+for the next token and, optionally, prompt tokens as a
13
+first argument, and returns a modified tensor of logits
14
+to sample from."""
15
+
16
+
17
+def get_bad_words_logits_processors(
18
+ bad_words: List[str],
19
+ tokenizer: AnyTokenizer) -> List[LogitsProcessor]:
20
+ bad_words_ids: List[List[int]] = list()
21
+
22
+ for bad_word in bad_words:
23
+ # To prohibit words both at the beginning
24
+ # and in the middle of text
25
+ # (related to add_prefix_space tokenizer parameter)
26
+ for add_prefix_space in [False, True]:
27
+ prefix = " " if add_prefix_space else ""
28
+ prompt = prefix + bad_word.lstrip()
29
+
30
+ if isinstance(tokenizer, MistralTokenizer):
31
+ # Mistral tokenizers should not add special tokens
32
+ prompt_token_ids = tokenizer.encode(prompt=prompt)
33
+ else:
34
+ prompt_token_ids = tokenizer.encode(text=prompt,
35
+ add_special_tokens=False)
36
+
37
+ # If no space at the beginning
38
+ # or if prefix space produces a new word token
39
+ if (not add_prefix_space) or (
40
+ add_prefix_space
41
+ and prompt_token_ids[0] != bad_words_ids[-1][0]
42
+ and len(prompt_token_ids) == len(bad_words_ids[-1])):
43
+ bad_words_ids.append(prompt_token_ids)
44
+
45
+ return [NoBadWordsLogitsProcessor(bad_words_ids=bad_words_ids)]
46
+
47
+
48
+class NoBadWordsLogitsProcessor:
49
+ _SMALLEST_LOGIT = float("-inf")
50
+ _NEUTRAL_LOGIT = 0.0
51
+
52
+ def __init__(self, bad_words_ids: List[List[int]]):
53
+ self.bad_words_ids = bad_words_ids
54
+ self.word_bias: torch.FloatTensor = None
55
+
56
+ def __call__(
57
+ self,
58
+ past_tokens_ids: Union[List[int], Tuple[int]],
59
+ logits: torch.FloatTensor,
60
+ ) -> torch.Tensor:
61
+ if self.word_bias is None:
62
+ self._init_word_bias(logits=logits)
63
+
64
+ last_token_bias = torch.zeros_like(logits)
65
+
66
+ for bad_word_ids in self.bad_words_ids:
67
+ if len(bad_word_ids) == 1: # 1-token words already processed
68
+ continue
69
+
70
+ if len(bad_word_ids) > len(past_tokens_ids) + 1:
71
+ continue
72
+
73
+ prefix_length = len(bad_word_ids) - 1
74
+ last_token_id = bad_word_ids[-1]
75
+ actual_prefix = past_tokens_ids[-prefix_length:]
76
+ expected_prefix = bad_word_ids[:prefix_length]
77
+
78
+ assert len(actual_prefix) == len(expected_prefix)
79
+
80
+ is_match = tuple(actual_prefix) == tuple(expected_prefix)
81
+ last_token_bias[last_token_id] += (self._SMALLEST_LOGIT if is_match
82
+ else self._NEUTRAL_LOGIT)
83
+
84
+ logits = logits + self.word_bias + last_token_bias
85
+
86
+ return logits
87
+
88
+ def _init_word_bias(self, logits: torch.FloatTensor) -> None:
89
+ # Code based on NoBadWordsLogitsProcessor and SequenceBiasLogitsProcessor # noqa: E501
90
+ # from https://github.com/huggingface/transformers/blob/main/src/transformers/generation/logits_process.py
91
+
92
+ vocab_size = logits.shape[-1]
93
+
94
+ self._check_token_ids_bounds(vocab_size=vocab_size)
95
+
96
+ self.word_bias = torch.zeros((vocab_size, ),
97
+ dtype=torch.float,
98
+ device=logits.device)
99
+
100
+ for bad_word_ids in self.bad_words_ids:
101
+ if len(bad_word_ids) == 1:
102
+ bad_word_id = bad_word_ids[-1]
103
+ self.word_bias[bad_word_id] = self._SMALLEST_LOGIT
104
+
105
+ def _check_token_ids_bounds(self, vocab_size: int) -> None:
106
+ invalid_token_ids = []
107
+
108
+ for bad_word_ids in self.bad_words_ids:
109
+ for token_id in bad_word_ids:
110
+ if token_id < 0 or token_id >= vocab_size:
111
+ invalid_token_ids.append(token_id)
112
+
113
+ if len(invalid_token_ids) > 0:
114
+ raise ValueError(
115
+ f"The model vocabulary size is {vocab_size},"
116
+ f" but the following tokens"
117
+ f" were specified as bad: {invalid_token_ids}."
118
+ f" All token id values should be integers satisfying:"
119
+ f" 0 <= token_id < {vocab_size}.")
vllm/model_executor/guided_decoding/__init__.py
+2
-1
@@ -1,6 +1,7 @@
1
from typing import Optional
2
3
-from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor
3
+from vllm.logits_process import LogitsProcessor
4
+from vllm.sampling_params import GuidedDecodingParams
5
6
7
async def get_guided_decoding_logits_processor(
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+2
-1
@@ -9,7 +9,8 @@ from lmformatenforcer.integrations.vllm import (
9
build_vllm_logits_processor, build_vllm_token_enforcer_tokenizer_data)
10
from transformers import PreTrainedTokenizerBase
11
12
-from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor
12
+from vllm.logits_process import LogitsProcessor
13
+from vllm.sampling_params import GuidedDecodingParams
14
15
16
def get_local_lm_format_enforcer_guided_decoding_logits_processor(
vllm/sampling_params.py
+20
-12
@@ -3,14 +3,14 @@ import copy
3
from dataclasses import dataclass
4
from enum import Enum, IntEnum
5
from functools import cached_property
6
-from typing import Any, Callable, Dict, List, Optional, Set, Union
6
+from typing import Any, Dict, List, Optional, Set, Union
7
8
import msgspec
9
-import torch
9
from pydantic import BaseModel
10
from typing_extensions import Annotated
11
12
from vllm.logger import init_logger
13
+from vllm.logits_process import LogitsProcessor
14
15
logger = init_logger(__name__)
16
@@ -24,16 +24,6 @@ class SamplingType(IntEnum):
24
RANDOM_SEED = 2
25
26
27
-LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],
28
- Callable[[List[int], List[int], torch.Tensor],
29
- torch.Tensor]]
30
-"""LogitsProcessor is a function that takes a list
31
-of previously generated tokens, the logits tensor
32
-for the next token and, optionally, prompt tokens as a
33
-first argument, and returns a modified tensor of logits
34
-to sample from."""
35
-
36
-
27
# maybe make msgspec?
28
@dataclass
29
class GuidedDecodingParams:
@@ -139,6 +129,10 @@ class SamplingParams(
129
stop_token_ids: List of tokens that stop the generation when they are
130
generated. The returned output will contain the stop tokens unless
131
the stop tokens are special tokens.
132
+ bad_words: List of words that are not allowed to be generated.
133
+ More precisely, only the last token of a corresponding
134
+ token sequence is not allowed when the next generated token
135
+ can complete the sequence.
136
include_stop_str_in_output: Whether to include the stop strings in
137
output text. Defaults to False.
138
ignore_eos: Whether to ignore the EOS token and continue generating
@@ -186,6 +180,7 @@ class SamplingParams(
180
seed: Optional[int] = None
181
stop: Optional[Union[str, List[str]]] = None
182
stop_token_ids: Optional[List[int]] = None
183
+ bad_words: Optional[List[str]] = None
184
ignore_eos: bool = False
185
max_tokens: Optional[int] = 16
186
min_tokens: int = 0
@@ -228,6 +223,7 @@ class SamplingParams(
223
seed: Optional[int] = None,
224
stop: Optional[Union[str, List[str]]] = None,
225
stop_token_ids: Optional[List[int]] = None,
226
+ bad_words: Optional[List[str]] = None,
227
include_stop_str_in_output: bool = False,
228
ignore_eos: bool = False,
229
max_tokens: Optional[int] = 16,
@@ -267,6 +263,7 @@ class SamplingParams(
263
seed=seed,
264
stop=stop,
265
stop_token_ids=stop_token_ids,
266
+ bad_words=bad_words,
267
include_stop_str_in_output=include_stop_str_in_output,
268
ignore_eos=ignore_eos,
269
max_tokens=max_tokens,
@@ -298,26 +295,36 @@ class SamplingParams(
295
f"got n={self.n} and best_of={self.best_of}.")
296
self._real_n = self.n
297
self.n = self.best_of
298
+
299
if 0 < self.temperature < _MAX_TEMP:
300
logger.warning(
301
"temperature %s is less than %s, which may cause numerical "
302
"errors nan or inf in tensors. We have maxed it out to %s.",
303
self.temperature, _MAX_TEMP, _MAX_TEMP)
304
self.temperature = max(self.temperature, _MAX_TEMP)
305
+
306
if self.seed == -1:
307
self.seed = None
308
else:
309
self.seed = self.seed
310
+
311
if self.stop is None:
312
self.stop = []
313
elif isinstance(self.stop, str):
314
self.stop = [self.stop]
315
else:
316
self.stop = list(self.stop)
317
+
318
if self.stop_token_ids is None:
319
self.stop_token_ids = []
320
else:
321
self.stop_token_ids = list(self.stop_token_ids)
322
+
323
+ if self.bad_words is None:
324
+ self.bad_words = []
325
+ else:
326
+ self.bad_words = list(self.bad_words)
327
+
328
self.logprobs = 1 if self.logprobs is True else self.logprobs
329
self.prompt_logprobs = (1 if self.prompt_logprobs is True else
330
self.prompt_logprobs)
@@ -468,6 +475,7 @@ class SamplingParams(
475
f"seed={self.seed}, "
476
f"stop={self.stop}, "
477
f"stop_token_ids={self.stop_token_ids}, "
478
+ f"bad_words={self.bad_words}, "
479
f"include_stop_str_in_output={self.include_stop_str_in_output}, "
480
f"ignore_eos={self.ignore_eos}, "
481
f"max_tokens={self.max_tokens}, "