@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 07e981fdf4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend] Bad words sampling parameter (#9717)

Signed-off-by: Vasily Alexeev <alvasian@yandex.ru>

Vasiliy Alekseev committed Oct 26, 2024 at 19:29 UTC 07e981fdf43bb7a7186c782a5ad6b99b36c2fc19
6 files changed +339 -16
tests/samplers/test_no_bad_words.py new
+185
@@ -0,0 +1,185 @@
1 +"""Make sure bad_words works.
2 +
3 +Run `pytest tests/samplers/test_no_bad_words.py`.
4 +
5 +"""
6 +from typing import List, Optional
7 +
8 +from transformers import AutoTokenizer
9 +
10 +from vllm import LLM, SamplingParams
11 +
12 +
13 +def _generate(
14 + model: LLM,
15 + prompt: str,
16 + num_prompt_tokens: int,
17 + temperature: float = 0,
18 + bad_words: Optional[List[str]] = None,
19 +) -> List[int]:
20 + sampling_params = SamplingParams(
21 + temperature=temperature,
22 + bad_words=bad_words,
23 + )
24 +
25 + # [([output_token_ids, ], [output_text, ]), ]
26 + output = model.generate([prompt], sampling_params=sampling_params)
27 +
28 + output_token_ids = output[0][0][0][num_prompt_tokens:]
29 + # [0] first (and only) request output
30 + # [0] token_ids (not text)
31 + # [0] first (and only) output completion
32 +
33 + return output_token_ids
34 +
35 +
36 +class TestOneTokenBadWord:
37 + MODEL = "TheBloke/Llama-2-7B-fp16"
38 +
39 + PROMPT = "Hi! How are"
40 + TARGET_TOKEN = "you"
41 +
42 + def setup_method(self, method):
43 + self.tokenizer = AutoTokenizer.from_pretrained(self.MODEL,
44 + add_prefix_space=True)
45 +
46 + self.num_prompt_tokens = len(self._encode(self.PROMPT))
47 + self.target_token_id = self._encode(self.TARGET_TOKEN,
48 + add_special_tokens=False)[0]
49 +
50 + def test_one_token_bad_word(self, vllm_runner):
51 + with vllm_runner(self.MODEL) as llm:
52 + output_token_ids = self._generate(llm)
53 + assert output_token_ids[0] == self.target_token_id
54 +
55 + output_token_ids = self._generate(llm,
56 + bad_words=[self.TARGET_TOKEN])
57 + assert self.target_token_id not in output_token_ids
58 +
59 + def _generate(self,
60 + model: LLM,
61 + bad_words: Optional[List[str]] = None) -> List[int]:
62 + return _generate(
63 + model=model,
64 + prompt=self.PROMPT,
65 + num_prompt_tokens=self.num_prompt_tokens,
66 + bad_words=bad_words,
67 + )
68 +
69 + def _encode(self,
70 + prompt: str,
71 + add_special_tokens: bool = True) -> List[int]:
72 + return self.tokenizer(prompt,
73 + add_special_tokens=add_special_tokens).input_ids
74 +
75 +
76 +class TestTwoTokenBadWord:
77 + # Another model (with a different tokenizer behaviour)
78 + MODEL = "openai-community/gpt2"
79 +
80 + PROMPT = "How old are you? I am 10"
81 + TARGET_TOKEN1 = "years"
82 + TARGET_TOKEN2 = "old"
83 + NEIGHBOUR_TOKEN2 = "older"
84 +
85 + def setup_method(self, method):
86 + self.tokenizer = AutoTokenizer.from_pretrained(self.MODEL,
87 + add_prefix_space=True)
88 +
89 + self.num_prompt_tokens = len(self._encode(self.PROMPT))
90 + self.target_token_id1 = self._encode(self.TARGET_TOKEN1,
91 + add_special_tokens=False)[0]
92 + self.target_token_id2 = self._encode(self.TARGET_TOKEN2,
93 + add_special_tokens=False)[0]
94 + self.neighbour_token_id2 = self._encode(self.NEIGHBOUR_TOKEN2,
95 + add_special_tokens=False)[0]
96 +
97 + def test_two_token_bad_word(self, vllm_runner):
98 + with vllm_runner(self.MODEL) as llm:
99 + output_token_ids = self._generate(llm)
100 + assert output_token_ids[:2] == [
101 + self.target_token_id1, self.target_token_id2
102 + ]
103 +
104 + output_token_ids = self._generate(llm,
105 + bad_words=[self.TARGET_TOKEN1])
106 + assert self.target_token_id1 not in output_token_ids
107 +
108 + output_token_ids = self._generate(llm,
109 + bad_words=[self.TARGET_TOKEN2])
110 + assert output_token_ids[0] == self.target_token_id1
111 + assert self.target_token_id2 not in output_token_ids
112 +
113 + output_token_ids = self._generate(
114 + llm, bad_words=[f'{self.TARGET_TOKEN1} {self.TARGET_TOKEN2}'])
115 + assert output_token_ids[0] == self.target_token_id1
116 + assert output_token_ids[:2] != [
117 + self.target_token_id1, self.target_token_id2
118 + ]
119 + assert not self._contains(
120 + output_token_ids,
121 + [self.target_token_id1, self.target_token_id2])
122 + # Model dependent behaviour
123 + assert output_token_ids[:2] == [
124 + self.target_token_id1, self.neighbour_token_id2
125 + ]
126 +
127 + output_token_ids = self._generate(
128 + llm,
129 + bad_words=[
130 + f'{self.TARGET_TOKEN1} {self.TARGET_TOKEN2}',
131 + f'{self.TARGET_TOKEN1} {self.NEIGHBOUR_TOKEN2}'
132 + ])
133 + assert output_token_ids[0] == self.target_token_id1
134 + assert output_token_ids[:2] != [
135 + self.target_token_id1, self.target_token_id2
136 + ]
137 + assert not self._contains(
138 + output_token_ids,
139 + [self.target_token_id1, self.target_token_id2])
140 + assert output_token_ids[:2] != [
141 + self.target_token_id1, self.neighbour_token_id2
142 + ]
143 + assert not self._contains(
144 + output_token_ids,
145 + [self.target_token_id1, self.neighbour_token_id2])
146 + assert ((self.target_token_id2 in output_token_ids)
147 + or (self.neighbour_token_id2 in output_token_ids))
148 +
149 + def _generate(self,
150 + model: LLM,
151 + bad_words: Optional[List[str]] = None) -> List[int]:
152 + return _generate(
153 + model=model,
154 + prompt=self.PROMPT,
155 + num_prompt_tokens=self.num_prompt_tokens,
156 + bad_words=bad_words,
157 + )
158 +
159 + @staticmethod
160 + def _contains(sequence: List[int], subsequence: List[int]) -> bool:
161 + searched = False
162 +
163 + for start in range(len(sequence)):
164 + end = start + len(subsequence)
165 + current_subsequence = sequence[start:end]
166 +
167 + if len(current_subsequence) < len(subsequence):
168 + continue
169 +
170 + searched = True
171 +
172 + assert len(current_subsequence) == len(subsequence)
173 +
174 + if current_subsequence == subsequence:
175 + return True
176 +
177 + assert searched, "All subsequences did not match in length..."
178 +
179 + return False
180 +
181 + def _encode(self,
182 + prompt: str,
183 + add_special_tokens: bool = True) -> List[int]:
184 + return self.tokenizer(prompt,
185 + add_special_tokens=add_special_tokens).input_ids
vllm/engine/llm_engine.py
+11 -2
@@ -26,7 +26,8 @@ from vllm.engine.output_processor.interfaces import (
26 SequenceGroupOutputProcessor)
27 from vllm.engine.output_processor.stop_checker import StopChecker
28 from vllm.engine.output_processor.util import create_output_by_sequence_group
29 -from vllm.entrypoints.openai.logits_processors import get_logits_processors
29 +from vllm.entrypoints.openai.logits_processors import (
30 + get_logits_processors as get_openai_logits_processors)
31 from vllm.executor.executor_base import ExecutorBase
32 from vllm.executor.gpu_executor import GPUExecutor
33 from vllm.executor.ray_utils import initialize_ray_cluster
@@ -34,6 +35,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
35 EncoderDecoderInputs, InputRegistry, PromptType)
36 from vllm.inputs.preprocess import InputPreprocessor
37 from vllm.logger import init_logger
38 +from vllm.logits_process import get_bad_words_logits_processors
39 from vllm.lora.request import LoRARequest
40 from vllm.model_executor.guided_decoding import (
41 get_local_guided_decoding_logits_processor)
@@ -1963,6 +1965,7 @@ class LLMEngine:
1965 logits_processors field. Returns the modified sampling params."""
1966
1967 logits_processors = []
1968 +
1969 if (guided_decoding := sampling_params.guided_decoding) is not None:
1970
1971 logger.debug(
@@ -1984,7 +1987,7 @@ class LLMEngine:
1987 if (sampling_params.logit_bias or sampling_params.allowed_token_ids):
1988 tokenizer = self.get_tokenizer(lora_request=lora_request)
1989
1987 - processors = get_logits_processors(
1990 + processors = get_openai_logits_processors(
1991 logit_bias=sampling_params.logit_bias,
1992 allowed_token_ids=sampling_params.allowed_token_ids,
1993 tokenizer=tokenizer)
@@ -1994,6 +1997,12 @@ class LLMEngine:
1997 sampling_params.logit_bias = None
1998 sampling_params.allowed_token_ids = None
1999
2000 + if len(sampling_params.bad_words) > 0:
2001 + tokenizer = self.get_tokenizer(lora_request)
2002 + processors = get_bad_words_logits_processors(
2003 + bad_words=sampling_params.bad_words, tokenizer=tokenizer)
2004 + logits_processors.extend(processors)
2005 +
2006 if logits_processors:
2007 if sampling_params.logits_processors is None:
2008 sampling_params.logits_processors = logits_processors
vllm/logits_process.py new
+119
@@ -0,0 +1,119 @@
1 +from typing import Callable, List, Tuple, Union
2 +
3 +import torch
4 +
5 +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
6 +
7 +LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],
8 + Callable[[List[int], List[int], torch.Tensor],
9 + torch.Tensor]]
10 +"""LogitsProcessor is a function that takes a list
11 +of previously generated tokens, the logits tensor
12 +for the next token and, optionally, prompt tokens as a
13 +first argument, and returns a modified tensor of logits
14 +to sample from."""
15 +
16 +
17 +def get_bad_words_logits_processors(
18 + bad_words: List[str],
19 + tokenizer: AnyTokenizer) -> List[LogitsProcessor]:
20 + bad_words_ids: List[List[int]] = list()
21 +
22 + for bad_word in bad_words:
23 + # To prohibit words both at the beginning
24 + # and in the middle of text
25 + # (related to add_prefix_space tokenizer parameter)
26 + for add_prefix_space in [False, True]:
27 + prefix = " " if add_prefix_space else ""
28 + prompt = prefix + bad_word.lstrip()
29 +
30 + if isinstance(tokenizer, MistralTokenizer):
31 + # Mistral tokenizers should not add special tokens
32 + prompt_token_ids = tokenizer.encode(prompt=prompt)
33 + else:
34 + prompt_token_ids = tokenizer.encode(text=prompt,
35 + add_special_tokens=False)
36 +
37 + # If no space at the beginning
38 + # or if prefix space produces a new word token
39 + if (not add_prefix_space) or (
40 + add_prefix_space
41 + and prompt_token_ids[0] != bad_words_ids[-1][0]
42 + and len(prompt_token_ids) == len(bad_words_ids[-1])):
43 + bad_words_ids.append(prompt_token_ids)
44 +
45 + return [NoBadWordsLogitsProcessor(bad_words_ids=bad_words_ids)]
46 +
47 +
48 +class NoBadWordsLogitsProcessor:
49 + _SMALLEST_LOGIT = float("-inf")
50 + _NEUTRAL_LOGIT = 0.0
51 +
52 + def __init__(self, bad_words_ids: List[List[int]]):
53 + self.bad_words_ids = bad_words_ids
54 + self.word_bias: torch.FloatTensor = None
55 +
56 + def __call__(
57 + self,
58 + past_tokens_ids: Union[List[int], Tuple[int]],
59 + logits: torch.FloatTensor,
60 + ) -> torch.Tensor:
61 + if self.word_bias is None:
62 + self._init_word_bias(logits=logits)
63 +
64 + last_token_bias = torch.zeros_like(logits)
65 +
66 + for bad_word_ids in self.bad_words_ids:
67 + if len(bad_word_ids) == 1: # 1-token words already processed
68 + continue
69 +
70 + if len(bad_word_ids) > len(past_tokens_ids) + 1:
71 + continue
72 +
73 + prefix_length = len(bad_word_ids) - 1
74 + last_token_id = bad_word_ids[-1]
75 + actual_prefix = past_tokens_ids[-prefix_length:]
76 + expected_prefix = bad_word_ids[:prefix_length]
77 +
78 + assert len(actual_prefix) == len(expected_prefix)
79 +
80 + is_match = tuple(actual_prefix) == tuple(expected_prefix)
81 + last_token_bias[last_token_id] += (self._SMALLEST_LOGIT if is_match
82 + else self._NEUTRAL_LOGIT)
83 +
84 + logits = logits + self.word_bias + last_token_bias
85 +
86 + return logits
87 +
88 + def _init_word_bias(self, logits: torch.FloatTensor) -> None:
89 + # Code based on NoBadWordsLogitsProcessor and SequenceBiasLogitsProcessor # noqa: E501
90 + # from https://github.com/huggingface/transformers/blob/main/src/transformers/generation/logits_process.py
91 +
92 + vocab_size = logits.shape[-1]
93 +
94 + self._check_token_ids_bounds(vocab_size=vocab_size)
95 +
96 + self.word_bias = torch.zeros((vocab_size, ),
97 + dtype=torch.float,
98 + device=logits.device)
99 +
100 + for bad_word_ids in self.bad_words_ids:
101 + if len(bad_word_ids) == 1:
102 + bad_word_id = bad_word_ids[-1]
103 + self.word_bias[bad_word_id] = self._SMALLEST_LOGIT
104 +
105 + def _check_token_ids_bounds(self, vocab_size: int) -> None:
106 + invalid_token_ids = []
107 +
108 + for bad_word_ids in self.bad_words_ids:
109 + for token_id in bad_word_ids:
110 + if token_id < 0 or token_id >= vocab_size:
111 + invalid_token_ids.append(token_id)
112 +
113 + if len(invalid_token_ids) > 0:
114 + raise ValueError(
115 + f"The model vocabulary size is {vocab_size},"
116 + f" but the following tokens"
117 + f" were specified as bad: {invalid_token_ids}."
118 + f" All token id values should be integers satisfying:"
119 + f" 0 <= token_id < {vocab_size}.")
vllm/model_executor/guided_decoding/__init__.py
+2 -1
@@ -1,6 +1,7 @@
1 from typing import Optional
2
3 -from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor
3 +from vllm.logits_process import LogitsProcessor
4 +from vllm.sampling_params import GuidedDecodingParams
5
6
7 async def get_guided_decoding_logits_processor(
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+2 -1
@@ -9,7 +9,8 @@ from lmformatenforcer.integrations.vllm import (
9 build_vllm_logits_processor, build_vllm_token_enforcer_tokenizer_data)
10 from transformers import PreTrainedTokenizerBase
11
12 -from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor
12 +from vllm.logits_process import LogitsProcessor
13 +from vllm.sampling_params import GuidedDecodingParams
14
15
16 def get_local_lm_format_enforcer_guided_decoding_logits_processor(
vllm/sampling_params.py
+20 -12
@@ -3,14 +3,14 @@ import copy
3 from dataclasses import dataclass
4 from enum import Enum, IntEnum
5 from functools import cached_property
6 -from typing import Any, Callable, Dict, List, Optional, Set, Union
6 +from typing import Any, Dict, List, Optional, Set, Union
7
8 import msgspec
9 -import torch
9 from pydantic import BaseModel
10 from typing_extensions import Annotated
11
12 from vllm.logger import init_logger
13 +from vllm.logits_process import LogitsProcessor
14
15 logger = init_logger(__name__)
16
@@ -24,16 +24,6 @@ class SamplingType(IntEnum):
24 RANDOM_SEED = 2
25
26
27 -LogitsProcessor = Union[Callable[[List[int], torch.Tensor], torch.Tensor],
28 - Callable[[List[int], List[int], torch.Tensor],
29 - torch.Tensor]]
30 -"""LogitsProcessor is a function that takes a list
31 -of previously generated tokens, the logits tensor
32 -for the next token and, optionally, prompt tokens as a
33 -first argument, and returns a modified tensor of logits
34 -to sample from."""
35 -
36 -
27 # maybe make msgspec?
28 @dataclass
29 class GuidedDecodingParams:
@@ -139,6 +129,10 @@ class SamplingParams(
129 stop_token_ids: List of tokens that stop the generation when they are
130 generated. The returned output will contain the stop tokens unless
131 the stop tokens are special tokens.
132 + bad_words: List of words that are not allowed to be generated.
133 + More precisely, only the last token of a corresponding
134 + token sequence is not allowed when the next generated token
135 + can complete the sequence.
136 include_stop_str_in_output: Whether to include the stop strings in
137 output text. Defaults to False.
138 ignore_eos: Whether to ignore the EOS token and continue generating
@@ -186,6 +180,7 @@ class SamplingParams(
180 seed: Optional[int] = None
181 stop: Optional[Union[str, List[str]]] = None
182 stop_token_ids: Optional[List[int]] = None
183 + bad_words: Optional[List[str]] = None
184 ignore_eos: bool = False
185 max_tokens: Optional[int] = 16
186 min_tokens: int = 0
@@ -228,6 +223,7 @@ class SamplingParams(
223 seed: Optional[int] = None,
224 stop: Optional[Union[str, List[str]]] = None,
225 stop_token_ids: Optional[List[int]] = None,
226 + bad_words: Optional[List[str]] = None,
227 include_stop_str_in_output: bool = False,
228 ignore_eos: bool = False,
229 max_tokens: Optional[int] = 16,
@@ -267,6 +263,7 @@ class SamplingParams(
263 seed=seed,
264 stop=stop,
265 stop_token_ids=stop_token_ids,
266 + bad_words=bad_words,
267 include_stop_str_in_output=include_stop_str_in_output,
268 ignore_eos=ignore_eos,
269 max_tokens=max_tokens,
@@ -298,26 +295,36 @@ class SamplingParams(
295 f"got n={self.n} and best_of={self.best_of}.")
296 self._real_n = self.n
297 self.n = self.best_of
298 +
299 if 0 < self.temperature < _MAX_TEMP:
300 logger.warning(
301 "temperature %s is less than %s, which may cause numerical "
302 "errors nan or inf in tensors. We have maxed it out to %s.",
303 self.temperature, _MAX_TEMP, _MAX_TEMP)
304 self.temperature = max(self.temperature, _MAX_TEMP)
305 +
306 if self.seed == -1:
307 self.seed = None
308 else:
309 self.seed = self.seed
310 +
311 if self.stop is None:
312 self.stop = []
313 elif isinstance(self.stop, str):
314 self.stop = [self.stop]
315 else:
316 self.stop = list(self.stop)
317 +
318 if self.stop_token_ids is None:
319 self.stop_token_ids = []
320 else:
321 self.stop_token_ids = list(self.stop_token_ids)
322 +
323 + if self.bad_words is None:
324 + self.bad_words = []
325 + else:
326 + self.bad_words = list(self.bad_words)
327 +
328 self.logprobs = 1 if self.logprobs is True else self.logprobs
329 self.prompt_logprobs = (1 if self.prompt_logprobs is True else
330 self.prompt_logprobs)
@@ -468,6 +475,7 @@ class SamplingParams(
475 f"seed={self.seed}, "
476 f"stop={self.stop}, "
477 f"stop_token_ids={self.stop_token_ids}, "
478 + f"bad_words={self.bad_words}, "
479 f"include_stop_str_in_output={self.include_stop_str_in_output}, "
480 f"ignore_eos={self.ignore_eos}, "
481 f"max_tokens={self.max_tokens}, "