A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Added logits processor API to sampling params (#1469)
Noam Gat committed
Nov 3, 2023 at 23:12 UTC
555bdcc5a3c67779ec80c9fc89323ce8cb05913f
3 files changed
+70
-2
tests/samplers/test_sampler.py
+34
@@ -183,3 +183,37 @@ def test_sampler_mixed(seed: int):
183
continue
184
for nth_output in sequence_output.samples:
185
assert nth_output.output_token in expected_tokens
186
+
187
+
188
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
189
+def test_sampler_logits_processors(seed: int):
190
+ set_random_seed(seed)
191
+ batch_size = random.randint(1, 256)
192
+ input_tensor, _, sampler, worker = _prepare_test(batch_size)
193
+
194
+ # This sample logits processor gives infinite score to the i-th token,
195
+ # where i is the length of the input sequence.
196
+ # We therefore expect the output token sequence to be [0, 1, 2, ...]
197
+ def pick_ith(token_ids, logits):
198
+ logits[len(token_ids)] = float("inf")
199
+ return logits
200
+
201
+ seq_group_metadata_list = []
202
+ for i in range(batch_size):
203
+ seq_group_metadata_list.append(
204
+ SequenceGroupMetadata(
205
+ request_id=f"test_{i}",
206
+ is_prompt=True,
207
+ seq_data={0: SequenceData([1, 2, 3])},
208
+ sampling_params=SamplingParams(temperature=0,
209
+ logits_processors=[pick_ith]),
210
+ block_tables={0: [1]},
211
+ ))
212
+
213
+ _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
214
+ sampler_output = sampler(embedding=None,
215
+ hidden_states=input_tensor,
216
+ input_metadata=input_metadata)
217
+ for i, sequence_output in enumerate(sampler_output):
218
+ for idx, nth_output in enumerate(sequence_output.samples):
219
+ assert nth_output.output_token == idx
vllm/model_executor/layers/sampler.py
+24
@@ -47,6 +47,8 @@ class Sampler(nn.Module):
47
logits = _get_logits(hidden_states, embedding, embedding_bias,
48
self.vocab_size)
49
50
+ # Apply logits processors (if any).
51
+ logits = _apply_logits_processors(logits, input_metadata)
52
# Apply presence and frequency penalties.
53
output_tokens = _get_output_tokens(input_metadata)
54
assert len(output_tokens) == logits.shape[0]
@@ -155,6 +157,28 @@ def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
157
return output_tokens
158
159
160
+def _apply_logits_processors(logits: torch.Tensor,
161
+ input_metadata: InputMetadata) -> torch.Tensor:
162
+ logits_row_idx = 0
163
+ found_logits_processors = False
164
+ for seq_ids, sampling_params in input_metadata.seq_groups:
165
+ logits_processors = sampling_params.logits_processors
166
+ if logits_processors:
167
+ found_logits_processors = True
168
+ for seq_id in seq_ids:
169
+ logits_row = logits[logits_row_idx]
170
+ token_ids = input_metadata.seq_data[seq_id].output_token_ids
171
+ for logits_processor in logits_processors:
172
+ logits_row = logits_processor(token_ids, logits_row)
173
+ logits[logits_row_idx] = logits_row
174
+ logits_row_idx += 1
175
+ else:
176
+ logits_row_idx += len(seq_ids)
177
+ if found_logits_processors:
178
+ assert logits_row_idx == logits.shape[0]
179
+ return logits
180
+
181
+
182
def _apply_penalties(
183
logits: torch.Tensor,
184
output_tokens: List[List[int]],
vllm/sampling_params.py
+12
-2
@@ -1,7 +1,8 @@
1
"""Sampling parameters for text generation."""
2
from enum import IntEnum
3
from functools import cached_property
4
-from typing import List, Optional, Union
4
+from typing import Callable, List, Optional, Union
5
+import torch
6
7
_SAMPLING_EPS = 1e-5
8
@@ -12,6 +13,12 @@ class SamplingType(IntEnum):
13
BEAM = 2
14
15
16
+LogitsProcessor = Callable[[List[int], torch.Tensor], torch.Tensor]
17
+"""LogitsProcessor is a function that takes a list of previously generated
18
+tokens and a tensor of the logits for the next token, and returns a modified
19
+tensor of logits to sample from."""
20
+
21
+
22
class SamplingParams:
23
"""Sampling parameters for text generation.
24
@@ -73,6 +80,8 @@ class SamplingParams:
80
skip_special_tokens: Whether to skip special tokens in the output.
81
spaces_between_special_tokens: Whether to add spaces between special
82
tokens in the output. Defaults to True.
83
+ logits_processors: List of functions that modify logits based on
84
+ previously generated tokens.
85
"""
86
87
def __init__(
@@ -96,6 +105,7 @@ class SamplingParams:
105
prompt_logprobs: Optional[int] = None,
106
skip_special_tokens: bool = True,
107
spaces_between_special_tokens: bool = True,
108
+ logits_processors: Optional[List[LogitsProcessor]] = None,
109
) -> None:
110
self.n = n
111
self.best_of = best_of if best_of is not None else n
@@ -124,7 +134,7 @@ class SamplingParams:
134
self.prompt_logprobs = prompt_logprobs
135
self.skip_special_tokens = skip_special_tokens
136
self.spaces_between_special_tokens = spaces_between_special_tokens
127
-
137
+ self.logits_processors = logits_processors
138
self._verify_args()
139
if self.use_beam_search:
140
self._verify_beam_search()