@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 555bdcc5a3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Added logits processor API to sampling params (#1469)

Noam Gat committed Nov 3, 2023 at 23:12 UTC 555bdcc5a3c67779ec80c9fc89323ce8cb05913f
3 files changed +70 -2
tests/samplers/test_sampler.py
+34
@@ -183,3 +183,37 @@ def test_sampler_mixed(seed: int):
183 continue
184 for nth_output in sequence_output.samples:
185 assert nth_output.output_token in expected_tokens
186 +
187 +
188 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
189 +def test_sampler_logits_processors(seed: int):
190 + set_random_seed(seed)
191 + batch_size = random.randint(1, 256)
192 + input_tensor, _, sampler, worker = _prepare_test(batch_size)
193 +
194 + # This sample logits processor gives infinite score to the i-th token,
195 + # where i is the length of the input sequence.
196 + # We therefore expect the output token sequence to be [0, 1, 2, ...]
197 + def pick_ith(token_ids, logits):
198 + logits[len(token_ids)] = float("inf")
199 + return logits
200 +
201 + seq_group_metadata_list = []
202 + for i in range(batch_size):
203 + seq_group_metadata_list.append(
204 + SequenceGroupMetadata(
205 + request_id=f"test_{i}",
206 + is_prompt=True,
207 + seq_data={0: SequenceData([1, 2, 3])},
208 + sampling_params=SamplingParams(temperature=0,
209 + logits_processors=[pick_ith]),
210 + block_tables={0: [1]},
211 + ))
212 +
213 + _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
214 + sampler_output = sampler(embedding=None,
215 + hidden_states=input_tensor,
216 + input_metadata=input_metadata)
217 + for i, sequence_output in enumerate(sampler_output):
218 + for idx, nth_output in enumerate(sequence_output.samples):
219 + assert nth_output.output_token == idx
vllm/model_executor/layers/sampler.py
+24
@@ -47,6 +47,8 @@ class Sampler(nn.Module):
47 logits = _get_logits(hidden_states, embedding, embedding_bias,
48 self.vocab_size)
49
50 + # Apply logits processors (if any).
51 + logits = _apply_logits_processors(logits, input_metadata)
52 # Apply presence and frequency penalties.
53 output_tokens = _get_output_tokens(input_metadata)
54 assert len(output_tokens) == logits.shape[0]
@@ -155,6 +157,28 @@ def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
157 return output_tokens
158
159
160 +def _apply_logits_processors(logits: torch.Tensor,
161 + input_metadata: InputMetadata) -> torch.Tensor:
162 + logits_row_idx = 0
163 + found_logits_processors = False
164 + for seq_ids, sampling_params in input_metadata.seq_groups:
165 + logits_processors = sampling_params.logits_processors
166 + if logits_processors:
167 + found_logits_processors = True
168 + for seq_id in seq_ids:
169 + logits_row = logits[logits_row_idx]
170 + token_ids = input_metadata.seq_data[seq_id].output_token_ids
171 + for logits_processor in logits_processors:
172 + logits_row = logits_processor(token_ids, logits_row)
173 + logits[logits_row_idx] = logits_row
174 + logits_row_idx += 1
175 + else:
176 + logits_row_idx += len(seq_ids)
177 + if found_logits_processors:
178 + assert logits_row_idx == logits.shape[0]
179 + return logits
180 +
181 +
182 def _apply_penalties(
183 logits: torch.Tensor,
184 output_tokens: List[List[int]],
vllm/sampling_params.py
+12 -2
@@ -1,7 +1,8 @@
1 """Sampling parameters for text generation."""
2 from enum import IntEnum
3 from functools import cached_property
4 -from typing import List, Optional, Union
4 +from typing import Callable, List, Optional, Union
5 +import torch
6
7 _SAMPLING_EPS = 1e-5
8
@@ -12,6 +13,12 @@ class SamplingType(IntEnum):
13 BEAM = 2
14
15
16 +LogitsProcessor = Callable[[List[int], torch.Tensor], torch.Tensor]
17 +"""LogitsProcessor is a function that takes a list of previously generated
18 +tokens and a tensor of the logits for the next token, and returns a modified
19 +tensor of logits to sample from."""
20 +
21 +
22 class SamplingParams:
23 """Sampling parameters for text generation.
24
@@ -73,6 +80,8 @@ class SamplingParams:
80 skip_special_tokens: Whether to skip special tokens in the output.
81 spaces_between_special_tokens: Whether to add spaces between special
82 tokens in the output. Defaults to True.
83 + logits_processors: List of functions that modify logits based on
84 + previously generated tokens.
85 """
86
87 def __init__(
@@ -96,6 +105,7 @@ class SamplingParams:
105 prompt_logprobs: Optional[int] = None,
106 skip_special_tokens: bool = True,
107 spaces_between_special_tokens: bool = True,
108 + logits_processors: Optional[List[LogitsProcessor]] = None,
109 ) -> None:
110 self.n = n
111 self.best_of = best_of if best_of is not None else n
@@ -124,7 +134,7 @@ class SamplingParams:
134 self.prompt_logprobs = prompt_logprobs
135 self.skip_special_tokens = skip_special_tokens
136 self.spaces_between_special_tokens = spaces_between_special_tokens
127 -
137 + self.logits_processors = logits_processors
138 self._verify_args()
139 if self.use_beam_search:
140 self._verify_beam_search()