A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
feat: implement the min_tokens sampling parameter (#3124)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Travis Johnson committed
Mar 25, 2024 at 11:14 UTC
c13ad1b7bdf6fba5ab02225e6655bc26fae90192
5 files changed
+299
-12
tests/samplers/test_sampler.py
+222
-2
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.sampler import Sampler
10
from vllm.model_executor.utils import set_random_seed
11
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12
from vllm.worker.model_runner import ModelRunner
13
+from vllm.utils import Counter
14
15
16
class MockLogitsSampler(Sampler):
@@ -25,9 +26,8 @@ class MockLogitsSampler(Sampler):
26
def _prepare_test(
27
batch_size: int
28
) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, ModelRunner]:
28
- vocab_size = 32000
29
input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
30
- fake_logits = torch.full((batch_size, vocab_size),
30
+ fake_logits = torch.full((batch_size, VOCAB_SIZE),
31
1e-2,
32
dtype=input_tensor.dtype)
33
sampler = MockLogitsSampler(fake_logits)
@@ -35,6 +35,7 @@ def _prepare_test(
35
return input_tensor, fake_logits, sampler, model_runner
36
37
38
+VOCAB_SIZE = 32000
39
RANDOM_SEEDS = list(range(128))
40
CUDA_DEVICES = [
41
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
@@ -184,6 +185,225 @@ def test_sampler_all_beam(seed: int, device: str):
185
del model_runner
186
187
188
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
189
+@pytest.mark.parametrize("device", CUDA_DEVICES)
190
+def test_sampler_min_tokens_penalty(seed: int, device: str):
191
+ seq_id_counter = Counter(start=random.randint(0, 100))
192
+ set_random_seed(seed)
193
+ torch.set_default_device(device)
194
+
195
+ def create_sampling_params(min_tokens,
196
+ eos_token_id=0,
197
+ stop_token_ids=None):
198
+ sampling_params = SamplingParams(
199
+ min_tokens=min_tokens,
200
+ max_tokens=9999, # keep higher than max of min_tokens
201
+ stop_token_ids=stop_token_ids,
202
+ )
203
+ sampling_params.eos_token_id = eos_token_id
204
+ return sampling_params
205
+
206
+ def create_sequence_data(num_input=3, num_generated=0):
207
+ seq_data = SequenceData(
208
+ random.choices(range(0, VOCAB_SIZE), k=num_input))
209
+ if num_generated > 0:
210
+ seq_data.output_token_ids = random.choices(range(0, VOCAB_SIZE),
211
+ k=num_generated)
212
+ return seq_data
213
+
214
+ def generate_test_case():
215
+ # generate multiple seq groups but limit total batch size
216
+ batch_size = random.randint(1, 128)
217
+
218
+ expected_penalization = []
219
+ sequence_metadata_list = []
220
+ while batch_size > 0:
221
+ # 20% chance to generate prompt seq group with single sequence
222
+ is_prompt = random.random() < 0.2
223
+ num_seqs = 1 if is_prompt else random.randint(1, batch_size)
224
+
225
+ eos_token_id = random.randint(0, VOCAB_SIZE - 1)
226
+ min_tokens = random.randint(0, 50)
227
+ num_stop_tokens = random.randint(0, 8)
228
+ if num_stop_tokens > 0:
229
+ stop_token_ids = random.choices(range(0, VOCAB_SIZE - 1),
230
+ k=num_stop_tokens)
231
+ else:
232
+ stop_token_ids = None
233
+
234
+ sampling_params = create_sampling_params(
235
+ min_tokens=min_tokens,
236
+ eos_token_id=eos_token_id,
237
+ stop_token_ids=stop_token_ids)
238
+
239
+ seq_data = {}
240
+ seq_group_penalization = []
241
+ for _ in range(num_seqs):
242
+ num_input = random.randint(1, 100)
243
+ num_generated = random.randint(1, 100) if not is_prompt else 0
244
+ seq_data[next(seq_id_counter)] = create_sequence_data(
245
+ num_input=num_input, num_generated=num_generated)
246
+ seq_group_penalization.append(num_generated < min_tokens)
247
+
248
+ expected_penalization.extend(seq_group_penalization)
249
+ sequence_metadata_list.append(
250
+ SequenceGroupMetadata(
251
+ request_id=f"test_{batch_size}",
252
+ is_prompt=is_prompt,
253
+ seq_data=seq_data,
254
+ sampling_params=sampling_params,
255
+ block_tables={},
256
+ ))
257
+ batch_size -= num_seqs
258
+
259
+ return {
260
+ "expected_penalization": expected_penalization,
261
+ "seq_group_metadata_list": sequence_metadata_list,
262
+ }
263
+
264
+ # define some explicit test cases for edge case behavior
265
+ prompt_without_penalization = {
266
+ "expected_penalization": [False],
267
+ "seq_group_metadata_list": [
268
+ SequenceGroupMetadata(
269
+ request_id="test_1",
270
+ is_prompt=True,
271
+ seq_data={
272
+ next(seq_id_counter): create_sequence_data(),
273
+ },
274
+ sampling_params=create_sampling_params(0),
275
+ block_tables={},
276
+ ),
277
+ ]
278
+ }
279
+
280
+ prompt_with_penalization = {
281
+ "expected_penalization": [True],
282
+ "seq_group_metadata_list": [
283
+ SequenceGroupMetadata(
284
+ request_id="test_1",
285
+ is_prompt=True,
286
+ seq_data={
287
+ next(seq_id_counter): create_sequence_data(),
288
+ },
289
+ sampling_params=create_sampling_params(1),
290
+ block_tables={},
291
+ ),
292
+ ]
293
+ }
294
+
295
+ stop_penalizing_after_min_tokens = {
296
+ "expected_penalization": [False],
297
+ "seq_group_metadata_list": [
298
+ SequenceGroupMetadata(
299
+ request_id="test_1",
300
+ is_prompt=False,
301
+ seq_data={
302
+ next(seq_id_counter):
303
+ create_sequence_data(num_generated=1),
304
+ },
305
+ sampling_params=create_sampling_params(1),
306
+ block_tables={},
307
+ )
308
+ ]
309
+ }
310
+
311
+ stop_token_ids = [42, 99, 42, 0] # intentional duplication
312
+ simple_combination = {
313
+ "expected_penalization": [True, False, False],
314
+ "seq_group_metadata_list": [
315
+ SequenceGroupMetadata(
316
+ request_id="test_1",
317
+ is_prompt=False,
318
+ seq_data={
319
+ next(seq_id_counter):
320
+ create_sequence_data(num_generated=1),
321
+ next(seq_id_counter):
322
+ create_sequence_data(num_generated=100),
323
+ },
324
+ sampling_params=create_sampling_params(
325
+ 2, stop_token_ids=stop_token_ids),
326
+ block_tables={},
327
+ ),
328
+ SequenceGroupMetadata(
329
+ request_id="test_2",
330
+ is_prompt=True,
331
+ seq_data={
332
+ next(seq_id_counter): create_sequence_data(),
333
+ },
334
+ sampling_params=create_sampling_params(
335
+ 0, stop_token_ids=stop_token_ids),
336
+ block_tables={},
337
+ )
338
+ ]
339
+ }
340
+
341
+ if seed == 0:
342
+ test_cases = [
343
+ prompt_without_penalization,
344
+ prompt_with_penalization,
345
+ stop_penalizing_after_min_tokens,
346
+ simple_combination,
347
+ ]
348
+ else:
349
+ test_cases = [generate_test_case()]
350
+
351
+ def run_test_case(*,
352
+ expected_penalization=None,
353
+ seq_group_metadata_list=None):
354
+ assert expected_penalization, "Invalid test case"
355
+ assert seq_group_metadata_list, "Invalid test case"
356
+
357
+ batch_size = 0
358
+ prompt_lens = []
359
+ sampling_params_per_seq = []
360
+ for sgm in seq_group_metadata_list:
361
+ num_seqs = len(sgm.seq_data)
362
+ batch_size += num_seqs
363
+ sampling_params = sgm.sampling_params
364
+ for seq_id in sgm.seq_data:
365
+ prompt_lens.append(sgm.seq_data[seq_id].get_prompt_len())
366
+ sampling_params_per_seq.append(sampling_params)
367
+
368
+ _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
369
+ sampling_metadata = model_runner._prepare_sample(
370
+ seq_group_metadata_list,
371
+ prompt_lens=prompt_lens,
372
+ subquery_lens=prompt_lens)
373
+ # the logits tensor is modified in-place by the sampler
374
+ _ = sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
375
+
376
+ for logits_idx, (should_penalize, sampling_params) in enumerate(
377
+ zip(expected_penalization, sampling_params_per_seq)):
378
+
379
+ tokens_to_check = [sampling_params.eos_token_id]
380
+ if sampling_params.stop_token_ids:
381
+ tokens_to_check.extend(sampling_params.stop_token_ids)
382
+ tokens_to_check = set(tokens_to_check)
383
+
384
+ if should_penalize:
385
+ for token_id in tokens_to_check:
386
+ assert fake_logits[logits_idx, token_id] == -float(
387
+ 'inf'
388
+ ), f"Expected token {token_id} for logits row {logits_idx}"
389
+ " to be penalized"
390
+ # no other tokens should be set to -inf
391
+ assert torch.count_nonzero(
392
+ fake_logits[logits_idx, :] == -float('inf')) == len(
393
+ tokens_to_check
394
+ ), f"Expected only {len(tokens_to_check)} to be penalized"
395
+ else:
396
+ # no tokens should be set to -inf
397
+ assert torch.count_nonzero(
398
+ fake_logits[logits_idx, :] ==
399
+ -float('inf')) == 0, "No tokens should have been penalized"
400
+
401
+ del model_runner
402
+
403
+ for test_case in test_cases:
404
+ run_test_case(**test_case)
405
+
406
+
407
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
408
@pytest.mark.parametrize("device", CUDA_DEVICES)
409
def test_sampler_mixed(seed: int, device: str):
vllm/engine/llm_engine.py
+18
-10
@@ -282,6 +282,9 @@ class LLMEngine:
282
# Defensive copy of SamplingParams, which are used by the sampler,
283
# this doesn't deep-copy LogitsProcessor objects
284
sampling_params = sampling_params.clone()
285
+ # inject the eos token id into the sampling_params to support min_tokens
286
+ # processing
287
+ sampling_params.eos_token_id = seq.eos_token_id
288
289
# Create the sequence group.
290
seq_group = SequenceGroup(request_id, [seq], sampling_params,
@@ -713,6 +716,21 @@ class LLMEngine:
716
def _check_stop(self, seq: Sequence,
717
sampling_params: SamplingParams) -> None:
718
"""Stop the finished sequences."""
719
+ # Check if the sequence has reached max_model_len.
720
+ if seq.get_len() > self.scheduler_config.max_model_len:
721
+ seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
722
+ return
723
+
724
+ # Check if the sequence has reached max_tokens.
725
+ if seq.get_output_len() == sampling_params.max_tokens:
726
+ seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
727
+ return
728
+
729
+ # Check if the minimum number of tokens has been generated yet;
730
+ # skip the stop string/token checks if not
731
+ if seq.get_output_len() < sampling_params.min_tokens:
732
+ return
733
+
734
for stop_str in sampling_params.stop:
735
if seq.output_text.endswith(stop_str):
736
self._finalize_sequence(seq, sampling_params, stop_str)
@@ -725,16 +743,6 @@ class LLMEngine:
743
seq.status = SequenceStatus.FINISHED_STOPPED
744
return
745
728
- # Check if the sequence has reached max_model_len.
729
- if seq.get_len() > self.scheduler_config.max_model_len:
730
- seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
731
- return
732
-
733
- # Check if the sequence has reached max_tokens.
734
- if seq.get_output_len() == sampling_params.max_tokens:
735
- seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
736
- return
737
-
746
# Check if the sequence has generated the EOS token.
747
if ((not sampling_params.ignore_eos)
748
and seq.get_last_token_id() == seq.eos_token_id):
vllm/entrypoints/openai/protocol.py
+4
@@ -88,6 +88,7 @@ class ChatCompletionRequest(BaseModel):
88
length_penalty: Optional[float] = 1.0
89
early_stopping: Optional[bool] = False
90
ignore_eos: Optional[bool] = False
91
+ min_tokens: Optional[int] = 0
92
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
93
skip_special_tokens: Optional[bool] = True
94
spaces_between_special_tokens: Optional[bool] = True
@@ -165,6 +166,7 @@ class ChatCompletionRequest(BaseModel):
166
stop=self.stop,
167
stop_token_ids=self.stop_token_ids,
168
max_tokens=self.max_tokens,
169
+ min_tokens=self.min_tokens,
170
logprobs=self.top_logprobs if self.logprobs else None,
171
prompt_logprobs=self.top_logprobs if self.echo else None,
172
best_of=self.best_of,
@@ -224,6 +226,7 @@ class CompletionRequest(BaseModel):
226
early_stopping: Optional[bool] = False
227
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
228
ignore_eos: Optional[bool] = False
229
+ min_tokens: Optional[int] = 0
230
skip_special_tokens: Optional[bool] = True
231
spaces_between_special_tokens: Optional[bool] = True
232
# doc: end-completion-sampling-params
@@ -296,6 +299,7 @@ class CompletionRequest(BaseModel):
299
stop_token_ids=self.stop_token_ids,
300
ignore_eos=self.ignore_eos,
301
max_tokens=self.max_tokens if not echo_without_generation else 1,
302
+ min_tokens=self.min_tokens,
303
logprobs=self.logprobs,
304
use_beam_search=self.use_beam_search,
305
early_stopping=self.early_stopping,
vllm/model_executor/layers/sampler.py
+41
@@ -1,4 +1,5 @@
1
"""A layer that samples the next tokens from the model's outputs."""
2
+import itertools
3
from typing import Dict, List, Optional, Tuple
4
5
import torch
@@ -36,6 +37,10 @@ class Sampler(nn.Module):
37
assert logits is not None
38
_, vocab_size = logits.shape
39
40
+ # Apply min_tokens penalty which sets stop tokens to -inf if min_tokens
41
+ # have not been generated yet
42
+ logits = _apply_min_tokens_penalty(logits, sampling_metadata)
43
+
44
# Prepare sampling tensors with pinned memory to avoid blocking.
45
(sampling_tensors, do_penalties, do_top_p_top_k,
46
do_min_p) = SamplingTensors.from_sampling_metadata(
@@ -94,6 +99,42 @@ def _get_bin_counts_and_mask(
99
return bin_counts, mask
100
101
102
+def _apply_min_tokens_penalty(
103
+ logits: torch.Tensor,
104
+ sampling_metadata: SamplingMetadata,
105
+) -> torch.Tensor:
106
+ # list of indices in logits that will be set to -inf
107
+ logits_to_penalize = []
108
+ start_idx = 0
109
+ for seq_ids, sampling_params in sampling_metadata.seq_groups:
110
+ min_tokens = sampling_params.min_tokens
111
+ if min_tokens > 0:
112
+ seqs_to_penalize = []
113
+ for i, seq_id in enumerate(seq_ids):
114
+ seq_data = sampling_metadata.seq_data[seq_id]
115
+ if len(seq_data.output_token_ids) < min_tokens:
116
+ seqs_to_penalize.append(i)
117
+
118
+ if seqs_to_penalize:
119
+ # convert to the index into logits
120
+ seqs_to_penalize = [start_idx + i for i in seqs_to_penalize]
121
+ # use set() to remove any duplicates
122
+ token_ids_to_penalize = set(sampling_params.stop_token_ids +
123
+ [sampling_params.eos_token_id])
124
+ # itertools.product pairs each seq index with every token id
125
+ logits_to_penalize.extend(
126
+ itertools.product(seqs_to_penalize, token_ids_to_penalize))
127
+
128
+ start_idx += len(seq_ids)
129
+
130
+ if logits_to_penalize:
131
+ # use zip and * to group indices along each dimension
132
+ # eg. [ (1,2), (1,3), (5,6) ] -> ( (1,1,5), (2,3,6) )
133
+ logits[tuple(zip(*logits_to_penalize))] = -float("inf")
134
+
135
+ return logits
136
+
137
+
138
def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
139
output_tokens_tensor: torch.Tensor,
140
presence_penalties: torch.Tensor,
vllm/sampling_params.py
+14
@@ -79,6 +79,8 @@ class SamplingParams:
79
ignore_eos: Whether to ignore the EOS token and continue generating
80
tokens after the EOS token is generated.
81
max_tokens: Maximum number of tokens to generate per output sequence.
82
+ min_tokens: Minimum number of tokens to generate per output sequence
83
+ before EOS or stop_token_ids can be generated
84
logprobs: Number of log probabilities to return per output token.
85
Note that the implementation follows the OpenAI API: The return
86
result includes the log probabilities on the `logprobs` most likely
@@ -113,6 +115,7 @@ class SamplingParams:
115
include_stop_str_in_output: bool = False,
116
ignore_eos: bool = False,
117
max_tokens: Optional[int] = 16,
118
+ min_tokens: int = 0,
119
logprobs: Optional[int] = None,
120
prompt_logprobs: Optional[int] = None,
121
skip_special_tokens: bool = True,
@@ -144,6 +147,7 @@ class SamplingParams:
147
self.stop_token_ids = list(stop_token_ids)
148
self.ignore_eos = ignore_eos
149
self.max_tokens = max_tokens
150
+ self.min_tokens = min_tokens
151
self.logprobs = logprobs
152
self.prompt_logprobs = prompt_logprobs
153
self.skip_special_tokens = skip_special_tokens
@@ -161,6 +165,8 @@ class SamplingParams:
165
self.top_k = -1
166
self.min_p = 0.0
167
self._verify_greedy_sampling()
168
+ # injected by the engine
169
+ self.eos_token_id = None
170
171
def _verify_args(self) -> None:
172
if self.n < 1:
@@ -191,6 +197,13 @@ class SamplingParams:
197
if self.max_tokens is not None and self.max_tokens < 1:
198
raise ValueError(
199
f"max_tokens must be at least 1, got {self.max_tokens}.")
200
+ if self.min_tokens < 0:
201
+ raise ValueError(f"min_tokens must be greater than or equal to 0, "
202
+ f"got {self.min_tokens}.")
203
+ if self.max_tokens is not None and self.min_tokens > self.max_tokens:
204
+ raise ValueError(
205
+ f"min_tokens must be less than or equal to "
206
+ f"max_tokens={self.max_tokens}, got {self.min_tokens}.")
207
if self.logprobs is not None and self.logprobs < 0:
208
raise ValueError(
209
f"logprobs must be non-negative, got {self.logprobs}.")
@@ -272,6 +285,7 @@ class SamplingParams:
285
f"include_stop_str_in_output={self.include_stop_str_in_output}, "
286
f"ignore_eos={self.ignore_eos}, "
287
f"max_tokens={self.max_tokens}, "
288
+ f"min_tokens={self.min_tokens}, "
289
f"logprobs={self.logprobs}, "
290
f"prompt_logprobs={self.prompt_logprobs}, "
291
f"skip_special_tokens={self.skip_special_tokens}, "