A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] handle prompt_logprobs in _apply_min_tokens_penalty (#3876)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Apr 10, 2024 at 02:39 UTC
0258b7a94b08321ca01cf170f867b67c1920af87
2 files changed
+112
-23
tests/samplers/test_sampler.py
+94
-22
@@ -1,3 +1,4 @@
1
+import itertools
2
import random
3
from typing import List, Optional, Tuple
4
from unittest.mock import patch
@@ -194,11 +195,15 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
195
196
def create_sampling_params(min_tokens,
197
eos_token_id=0,
197
- stop_token_ids=None):
198
+ *,
199
+ stop_token_ids: Optional[List[str]] = None,
200
+ prompt_logprobs: Optional[int] = None):
201
sampling_params = SamplingParams(
202
min_tokens=min_tokens,
203
max_tokens=9999, # keep higher than max of min_tokens
204
stop_token_ids=stop_token_ids,
205
+ # requesting prompt_logprobs changes the structure of `logits`
206
+ prompt_logprobs=prompt_logprobs,
207
)
208
sampling_params.eos_token_id = eos_token_id
209
return sampling_params
@@ -217,9 +222,9 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
222
223
expected_penalization = []
224
sequence_metadata_list = []
225
+ # 20% chance to generate seq group metadata list with all prompts
226
+ is_prompt = random.random() < 0.2
227
while batch_size > 0:
221
- # 20% chance to generate prompt seq group with single sequence
222
- is_prompt = random.random() < 0.2
228
num_seqs = 1 if is_prompt else random.randint(1, batch_size)
229
230
eos_token_id = random.randint(0, VOCAB_SIZE - 1)
@@ -240,7 +245,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
245
seq_group_penalization = []
246
for _ in range(num_seqs):
247
num_input = random.randint(1, 100)
243
- num_generated = random.randint(1, 100) if not is_prompt else 0
248
+ num_generated = 0 if is_prompt else random.randint(1, 100)
249
seq_data[next(seq_id_counter)] = create_sequence_data(
250
num_input=num_input, num_generated=num_generated)
251
seq_group_penalization.append(num_generated < min_tokens)
@@ -292,6 +297,21 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
297
]
298
}
299
300
+ prompt_with_penalization_and_prompt_logprobs = {
301
+ "expected_penalization": [False, False, True],
302
+ "seq_group_metadata_list": [
303
+ SequenceGroupMetadata(
304
+ request_id="test_1",
305
+ is_prompt=True,
306
+ seq_data={
307
+ next(seq_id_counter): create_sequence_data(num_input=3),
308
+ },
309
+ sampling_params=create_sampling_params(1, prompt_logprobs=3),
310
+ block_tables={},
311
+ ),
312
+ ]
313
+ }
314
+
315
stop_penalizing_after_min_tokens = {
316
"expected_penalization": [False],
317
"seq_group_metadata_list": [
@@ -309,8 +329,34 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
329
}
330
331
stop_token_ids = [42, 99, 42, 0] # intentional duplication
312
- simple_combination = {
313
- "expected_penalization": [True, False, False],
332
+ prompt_combination = {
333
+ "expected_penalization": [False, True, False],
334
+ "seq_group_metadata_list": [
335
+ SequenceGroupMetadata(
336
+ request_id="test_2",
337
+ is_prompt=True,
338
+ seq_data={
339
+ next(seq_id_counter): create_sequence_data(num_input=2),
340
+ },
341
+ sampling_params=create_sampling_params(1, prompt_logprobs=3),
342
+ block_tables={},
343
+ ),
344
+ SequenceGroupMetadata(
345
+ request_id="test_3",
346
+ is_prompt=True,
347
+ seq_data={
348
+ next(seq_id_counter): create_sequence_data(),
349
+ },
350
+ sampling_params=create_sampling_params(
351
+ 0, stop_token_ids=stop_token_ids),
352
+ block_tables={},
353
+ )
354
+ ]
355
+ }
356
+
357
+ stop_token_ids = [1, 999, 37, 37] # intentional duplication
358
+ decode_combination = {
359
+ "expected_penalization": [True, False, False, True, False],
360
"seq_group_metadata_list": [
361
SequenceGroupMetadata(
362
request_id="test_1",
@@ -327,14 +373,19 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
373
),
374
SequenceGroupMetadata(
375
request_id="test_2",
330
- is_prompt=True,
376
+ is_prompt=False,
377
seq_data={
332
- next(seq_id_counter): create_sequence_data(),
378
+ next(seq_id_counter):
379
+ create_sequence_data(num_generated=20),
380
+ next(seq_id_counter):
381
+ create_sequence_data(num_generated=1),
382
+ next(seq_id_counter):
383
+ create_sequence_data(num_generated=10),
384
},
385
sampling_params=create_sampling_params(
335
- 0, stop_token_ids=stop_token_ids),
386
+ 10, prompt_logprobs=5, stop_token_ids=stop_token_ids),
387
block_tables={},
337
- )
388
+ ),
389
]
390
}
391
@@ -342,8 +393,10 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
393
test_cases = [
394
prompt_without_penalization,
395
prompt_with_penalization,
396
+ prompt_with_penalization_and_prompt_logprobs,
397
stop_penalizing_after_min_tokens,
346
- simple_combination,
398
+ prompt_combination,
399
+ decode_combination,
400
]
401
else:
402
test_cases = [generate_test_case()]
@@ -351,30 +404,49 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
404
def run_test_case(*,
405
expected_penalization=None,
406
seq_group_metadata_list=None):
354
- assert expected_penalization, "Invalid test case"
355
- assert seq_group_metadata_list, "Invalid test case"
407
+ assert expected_penalization, \
408
+ "Invalid test case, need expected_penalization"
409
+ assert seq_group_metadata_list, \
410
+ "Invalid test case, need seq_group_metadata_list"
411
412
batch_size = 0
413
prompt_lens = []
359
- sampling_params_per_seq = []
414
+ sampling_params_per_row = []
415
for sgm in seq_group_metadata_list:
361
- num_seqs = len(sgm.seq_data)
362
- batch_size += num_seqs
416
sampling_params = sgm.sampling_params
364
- for seq_id in sgm.seq_data:
365
- prompt_lens.append(sgm.seq_data[seq_id].get_prompt_len())
366
- sampling_params_per_seq.append(sampling_params)
417
+
418
+ num_rows = len(sgm.seq_data)
419
+ if sgm.is_prompt:
420
+ # a prompt seq_group has only one sequence
421
+ seq_data = next(iter(sgm.seq_data.values()))
422
+ prompt_len = seq_data.get_prompt_len()
423
+ prompt_lens.append(prompt_len)
424
+
425
+ if sgm.sampling_params.prompt_logprobs:
426
+ # with prompt_logprobs each token in the prompt has a row in
427
+ # logits
428
+ num_rows = prompt_len
429
+
430
+ batch_size += num_rows
431
+ sampling_params_per_row.extend(
432
+ itertools.repeat(sampling_params, num_rows))
433
+
434
+ assert len(
435
+ expected_penalization
436
+ ) == batch_size, \
437
+ ("Invalid test case, expected_penalization does not match computed"
438
+ "batch size")
439
440
_, fake_logits, sampler, model_runner = _prepare_test(batch_size)
441
sampling_metadata = model_runner._prepare_sample(
442
seq_group_metadata_list,
371
- prompt_lens=prompt_lens,
372
- subquery_lens=prompt_lens)
443
+ prompt_lens=prompt_lens if prompt_lens else None,
444
+ subquery_lens=prompt_lens if prompt_lens else None)
445
# the logits tensor is modified in-place by the sampler
446
_ = sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
447
448
for logits_idx, (should_penalize, sampling_params) in enumerate(
377
- zip(expected_penalization, sampling_params_per_seq)):
449
+ zip(expected_penalization, sampling_params_per_row)):
450
451
tokens_to_check = [sampling_params.eos_token_id]
452
if sampling_params.stop_token_ids:
vllm/model_executor/layers/sampler.py
+18
-1
@@ -27,6 +27,12 @@ class Sampler(nn.Module):
27
6. Sample the next tokens.
28
Here, each sequence group within the batch can have different sampling
29
parameters (e.g., sampling method, temperature, top-p, top-k, etc.).
30
+
31
+ The structure of the logits tensor is coupled with the seq_groups in
32
+ sampling_metadata. Typically, each sequence in each seq_group has one row in
33
+ logits for the next token to be sampled; however, for a seq_group with a
34
+ prompt request with the prompt_logprobs sampling parameter, there are rows
35
+ in logits for each token in the input prompt.
36
"""
37
38
def forward(
@@ -106,7 +112,16 @@ def _apply_min_tokens_penalty(
112
# list of indices in logits that will be set to -inf
113
logits_to_penalize = []
114
start_idx = 0
109
- for seq_ids, sampling_params in sampling_metadata.seq_groups:
115
+ for i, seq_group in enumerate(sampling_metadata.seq_groups):
116
+ seq_ids, sampling_params = seq_group
117
+
118
+ # handle prompt_logprobs by skipping rows in logits added for the prompt
119
+ # tokens (prompt logprobs are not penalized)
120
+ if (i < sampling_metadata.num_prompts
121
+ and sampling_params.prompt_logprobs is not None):
122
+ assert len(seq_ids) == 1
123
+ start_idx += sampling_metadata.prompt_lens[i] - 1
124
+
125
min_tokens = sampling_params.min_tokens
126
if min_tokens > 0:
127
seqs_to_penalize = []
@@ -132,6 +147,8 @@ def _apply_min_tokens_penalty(
147
# eg. [ (1,2), (1,3), (5,6) ] -> ( (1,1,5), (2,3,6) )
148
logits[tuple(zip(*logits_to_penalize))] = -float("inf")
149
150
+ # verifies that no rows in logits were missed unexpectedly
151
+ assert start_idx == logits.shape[0]
152
return logits
153
154