A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix for inconsistent behaviour related to sampling and repetition penalties (#5639)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jun 18, 2024 at 23:18 UTC
8a173382c80d6730e1bbc81f932ac3721ab2cd9d
1 file changed
+14
-8
vllm/model_executor/sampling_metadata.py
+14
-8
@@ -386,18 +386,10 @@ class SamplingTensors:
386
presence_penalties += [0] * prefill_len
387
frequency_penalties += [0] * prefill_len
388
repetition_penalties += [1] * prefill_len
389
- if do_penalties:
390
- prompt_tokens.extend([] for _ in range(prefill_len))
391
- output_tokens.extend([] for _ in range(prefill_len))
389
390
if seq_group.do_sample:
391
sample_lens = len(seq_group.sample_indices)
392
assert sample_lens == len(seq_ids)
396
- for seq_id in seq_ids:
397
- seq_data = seq_group.seq_data[seq_id]
398
- if do_penalties:
399
- prompt_tokens.append(seq_data.prompt_token_ids)
400
- output_tokens.append(seq_data.output_token_ids)
393
temperatures += [temperature] * len(seq_ids)
394
top_ps += [top_p] * len(seq_ids)
395
top_ks += [top_k] * len(seq_ids)
@@ -424,6 +416,20 @@ class SamplingTensors:
416
sampling_seeds.append(seq_seeds)
417
sample_indices.extend(seq_group.sample_indices)
418
419
+ if do_penalties:
420
+ for seq_group in sampling_metadata.seq_groups:
421
+ seq_ids = seq_group.seq_ids
422
+ if (seq_group.is_prompt
423
+ and sampling_params.prompt_logprobs is not None):
424
+ prefill_len = len(seq_group.prompt_logprob_indices)
425
+ prompt_tokens.extend([] for _ in range(prefill_len))
426
+ output_tokens.extend([] for _ in range(prefill_len))
427
+ if seq_group.do_sample:
428
+ for seq_id in seq_ids:
429
+ seq_data = seq_group.seq_data[seq_id]
430
+ prompt_tokens.append(seq_data.prompt_token_ids)
431
+ output_tokens.append(seq_data.output_token_ids)
432
+
433
sampling_tensors = SamplingTensors.from_lists(
434
temperatures, top_ps, top_ks, min_ps, presence_penalties,
435
frequency_penalties, repetition_penalties, sampling_seeds,