@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a173382c8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix for inconsistent behaviour related to sampling and repetition penalties (#5639)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Jun 18, 2024 at 23:18 UTC 8a173382c80d6730e1bbc81f932ac3721ab2cd9d
1 file changed +14 -8
vllm/model_executor/sampling_metadata.py
+14 -8
@@ -386,18 +386,10 @@ class SamplingTensors:
386 presence_penalties += [0] * prefill_len
387 frequency_penalties += [0] * prefill_len
388 repetition_penalties += [1] * prefill_len
389 - if do_penalties:
390 - prompt_tokens.extend([] for _ in range(prefill_len))
391 - output_tokens.extend([] for _ in range(prefill_len))
389
390 if seq_group.do_sample:
391 sample_lens = len(seq_group.sample_indices)
392 assert sample_lens == len(seq_ids)
396 - for seq_id in seq_ids:
397 - seq_data = seq_group.seq_data[seq_id]
398 - if do_penalties:
399 - prompt_tokens.append(seq_data.prompt_token_ids)
400 - output_tokens.append(seq_data.output_token_ids)
393 temperatures += [temperature] * len(seq_ids)
394 top_ps += [top_p] * len(seq_ids)
395 top_ks += [top_k] * len(seq_ids)
@@ -424,6 +416,20 @@ class SamplingTensors:
416 sampling_seeds.append(seq_seeds)
417 sample_indices.extend(seq_group.sample_indices)
418
419 + if do_penalties:
420 + for seq_group in sampling_metadata.seq_groups:
421 + seq_ids = seq_group.seq_ids
422 + if (seq_group.is_prompt
423 + and sampling_params.prompt_logprobs is not None):
424 + prefill_len = len(seq_group.prompt_logprob_indices)
425 + prompt_tokens.extend([] for _ in range(prefill_len))
426 + output_tokens.extend([] for _ in range(prefill_len))
427 + if seq_group.do_sample:
428 + for seq_id in seq_ids:
429 + seq_data = seq_group.seq_data[seq_id]
430 + prompt_tokens.append(seq_data.prompt_token_ids)
431 + output_tokens.append(seq_data.output_token_ids)
432 +
433 sampling_tensors = SamplingTensors.from_lists(
434 temperatures, top_ps, top_ks, min_ps, presence_penalties,
435 frequency_penalties, repetition_penalties, sampling_seeds,