@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e5150f2c28
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Added test for sampling repetition penalty bug. (#5659)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Jun 19, 2024 at 08:03 UTC e5150f2c281f052df42121ae60827156abe57173
1 file changed +69
tests/samplers/test_sampler.py
+69
@@ -631,3 +631,72 @@ def test_sampler_top_k_top_p(seed: int, device: str):
631 hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
632 assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
633 assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
634 +
635 +
636 +@pytest.mark.parametrize("device", CUDA_DEVICES)
637 +def test_sampler_repetition_penalty_mixed(device: str):
638 +
639 + vocab_size = 8
640 +
641 + def test_sampling_params(sampling_params: List[SamplingParams]):
642 +
643 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
644 + seq_lens: List[int] = []
645 + for i in range(2):
646 + seq_group_metadata_list.append(
647 + SequenceGroupMetadata(
648 + request_id=f"test_{i}",
649 + is_prompt=True,
650 + seq_data={0: SequenceData([1, 2, 3])},
651 + sampling_params=sampling_params[i],
652 + block_tables={0: [1]},
653 + ))
654 + seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
655 +
656 + sampling_metadata = SamplingMetadata.prepare(
657 + seq_group_metadata_list,
658 + seq_lens,
659 + query_lens=seq_lens,
660 + device=device,
661 + pin_memory=is_pin_memory_available())
662 +
663 + fake_logits = torch.full((2, vocab_size),
664 + 1e-2,
665 + device=device,
666 + dtype=torch.float16)
667 +
668 + fake_logits[:, 5] = 1.1e-2
669 + fake_logits[:, 1] = 1.2e-2
670 +
671 + sampler = MockLogitsSampler(fake_logits)
672 +
673 + sampler_output = sampler(logits=fake_logits,
674 + sampling_metadata=sampling_metadata)
675 +
676 + generated_tokens = []
677 + for output in sampler_output:
678 + generated_tokens.append(output.samples[0].output_token)
679 +
680 + return generated_tokens
681 +
682 + # one configuration is greedy with repetition_penalty
683 + sampling_params_rep = SamplingParams(
684 + temperature=0.0,
685 + repetition_penalty=2.0,
686 + )
687 +
688 + # other configuration is sampling w/o repetition_penalty
689 + sampling_params_sample = SamplingParams(
690 + temperature=1.0,
691 + top_k=1,
692 + seed=42,
693 + )
694 +
695 + tokens1 = test_sampling_params(
696 + [sampling_params_rep, sampling_params_sample])
697 +
698 + tokens2 = test_sampling_params(
699 + [sampling_params_sample, sampling_params_rep])
700 +
701 + assert tokens1[0] == tokens2[1]
702 + assert tokens1[1] == tokens2[0]