@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1bf905ddaa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix][SpecDecode] apply sampling parameters to target probabilities for consistency in rejection sampling. (#10198)

Signed-off-by: jeongin601 <0200angela@gmail.com> Signed-off-by: jeong_in.bae <jeong_in.bae@navercorp.com>

jeongin601 committed Nov 27, 2024 at 14:07 UTC 1bf905ddaa969e6458fe0d15a1db80318f39fade
3 files changed +10 -14
tests/spec_decode/e2e/test_mlp_correctness.py
+1 -1
@@ -203,7 +203,7 @@ def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
203 @pytest.mark.parametrize("test_llm_kwargs", [{"seed": 5}])
204 @pytest.mark.parametrize("output_len", [64])
205 @pytest.mark.parametrize("batch_size", [1, 32])
206 -@pytest.mark.parametrize("temperature", [0.1, 1.0])
206 +@pytest.mark.parametrize("temperature", [1.0])
207 @pytest.mark.parametrize("seed", [1])
208 def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
209 per_test_common_llm_kwargs,
tests/spec_decode/test_batch_expansion.py
+8
@@ -90,6 +90,14 @@ def test_create_single_target_seq_group_metadata(k: int):
90 )
91
92 assert output.request_id == input_seq_group_metadata.request_id
93 + assert output.sampling_params.repetition_penalty == \
94 + input_seq_group_metadata.sampling_params.repetition_penalty
95 + assert output.sampling_params.temperature == \
96 + input_seq_group_metadata.sampling_params.temperature
97 + assert output.sampling_params.top_p == \
98 + input_seq_group_metadata.sampling_params.top_p
99 + assert output.sampling_params.top_k == \
100 + input_seq_group_metadata.sampling_params.top_k
101 assert len(output.seq_data) == 1
102 assert output.seq_data[target_seq_id].get_prompt_token_ids() == tuple(
103 prompt_tokens)
vllm/spec_decode/batch_expansion.py
+1 -13
@@ -307,28 +307,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
307 token_ids_to_score = self._get_token_ids_to_score(
308 proposal_token_ids[batch_index])
309
310 - # Use simpler sampling parameters apart from for final token
311 - # (in particular don't do seeded sampling) since those sampled tokens
312 - # aren't used.
313 - # We don't replace the sampling_params in the greedy case because
314 - # this also controls whether the probs get modified in the sampler
315 - # (see use of _modify_greedy_probs_inplace there).
310 sampling_params = input_seq_group_metadata.sampling_params
317 - non_bonus_sampling_params = DEFAULT_SIMPLE_SAMPLING_PARAMS \
318 - if sampling_params.temperature else sampling_params
319 -
311 target_seq_group_metadata_list: List[SequenceGroupMetadata] = []
321 - last_index = len(token_ids_to_score) - 1
312 for i, token_ids in enumerate(token_ids_to_score):
323 - target_sampling_params = sampling_params if i == last_index \
324 - else non_bonus_sampling_params
313 target_seq_group_metadata_list.append(
314 self._create_single_target_seq_group_metadata(
315 input_seq_group_metadata,
316 input_seq_id,
317 next(target_seq_ids_iter),
318 token_ids,
331 - sampling_params=target_sampling_params,
319 + sampling_params=sampling_params,
320 ))
321
322 return target_seq_group_metadata_list