A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix][SpecDecode] apply sampling parameters to target probabilities for consistency in rejection sampling. (#10198)
Signed-off-by: jeongin601 <0200angela@gmail.com> Signed-off-by: jeong_in.bae <jeong_in.bae@navercorp.com>
jeongin601 committed
Nov 27, 2024 at 14:07 UTC
1bf905ddaa969e6458fe0d15a1db80318f39fade
3 files changed
+10
-14
tests/spec_decode/e2e/test_mlp_correctness.py
+1
-1
@@ -203,7 +203,7 @@ def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
203
@pytest.mark.parametrize("test_llm_kwargs", [{"seed": 5}])
204
@pytest.mark.parametrize("output_len", [64])
205
@pytest.mark.parametrize("batch_size", [1, 32])
206
-@pytest.mark.parametrize("temperature", [0.1, 1.0])
206
+@pytest.mark.parametrize("temperature", [1.0])
207
@pytest.mark.parametrize("seed", [1])
208
def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
209
per_test_common_llm_kwargs,
tests/spec_decode/test_batch_expansion.py
+8
@@ -90,6 +90,14 @@ def test_create_single_target_seq_group_metadata(k: int):
90
)
91
92
assert output.request_id == input_seq_group_metadata.request_id
93
+ assert output.sampling_params.repetition_penalty == \
94
+ input_seq_group_metadata.sampling_params.repetition_penalty
95
+ assert output.sampling_params.temperature == \
96
+ input_seq_group_metadata.sampling_params.temperature
97
+ assert output.sampling_params.top_p == \
98
+ input_seq_group_metadata.sampling_params.top_p
99
+ assert output.sampling_params.top_k == \
100
+ input_seq_group_metadata.sampling_params.top_k
101
assert len(output.seq_data) == 1
102
assert output.seq_data[target_seq_id].get_prompt_token_ids() == tuple(
103
prompt_tokens)
vllm/spec_decode/batch_expansion.py
+1
-13
@@ -307,28 +307,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
307
token_ids_to_score = self._get_token_ids_to_score(
308
proposal_token_ids[batch_index])
309
310
- # Use simpler sampling parameters apart from for final token
311
- # (in particular don't do seeded sampling) since those sampled tokens
312
- # aren't used.
313
- # We don't replace the sampling_params in the greedy case because
314
- # this also controls whether the probs get modified in the sampler
315
- # (see use of _modify_greedy_probs_inplace there).
310
sampling_params = input_seq_group_metadata.sampling_params
317
- non_bonus_sampling_params = DEFAULT_SIMPLE_SAMPLING_PARAMS \
318
- if sampling_params.temperature else sampling_params
319
-
311
target_seq_group_metadata_list: List[SequenceGroupMetadata] = []
321
- last_index = len(token_ids_to_score) - 1
312
for i, token_ids in enumerate(token_ids_to_score):
323
- target_sampling_params = sampling_params if i == last_index \
324
- else non_bonus_sampling_params
313
target_seq_group_metadata_list.append(
314
self._create_single_target_seq_group_metadata(
315
input_seq_group_metadata,
316
input_seq_id,
317
next(target_seq_ids_iter),
318
token_ids,
331
- sampling_params=target_sampling_params,
319
+ sampling_params=sampling_params,
320
))
321
322
return target_seq_group_metadata_list