A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Unify rank computation across regular decoding and speculative decoding (#7899)
Jonas M. Kübler committed
Aug 29, 2024 at 07:18 UTC
f205c09854853172a446c92aa81eb7199da324ab
2 files changed
+22
-3
tests/spec_decode/test_utils.py
+20
-1
@@ -4,10 +4,12 @@ import pytest
4
import torch
5
6
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7
+from vllm.model_executor.layers.sampler import _get_ranks
8
from vllm.model_executor.layers.typical_acceptance_sampler import (
9
TypicalAcceptanceSampler)
10
from vllm.sequence import SequenceGroupMetadata, get_all_seq_ids
10
-from vllm.spec_decode.util import split_batch_by_proposal_len
11
+from vllm.spec_decode.util import (get_sampled_token_logprobs,
12
+ split_batch_by_proposal_len)
13
14
15
def test_get_all_seq_ids():
@@ -126,3 +128,20 @@ def mock_spec_decode_sampler(acceptance_sampler_method):
128
return sampler
129
else:
130
raise ValueError(f"Invalid sampler name {acceptance_sampler_method}")
131
+
132
+
133
+def test_get_sampled_token_logprobs():
134
+ """Verify get_sampled_token_logprobs returns consistent rankings
135
+ with regular get_ranks when probabilities match exactly.
136
+ """
137
+ logprob_tensor = torch.tensor(
138
+ [[[-.1, -.1]] * 2]) # shape (num_steps, batch_size, vocab_size)
139
+ sampled_token_tensor = torch.tensor([[1,
140
+ 0]]) # shape (num_steps, batch_size)
141
+ ranks_spec_dec, _ = get_sampled_token_logprobs(logprob_tensor,
142
+ sampled_token_tensor)
143
+
144
+ ranks_regular = _get_ranks(logprob_tensor.reshape((2, -1)),
145
+ sampled_token_tensor.reshape(-1))
146
+
147
+ assert torch.equal(ranks_spec_dec.reshape(-1), ranks_regular)
vllm/spec_decode/util.py
+2
-2
@@ -43,8 +43,8 @@ def get_sampled_token_logprobs(
43
sampled_token_ids, ]
44
expanded_selected_logprobs = selected_logprobs.unsqueeze(-1).expand(
45
-1, -1, vocab_size)
46
- sampled_token_ids_ranks = (logprob_tensor >=
47
- expanded_selected_logprobs).sum(-1)
46
+ sampled_token_ids_ranks = (logprob_tensor >
47
+ expanded_selected_logprobs).sum(-1).add_(1)
48
49
return sampled_token_ids_ranks, selected_logprobs
50