@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f205c09854
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Unify rank computation across regular decoding and speculative decoding (#7899)

Jonas M. Kübler committed Aug 29, 2024 at 07:18 UTC f205c09854853172a446c92aa81eb7199da324ab
2 files changed +22 -3
tests/spec_decode/test_utils.py
+20 -1
@@ -4,10 +4,12 @@ import pytest
4 import torch
5
6 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7 +from vllm.model_executor.layers.sampler import _get_ranks
8 from vllm.model_executor.layers.typical_acceptance_sampler import (
9 TypicalAcceptanceSampler)
10 from vllm.sequence import SequenceGroupMetadata, get_all_seq_ids
10 -from vllm.spec_decode.util import split_batch_by_proposal_len
11 +from vllm.spec_decode.util import (get_sampled_token_logprobs,
12 + split_batch_by_proposal_len)
13
14
15 def test_get_all_seq_ids():
@@ -126,3 +128,20 @@ def mock_spec_decode_sampler(acceptance_sampler_method):
128 return sampler
129 else:
130 raise ValueError(f"Invalid sampler name {acceptance_sampler_method}")
131 +
132 +
133 +def test_get_sampled_token_logprobs():
134 + """Verify get_sampled_token_logprobs returns consistent rankings
135 + with regular get_ranks when probabilities match exactly.
136 + """
137 + logprob_tensor = torch.tensor(
138 + [[[-.1, -.1]] * 2]) # shape (num_steps, batch_size, vocab_size)
139 + sampled_token_tensor = torch.tensor([[1,
140 + 0]]) # shape (num_steps, batch_size)
141 + ranks_spec_dec, _ = get_sampled_token_logprobs(logprob_tensor,
142 + sampled_token_tensor)
143 +
144 + ranks_regular = _get_ranks(logprob_tensor.reshape((2, -1)),
145 + sampled_token_tensor.reshape(-1))
146 +
147 + assert torch.equal(ranks_spec_dec.reshape(-1), ranks_regular)
vllm/spec_decode/util.py
+2 -2
@@ -43,8 +43,8 @@ def get_sampled_token_logprobs(
43 sampled_token_ids, ]
44 expanded_selected_logprobs = selected_logprobs.unsqueeze(-1).expand(
45 -1, -1, vocab_size)
46 - sampled_token_ids_ranks = (logprob_tensor >=
47 - expanded_selected_logprobs).sum(-1)
46 + sampled_token_ids_ranks = (logprob_tensor >
47 + expanded_selected_logprobs).sum(-1).add_(1)
48
49 return sampled_token_ids_ranks, selected_logprobs
50