@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5c60c8c423
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[SpecDecode] [Minor] Fix spec decode sampler tests (#7183)

Lily Liu committed Aug 6, 2024 at 10:40 UTC 5c60c8c423197bcf20fdc5217d79b78532033f04
3 files changed +22 -19
tests/samplers/test_rejection_sampler.py
+7 -7
@@ -25,7 +25,7 @@ def mock_causal_accepted_tensor(
25
26 accepted = (torch.arange(k).expand(batch_size, k) <=
27 last_accepted_indices.unsqueeze(-1).broadcast_to(
28 - batch_size, k)).to(device="cuda")
28 + batch_size, k))
29
30 # Sprinkle accepted values after the contiguous initial accepted values.
31 # This replicates the behavior of rejection sampling, which may "accept"
@@ -33,7 +33,7 @@ def mock_causal_accepted_tensor(
33 sprinkle_candidates = (
34 torch.arange(k).expand(batch_size, k) >
35 last_accepted_indices.unsqueeze(-1).broadcast_to(batch_size, k) + 1)
36 - sprinkle = torch.rand(batch_size, k, device="cuda") > 0.5
36 + sprinkle = torch.rand(batch_size, k) > 0.5
37 accepted[sprinkle_candidates] = sprinkle[sprinkle_candidates]
38 return accepted
39
@@ -86,7 +86,7 @@ def test_correct_output_format(which_tokens_accepted: str,
86
87 rejection_sampler = RejectionSampler(
88 disable_bonus_tokens=disable_bonus_tokens)
89 - rejection_sampler.init_gpu_tensors(rank=0)
89 + rejection_sampler.init_gpu_tensors(device=device)
90 output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
91 accepted,
92 recovered_token_ids,
@@ -138,7 +138,7 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
138 device: str):
139 torch.set_default_device(device)
140 rejection_sampler = RejectionSampler()
141 - rejection_sampler.init_gpu_tensors(rank=0)
141 + rejection_sampler.init_gpu_tensors(device=device)
142
143 draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
144 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -167,7 +167,7 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
167 device: str):
168 torch.set_default_device(device)
169 rejection_sampler = RejectionSampler()
170 - rejection_sampler.init_gpu_tensors(rank=0)
170 + rejection_sampler.init_gpu_tensors(device=device)
171
172 draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
173 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -211,7 +211,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
211 torch.set_default_device(device)
212
213 rejection_sampler = RejectionSampler(strict_mode=True)
214 - rejection_sampler.init_gpu_tensors(rank=0)
214 + rejection_sampler.init_gpu_tensors(device=device)
215
216 draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
217 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -339,7 +339,7 @@ class _CorrectnessTestHelper:
339 self.vocab_size = vocab_size
340 self.vocab_range = (0, vocab_size)
341
342 - self.rejection_sampler.init_gpu_tensors(rank=0)
342 + self.rejection_sampler.init_gpu_tensors(device=0)
343
344 # Keep test simple, use k=1
345 self.k = 1
tests/samplers/test_typical_acceptance_sampler.py
+9 -9
@@ -78,7 +78,7 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
78 """
79 torch.set_default_device(device)
80 typical_acceptance_sampler = get_acceptance_sampler()
81 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
81 + typical_acceptance_sampler.init_gpu_tensors(device=device)
82 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
83 bonus_token_ids = torch.randint(low=0,
84 high=vocab_size,
@@ -111,7 +111,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
111 vocab_size = 30_000
112 torch.set_default_device(device)
113 typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
114 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
114 + typical_acceptance_sampler.init_gpu_tensors(device=device)
115 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
116 bonus_token_ids = torch.randint(low=0,
117 high=vocab_size,
@@ -171,7 +171,7 @@ def test_uniform_target_distribution_accepts_all_tokens(
171 torch.set_default_device(device)
172 typical_acceptance_sampler = get_acceptance_sampler(
173 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
174 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
174 + typical_acceptance_sampler.init_gpu_tensors(device=device)
175 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
176 draft_token_ids = torch.randint(low=0,
177 high=vocab_size,
@@ -225,7 +225,7 @@ def test_temperature_zero_target_distribution(seed: int,
225
226 typical_acceptance_sampler = get_acceptance_sampler(
227 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
228 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
228 + typical_acceptance_sampler.init_gpu_tensors(device=device)
229 # Simulate temperature 0 probability distribution for target probabilities
230 # and create target probabilities such that only 1 token id has
231 # probability 1.0
@@ -285,7 +285,7 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
285 torch.set_default_device(device)
286 typical_acceptance_sampler = get_acceptance_sampler(
287 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
288 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
288 + typical_acceptance_sampler.init_gpu_tensors(device=device)
289 # For sequences 0 and 2 set the distribution to a temperature
290 # zero distribution. For sequences 1 and 3 set it to a uniform
291 # distribution.
@@ -352,7 +352,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
352 torch.set_default_device(device)
353 typical_acceptance_sampler = get_acceptance_sampler(
354 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
355 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
355 + typical_acceptance_sampler.init_gpu_tensors(device=device)
356 # Create a temperature zero target probability distribution and ensure
357 # all draft token ids correspond to the tokens with 1.0 probability.
358 # Verify that all of them are accepted.
@@ -414,7 +414,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
414 torch.set_default_device(device)
415 typical_acceptance_sampler = get_acceptance_sampler(
416 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
417 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
417 + typical_acceptance_sampler.init_gpu_tensors(device=device)
418 # Simulate temperature 0 probability distribution for target
419 # probabilities and create target probabilities such that only 1 token
420 # id has probability 1.0 and others have a very low probability of
@@ -447,7 +447,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
447 disable_bonus_tokens=disable_bonus_tokens,
448 posterior_threshold=0.0,
449 posterior_alpha=0.0)
450 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
450 + typical_acceptance_sampler.init_gpu_tensors(device=device)
451 output_token_ids = typical_acceptance_sampler(
452 target_probs,
453 bonus_token_ids,
@@ -485,7 +485,7 @@ def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
485 torch.set_default_device(device)
486 typical_acceptance_sampler = get_acceptance_sampler(
487 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
488 - typical_acceptance_sampler.init_gpu_tensors(rank=0)
488 + typical_acceptance_sampler.init_gpu_tensors(device=device)
489 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
490 expected_replacement_tokens = -torch.ones(
491 (batch_size, k), dtype=torch.long)
vllm/model_executor/layers/spec_decode_base_sampler.py
+6 -3
@@ -1,5 +1,5 @@
1 from abc import abstractmethod
2 -from typing import Dict, Optional
2 +from typing import Dict, Optional, Union
3
4 import torch
5 import torch.jit
@@ -36,9 +36,12 @@ class SpecDecodeBaseSampler(nn.Module):
36 self.num_emitted_tokens: Optional[torch.Tensor] = None
37 self.num_draft_tokens: int = 0
38
39 - def init_gpu_tensors(self, rank: int) -> None:
39 + def init_gpu_tensors(self, device: Union[int, str]) -> None:
40 assert self.num_accepted_tokens is None
41 - device = f"cuda:{rank}"
41 + if isinstance(device, int):
42 + device = f"cuda:{device}"
43 + elif not isinstance(device, str):
44 + raise ValueError(f"Device must be int or str, get {type(device)}")
45 self.num_accepted_tokens = torch.tensor(0,
46 dtype=torch.long,
47 device=device)