A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[SpecDecode] [Minor] Fix spec decode sampler tests (#7183)
Lily Liu committed
Aug 6, 2024 at 10:40 UTC
5c60c8c423197bcf20fdc5217d79b78532033f04
3 files changed
+22
-19
tests/samplers/test_rejection_sampler.py
+7
-7
@@ -25,7 +25,7 @@ def mock_causal_accepted_tensor(
25
26
accepted = (torch.arange(k).expand(batch_size, k) <=
27
last_accepted_indices.unsqueeze(-1).broadcast_to(
28
- batch_size, k)).to(device="cuda")
28
+ batch_size, k))
29
30
# Sprinkle accepted values after the contiguous initial accepted values.
31
# This replicates the behavior of rejection sampling, which may "accept"
@@ -33,7 +33,7 @@ def mock_causal_accepted_tensor(
33
sprinkle_candidates = (
34
torch.arange(k).expand(batch_size, k) >
35
last_accepted_indices.unsqueeze(-1).broadcast_to(batch_size, k) + 1)
36
- sprinkle = torch.rand(batch_size, k, device="cuda") > 0.5
36
+ sprinkle = torch.rand(batch_size, k) > 0.5
37
accepted[sprinkle_candidates] = sprinkle[sprinkle_candidates]
38
return accepted
39
@@ -86,7 +86,7 @@ def test_correct_output_format(which_tokens_accepted: str,
86
87
rejection_sampler = RejectionSampler(
88
disable_bonus_tokens=disable_bonus_tokens)
89
- rejection_sampler.init_gpu_tensors(rank=0)
89
+ rejection_sampler.init_gpu_tensors(device=device)
90
output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
91
accepted,
92
recovered_token_ids,
@@ -138,7 +138,7 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
138
device: str):
139
torch.set_default_device(device)
140
rejection_sampler = RejectionSampler()
141
- rejection_sampler.init_gpu_tensors(rank=0)
141
+ rejection_sampler.init_gpu_tensors(device=device)
142
143
draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
144
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -167,7 +167,7 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
167
device: str):
168
torch.set_default_device(device)
169
rejection_sampler = RejectionSampler()
170
- rejection_sampler.init_gpu_tensors(rank=0)
170
+ rejection_sampler.init_gpu_tensors(device=device)
171
172
draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
173
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -211,7 +211,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
211
torch.set_default_device(device)
212
213
rejection_sampler = RejectionSampler(strict_mode=True)
214
- rejection_sampler.init_gpu_tensors(rank=0)
214
+ rejection_sampler.init_gpu_tensors(device=device)
215
216
draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
217
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -339,7 +339,7 @@ class _CorrectnessTestHelper:
339
self.vocab_size = vocab_size
340
self.vocab_range = (0, vocab_size)
341
342
- self.rejection_sampler.init_gpu_tensors(rank=0)
342
+ self.rejection_sampler.init_gpu_tensors(device=0)
343
344
# Keep test simple, use k=1
345
self.k = 1
tests/samplers/test_typical_acceptance_sampler.py
+9
-9
@@ -78,7 +78,7 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
78
"""
79
torch.set_default_device(device)
80
typical_acceptance_sampler = get_acceptance_sampler()
81
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
81
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
82
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
83
bonus_token_ids = torch.randint(low=0,
84
high=vocab_size,
@@ -111,7 +111,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
111
vocab_size = 30_000
112
torch.set_default_device(device)
113
typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
114
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
114
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
115
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
116
bonus_token_ids = torch.randint(low=0,
117
high=vocab_size,
@@ -171,7 +171,7 @@ def test_uniform_target_distribution_accepts_all_tokens(
171
torch.set_default_device(device)
172
typical_acceptance_sampler = get_acceptance_sampler(
173
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
174
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
174
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
175
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
176
draft_token_ids = torch.randint(low=0,
177
high=vocab_size,
@@ -225,7 +225,7 @@ def test_temperature_zero_target_distribution(seed: int,
225
226
typical_acceptance_sampler = get_acceptance_sampler(
227
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
228
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
228
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
229
# Simulate temperature 0 probability distribution for target probabilities
230
# and create target probabilities such that only 1 token id has
231
# probability 1.0
@@ -285,7 +285,7 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
285
torch.set_default_device(device)
286
typical_acceptance_sampler = get_acceptance_sampler(
287
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
288
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
288
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
289
# For sequences 0 and 2 set the distribution to a temperature
290
# zero distribution. For sequences 1 and 3 set it to a uniform
291
# distribution.
@@ -352,7 +352,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
352
torch.set_default_device(device)
353
typical_acceptance_sampler = get_acceptance_sampler(
354
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
355
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
355
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
356
# Create a temperature zero target probability distribution and ensure
357
# all draft token ids correspond to the tokens with 1.0 probability.
358
# Verify that all of them are accepted.
@@ -414,7 +414,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
414
torch.set_default_device(device)
415
typical_acceptance_sampler = get_acceptance_sampler(
416
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
417
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
417
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
418
# Simulate temperature 0 probability distribution for target
419
# probabilities and create target probabilities such that only 1 token
420
# id has probability 1.0 and others have a very low probability of
@@ -447,7 +447,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
447
disable_bonus_tokens=disable_bonus_tokens,
448
posterior_threshold=0.0,
449
posterior_alpha=0.0)
450
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
450
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
451
output_token_ids = typical_acceptance_sampler(
452
target_probs,
453
bonus_token_ids,
@@ -485,7 +485,7 @@ def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
485
torch.set_default_device(device)
486
typical_acceptance_sampler = get_acceptance_sampler(
487
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
488
- typical_acceptance_sampler.init_gpu_tensors(rank=0)
488
+ typical_acceptance_sampler.init_gpu_tensors(device=device)
489
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
490
expected_replacement_tokens = -torch.ones(
491
(batch_size, k), dtype=torch.long)
vllm/model_executor/layers/spec_decode_base_sampler.py
+6
-3
@@ -1,5 +1,5 @@
1
from abc import abstractmethod
2
-from typing import Dict, Optional
2
+from typing import Dict, Optional, Union
3
4
import torch
5
import torch.jit
@@ -36,9 +36,12 @@ class SpecDecodeBaseSampler(nn.Module):
36
self.num_emitted_tokens: Optional[torch.Tensor] = None
37
self.num_draft_tokens: int = 0
38
39
- def init_gpu_tensors(self, rank: int) -> None:
39
+ def init_gpu_tensors(self, device: Union[int, str]) -> None:
40
assert self.num_accepted_tokens is None
41
- device = f"cuda:{rank}"
41
+ if isinstance(device, int):
42
+ device = f"cuda:{device}"
43
+ elif not isinstance(device, str):
44
+ raise ValueError(f"Device must be int or str, get {type(device)}")
45
self.num_accepted_tokens = torch.tensor(0,
46
dtype=torch.long,
47
device=device)