@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 86c2d8fd1c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix spec decoding when seed is none in a batch (#10863)

Signed-off-by: Wallas Santos <wallashss@ibm.com>

Wallas Henrique committed Dec 20, 2024 at 02:15 UTC 86c2d8fd1cb27e607928ca8c92fa20d9694d2e4b
2 files changed +66 -7
tests/samplers/test_rejection_sampler.py
+63
@@ -200,6 +200,69 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
200 assert torch.equal(results[j][i], results[0][i])
201
202
203 +@pytest.mark.parametrize("k", [1, 3, 6])
204 +@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
205 +@pytest.mark.parametrize("batch_size", [3, 8, 32, 128])
206 +@pytest.mark.parametrize("device", CUDA_DEVICES)
207 +@pytest.mark.parametrize("use_flashinfer", [True, False])
208 +@torch.inference_mode()
209 +def test_mixed_seeded_batch(k: int, vocab_size: int, batch_size: int,
210 + device: str, use_flashinfer: bool):
211 + torch.set_default_device(device)
212 + set_random_seed(0)
213 + draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
214 + target_probs = torch.rand(batch_size,
215 + k + 1,
216 + vocab_size,
217 + dtype=torch.float32)
218 + bonus_token_ids = torch.randint(low=0,
219 + high=vocab_size,
220 + size=(batch_size, 1),
221 + dtype=torch.int64)
222 + draft_token_ids = torch.randint(low=0,
223 + high=vocab_size,
224 + size=(batch_size, k),
225 + dtype=torch.int64)
226 +
227 + single_batches = []
228 + for i in range(batch_size):
229 + single_batches.append((draft_probs[i].clone().unsqueeze(0),
230 + draft_token_ids[i].clone().unsqueeze(0),
231 + target_probs[i].clone().unsqueeze(0),
232 + bonus_token_ids[i].clone().unsqueeze(0),
233 + draft_token_ids[i].clone().unsqueeze(0)))
234 +
235 + set_random_seed(0)
236 + rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
237 + rejection_sampler.init_gpu_tensors(device=device)
238 +
239 + results = []
240 + seeded_seqs = {
241 + i: torch.Generator(device=device).manual_seed(i)
242 + for i in range(1, batch_size) # 0 is seed None
243 + }
244 + batch_result = rejection_sampler(target_probs.clone(),
245 + bonus_token_ids.clone(),
246 + draft_probs.clone(),
247 + draft_token_ids.clone(), seeded_seqs)
248 +
249 + set_random_seed(0)
250 +
251 + rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
252 + rejection_sampler.init_gpu_tensors(device=device)
253 + for i in range(batch_size):
254 + request_seeded_seqs = {
255 + 0: torch.Generator(device=device).manual_seed(i)
256 + } if seeded_seqs.get(i) is not None else None
257 + (draft_probs, draft_token_ids, target_probs, bonus_token_ids,
258 + draft_token_ids) = single_batches[i]
259 + results.append(
260 + rejection_sampler(target_probs, bonus_token_ids, draft_probs,
261 + draft_token_ids, request_seeded_seqs))
262 + for i in range(batch_size):
263 + assert torch.equal(batch_result[i], results[i].squeeze(0))
264 +
265 +
266 @pytest.mark.parametrize("k", [1, 3, 6])
267 @pytest.mark.parametrize("vocab_size", [30_000, 50_000])
268 @pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
vllm/model_executor/layers/rejection_sampler.py
+3 -7
@@ -1,6 +1,6 @@
1 from functools import cached_property
2 from importlib.util import find_spec
3 -from typing import Dict, List, Optional, Tuple
3 +from typing import Dict, Optional, Tuple
4
5 import torch
6 import torch.jit
@@ -386,16 +386,12 @@ def _multinomial(
386 if not seeded_seqs:
387 q.exponential_(1.0)
388 else:
389 - non_seeded_indices: List[int] = []
389 start = 0
390 for idx in range(len(q) // k):
391 end = start + k
392 generator = seeded_seqs.get(idx)
394 - if generator is None:
395 - non_seeded_indices.extend(list(range(start, end)))
396 - else:
397 - q[start:end].exponential_(1.0, generator=generator)
393 + # Note: generator might be None for non seeded
394 + q[start:end].exponential_(1.0, generator=generator)
395 start = end
399 - q[non_seeded_indices].exponential_(1.0)
396
397 return probs.div_(q).argmax(dim=1).view(-1, num_samples)