@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d4201e06d5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Make spec. decode respect per-request seed. (#6034)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Thomas Parnell committed Jul 19, 2024 at 04:22 UTC d4201e06d5ec3384e06b20816ad6b2f1d1fb1441
8 files changed +293 -46
tests/samplers/test_rejection_sampler.py
+53 -3
@@ -150,9 +150,54 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
150 high=vocab_size,
151 size=(batch_size, k),
152 dtype=torch.int64)
153 + generators = [None] * batch_size
154
155 rejection_sampler(target_probs, bonus_token_ids, draft_probs,
155 - draft_token_ids)
156 + draft_token_ids, generators)
157 +
158 +
159 +@pytest.mark.parametrize("frac_seeded", [0.0, 0.25, 0.5, 1.0])
160 +@pytest.mark.parametrize("k", [1, 3, 6])
161 +@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
162 +@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
163 +@pytest.mark.parametrize("n_rep", [100])
164 +@pytest.mark.parametrize("device", CUDA_DEVICES)
165 +@torch.inference_mode()
166 +def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
167 + frac_seeded: float, n_rep: int,
168 + device: str):
169 + torch.set_default_device(device)
170 + rejection_sampler = RejectionSampler()
171 + rejection_sampler.init_gpu_tensors(rank=0)
172 +
173 + draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
174 + target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
175 + bonus_token_ids = torch.randint(low=0,
176 + high=vocab_size,
177 + size=(batch_size, 1),
178 + dtype=torch.int64)
179 + draft_token_ids = torch.randint(low=0,
180 + high=vocab_size,
181 + size=(batch_size, k),
182 + dtype=torch.int64)
183 +
184 + seeded_mask = torch.rand(batch_size, dtype=torch.float32) <= frac_seeded
185 +
186 + results = []
187 + for _ in range(n_rep):
188 + generators = [
189 + torch.Generator(
190 + device=device).manual_seed(i) if seeded_mask[i] else None
191 + for i in range(batch_size)
192 + ]
193 + results.append(
194 + rejection_sampler(target_probs, bonus_token_ids, draft_probs,
195 + draft_token_ids, generators))
196 +
197 + for i in range(batch_size):
198 + if seeded_mask[i]:
199 + for j in range(1, n_rep):
200 + assert torch.equal(results[j][i], results[0][i])
201
202
203 @pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
@@ -197,10 +242,11 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
242 raise AssertionError()
243
244 oob_token_ids[0][0] = rogue_token_id
245 + generators = [None] * batch_size
246
247 with pytest.raises(AssertionError):
248 rejection_sampler(target_probs, bonus_token_ids, draft_probs,
203 - draft_token_ids)
249 + draft_token_ids, generators)
250
251
252 @pytest.mark.parametrize("draft_and_target_probs_equal", [True, False])
@@ -371,11 +417,15 @@ class _CorrectnessTestHelper:
417 dtype=torch.int64,
418 device="cuda").repeat(num_samples, 1)
419
420 + # unseeded
421 + generators = [None]
422 +
423 # Get output tokens via rejection sampling.
424 output_token_ids = self.rejection_sampler(target_probs.to("cuda"),
425 bonus_token_ids.to("cuda"),
426 draft_probs.to("cuda"),
378 - draft_token_ids.to("cuda"))
427 + draft_token_ids.to("cuda"),
428 + generators)
429
430 # Remove bonus tokens
431 output_token_ids = output_token_ids[:, :-1].flatten()
tests/spec_decode/e2e/conftest.py
+44 -8
@@ -1,6 +1,6 @@
1 import asyncio
2 from itertools import cycle
3 -from typing import Dict, List, Optional, Tuple, Union
3 +from typing import Dict, List, Optional, Sequence, Tuple, Union
4
5 import pytest
6 import ray
@@ -128,7 +128,9 @@ class AsyncLLM:
128 try:
129 for i in range(num_requests):
130 prompt = prompts[i] if prompts is not None else None
131 - res = asyncio.run(get_output(prompt, sampling_params))
131 + params = sampling_params[i] if isinstance(
132 + sampling_params, Sequence) else sampling_params
133 + res = asyncio.run(get_output(prompt, params))
134 outputs.append(res)
135 finally:
136 ray.shutdown()
@@ -267,7 +269,31 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
269 the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
270 the same when temperature is zero.
271 """
270 - temperature = 0.0
272 +
273 + run_equality_correctness_test(baseline_llm_generator,
274 + test_llm_generator,
275 + batch_size,
276 + max_output_len,
277 + force_output_len,
278 + temperature=0.0,
279 + seeded=False,
280 + print_tokens=print_tokens,
281 + ensure_all_accepted=ensure_all_accepted)
282 +
283 +
284 +def run_equality_correctness_test(baseline_llm_generator,
285 + test_llm_generator,
286 + batch_size,
287 + max_output_len,
288 + force_output_len: bool,
289 + temperature: float,
290 + seeded: bool,
291 + print_tokens: bool = False,
292 + ensure_all_accepted: bool = False):
293 + """Helper method that compares the outputs of both the baseline LLM and
294 + the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
295 + the same when temperature is zero (or when temperature is > 0 and seeded).
296 + """
297
298 prompts = [
299 "Hello, my name is",
@@ -286,11 +312,21 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
312 # sampling params to ignore eos token.
313 ignore_eos = force_output_len
314
289 - sampling_params = SamplingParams(
290 - max_tokens=max_output_len,
291 - ignore_eos=ignore_eos,
292 - temperature=temperature,
293 - )
315 + if seeded:
316 + sampling_params = [
317 + SamplingParams(
318 + max_tokens=max_output_len,
319 + ignore_eos=ignore_eos,
320 + temperature=temperature,
321 + seed=i,
322 + ) for i in range(len(prompts))
323 + ]
324 + else:
325 + sampling_params = SamplingParams(
326 + max_tokens=max_output_len,
327 + ignore_eos=ignore_eos,
328 + temperature=temperature,
329 + )
330
331 (spec_batch_tokens, spec_batch_token_ids,
332 acceptance_rate) = get_output_from_llm_generator(test_llm_generator,
tests/spec_decode/e2e/test_seed.py new
+44
@@ -0,0 +1,44 @@
1 +import pytest
2 +
3 +from .conftest import run_equality_correctness_test
4 +
5 +
6 +@pytest.mark.parametrize(
7 + "common_llm_kwargs",
8 + [{
9 + "model": "JackFram/llama-68m",
10 +
11 + # Skip cuda graph recording for fast test.
12 + "enforce_eager": True,
13 +
14 + # Required for spec decode.
15 + "use_v2_block_manager": True,
16 +
17 + # speculative model
18 + "speculative_model": "JackFram/llama-160m",
19 +
20 + # num speculative tokens
21 + "num_speculative_tokens": 3,
22 + }])
23 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
24 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
25 +@pytest.mark.parametrize("batch_size", [1, 8, 32])
26 +@pytest.mark.parametrize("temperature", [0.1, 1.0])
27 +@pytest.mark.parametrize(
28 + "output_len",
29 + [
30 + # Use smaller output len for fast test.
31 + 10,
32 + ])
33 +@pytest.mark.parametrize("seed", [1])
34 +def test_seeded_consistency(baseline_llm_generator, batch_size: int,
35 + temperature: float, output_len: int):
36 + """Verify outputs are consistent across multiple runs with same seed
37 + """
38 + run_equality_correctness_test(baseline_llm_generator,
39 + baseline_llm_generator,
40 + batch_size,
41 + max_output_len=output_len,
42 + temperature=temperature,
43 + seeded=True,
44 + force_output_len=True)
vllm/model_executor/layers/rejection_sampler.py
+86 -20
@@ -1,14 +1,14 @@
1 from functools import cached_property
2 -from typing import Tuple
2 +from typing import List, Optional, Tuple
3
4 import torch
5 import torch.jit
6
7 from vllm.model_executor.layers.spec_decode_base_sampler import (
8 - SpecDecodeBaseSampler)
8 + SpecDecodeStochasticBaseSampler)
9
10
11 -class RejectionSampler(SpecDecodeBaseSampler):
11 +class RejectionSampler(SpecDecodeStochasticBaseSampler):
12 """Apply modified rejection sampling as described in "Accelerating Large
13 Language Model Decoding with Speculative Sampling"
14 https://arxiv.org/pdf/2302.01318.pdf.
@@ -36,6 +36,7 @@ class RejectionSampler(SpecDecodeBaseSampler):
36 bonus_token_ids: torch.Tensor,
37 draft_probs: torch.Tensor,
38 draft_token_ids: torch.Tensor,
39 + generators: List[Optional[torch.Generator]],
40 ) -> torch.Tensor:
41 """Sample token ids using rejection sampling. This accepts or rejects
42 tokens proposed by the draft model using the probability of each token
@@ -82,6 +83,7 @@ class RejectionSampler(SpecDecodeBaseSampler):
83 target_probs,
84 draft_probs,
85 draft_token_ids,
86 + generators,
87 ))
88
89 output_token_ids = self._create_output(
@@ -94,10 +96,11 @@ class RejectionSampler(SpecDecodeBaseSampler):
96 return output_token_ids
97
98 def _batch_modified_rejection_sampling(
97 - self,
98 - target_probs: torch.Tensor, # [batch_size, k, vocab_size]
99 - draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
100 - draft_token_ids: torch.Tensor, # [batch_size, k]
99 + self,
100 + target_probs: torch.Tensor, # [batch_size, k, vocab_size]
101 + draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
102 + draft_token_ids: torch.Tensor, # [batch_size, k]
103 + generators: List[Optional[torch.Generator]],
104 ) -> Tuple[torch.Tensor, torch.Tensor]:
105 """Perform modified rejection sampling on each sequence.
106
@@ -114,22 +117,33 @@ class RejectionSampler(SpecDecodeBaseSampler):
117
118 # shape [batch_size, k]
119 accepted = self._get_accepted(target_probs, draft_probs,
117 - draft_token_ids)
120 + draft_token_ids, generators)
121
122 recovered_probs = self._get_recovered_probs(
123 target_probs, draft_probs).reshape(batch_size * k, vocab_size)
124
125 + seed_indices, non_seed_indices = self._split_batch_by_seeded(
126 + generators, k=k)
127 +
128 # NOTE: the recovered_probs are overwritten by this method.
123 - recovered_token_ids = _multinomial(recovered_probs,
124 - num_samples=1).reshape(
125 - batch_size, k)
129 + recovered_token_ids = _multinomial(
130 + recovered_probs,
131 + num_samples=1,
132 + k=k,
133 + generators=generators,
134 + seed_indices=seed_indices,
135 + # this arg is unused when None but torch.jit requires a list
136 + non_seed_indices=non_seed_indices or [],
137 + ).reshape(batch_size, k)
138 +
139 return accepted, recovered_token_ids
140
141 def _get_accepted(
129 - self,
130 - target_probs: torch.Tensor, # [batch_size, k, vocab_size]
131 - draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
132 - draft_token_ids: torch.Tensor, # [batch_size, k]
142 + self,
143 + target_probs: torch.Tensor, # [batch_size, k, vocab_size]
144 + draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
145 + draft_token_ids: torch.Tensor, # [batch_size, k]
146 + generators: List[Optional[torch.Generator]],
147 ) -> torch.Tensor:
148 r"""Create bool matrix over the proposed draft tokens. If
149 True, then a token can be accepted, else it should be
@@ -164,10 +178,28 @@ class RejectionSampler(SpecDecodeBaseSampler):
178 selected_target_probs = target_probs[batch_indices, probs_indicies,
179 draft_token_ids]
180
167 - uniform_rand = torch.rand(batch_size,
168 - k,
169 - dtype=self.probs_dtype,
170 - device=target_probs.device)
181 + seed_indices, non_seed_indices = self._split_batch_by_seeded(
182 + generators)
183 +
184 + if len(seed_indices) == 0:
185 + uniform_rand = torch.rand_like(selected_target_probs)
186 + else:
187 + uniform_rand = torch.empty_like(selected_target_probs)
188 +
189 + for idx in seed_indices:
190 + uniform_rand[idx, :] = torch.rand(1,
191 + k,
192 + dtype=self.probs_dtype,
193 + device=target_probs.device,
194 + generator=generators[idx])
195 +
196 + if non_seed_indices:
197 + uniform_rand[non_seed_indices, :] = torch.rand(
198 + len(non_seed_indices),
199 + k,
200 + dtype=self.probs_dtype,
201 + device=target_probs.device)
202 +
203 capped_ratio = torch.minimum(
204 selected_target_probs / selected_draft_probs,
205 torch.full((1, ), 1, device=target_probs.device))
@@ -240,6 +272,27 @@ class RejectionSampler(SpecDecodeBaseSampler):
272 """
273 return torch.finfo(self.probs_dtype).tiny
274
275 + # partition batch into indices for which a generator is provided
276 + # and indicies for which no generator is provided
277 + @staticmethod
278 + def _split_batch_by_seeded(
279 + generators: List[Optional[torch.Generator]],
280 + k: int = 1,
281 + ) -> Tuple[List[int], Optional[List[int]]]:
282 +
283 + if all(generator is None for generator in generators):
284 + seed_indices: List[int] = []
285 + non_seed_indices: Optional[List[int]] = None
286 + else:
287 + seed_indices, non_seed_indices = [], []
288 + for i, generator in enumerate(generators):
289 + if generator is None:
290 + non_seed_indices.extend(range(k * i, k * (i + 1)))
291 + else:
292 + seed_indices.extend(range(k * i, k * (i + 1)))
293 +
294 + return seed_indices, non_seed_indices
295 +
296
297 # torch.multinomial forces a GPU<->CPU sync.
298 # Therefore, we use an optimized implementation instead that skips the sync.
@@ -250,12 +303,25 @@ class RejectionSampler(SpecDecodeBaseSampler):
303 def _multinomial(
304 probs: torch.Tensor,
305 num_samples: int,
306 + k: int,
307 + generators: List[Optional[torch.Generator]],
308 + seed_indices: List[int],
309 + non_seed_indices: List[int],
310 ) -> torch.Tensor:
311 +
312 if num_samples > 1:
313 # This is equivalent to torch.repeat_interleaved (which also
314 # forces a GPU<->CPU sync).
315 probs = probs[:, None, :].expand(probs.shape[0], num_samples,
316 probs.shape[1]).contiguous().view(
317 -1, probs.shape[1])
260 - q = torch.empty_like(probs).exponential_(1.0)
318 +
319 + q = torch.empty_like(probs)
320 + if len(seed_indices) == 0:
321 + q.exponential_(1.0)
322 + else:
323 + q[non_seed_indices].exponential_(1.0)
324 + for idx in seed_indices:
325 + q[idx].exponential_(1.0, generator=generators[idx // k])
326 +
327 return probs.div_(q).argmax(dim=1).view(-1, num_samples)
vllm/model_executor/layers/spec_decode_base_sampler.py
+34 -11
@@ -1,5 +1,5 @@
1 from abc import abstractmethod
2 -from typing import Optional
2 +from typing import List, Optional
3
4 import torch
5 import torch.jit
@@ -54,16 +54,6 @@ class SpecDecodeBaseSampler(nn.Module):
54 def token_id_dtype(self):
55 return torch.int64
56
57 - @abstractmethod
58 - def forward(
59 - self,
60 - target_probs: torch.Tensor,
61 - bonus_token_ids: torch.Tensor,
62 - draft_probs: torch.Tensor,
63 - draft_token_ids: torch.Tensor,
64 - ) -> torch.Tensor:
65 - raise NotImplementedError
66 -
57 def _create_output(
58 self,
59 accepted: torch.Tensor, # [batch_size, k]
@@ -217,3 +207,36 @@ class SpecDecodeBaseSampler(nn.Module):
207 assert torch.all(bonus_token_ids >= 0)
208 assert torch.all(draft_token_ids < vocab_size)
209 assert torch.all(draft_token_ids >= 0)
210 +
211 +
212 +class SpecDecodeDeterministicBaseSampler(SpecDecodeBaseSampler):
213 + """Base class for samplers used for Speculative Decoding verification
214 + step which are deterministic.
215 + """
216 +
217 + @abstractmethod
218 + def forward(
219 + self,
220 + target_probs: torch.Tensor,
221 + bonus_token_ids: torch.Tensor,
222 + draft_probs: torch.Tensor,
223 + draft_token_ids: torch.Tensor,
224 + ) -> torch.Tensor:
225 + raise NotImplementedError
226 +
227 +
228 +class SpecDecodeStochasticBaseSampler(SpecDecodeBaseSampler):
229 + """Base class for samplers used for Speculative Decoding verification
230 + step which are stochastic
231 + """
232 +
233 + @abstractmethod
234 + def forward(
235 + self,
236 + target_probs: torch.Tensor,
237 + bonus_token_ids: torch.Tensor,
238 + draft_probs: torch.Tensor,
239 + draft_token_ids: torch.Tensor,
240 + generators: List[Optional[torch.Generator]],
241 + ) -> torch.Tensor:
242 + raise NotImplementedError
vllm/model_executor/layers/typical_acceptance_sampler.py
+2 -2
@@ -2,10 +2,10 @@ import torch
2 import torch.jit
3
4 from vllm.model_executor.layers.spec_decode_base_sampler import (
5 - SpecDecodeBaseSampler)
5 + SpecDecodeDeterministicBaseSampler)
6
7
8 -class TypicalAcceptanceSampler(SpecDecodeBaseSampler):
8 +class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
9 """Apply typical acceptance sampling as described in section 3.3.1 in
10 "MEDUSA: Simple LLM Inference Acceleration Framework with
11 Multiple Decoding Heads"
vllm/spec_decode/batch_expansion.py
+12 -1
@@ -4,7 +4,8 @@ from typing import Iterator, List, Tuple
4 import torch
5
6 from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
7 - SequenceGroupMetadata, get_all_seq_ids)
7 + SequenceGroupMetadata, SequenceGroupState,
8 + get_all_seq_ids)
9 from vllm.spec_decode.interfaces import (SpeculativeProposals,
10 SpeculativeScorer, SpeculativeScores)
11 from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
@@ -292,6 +293,15 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
293 for data in new_seq_data_dict.values():
294 data.update_num_computed_tokens(data.get_len() - 1)
295
296 + if (seq_group_metadata.state is not None
297 + and seq_group_metadata.state.generator is not None):
298 + generator = torch.Generator(
299 + device=seq_group_metadata.state.generator.device)
300 + generator.set_state(seq_group_metadata.state.generator.get_state())
301 + state = SequenceGroupState(generator=generator)
302 + else:
303 + state = None
304 +
305 return SequenceGroupMetadata(
306 request_id=seq_group_metadata.request_id,
307 is_prompt=seq_group_metadata.is_prompt,
@@ -302,6 +312,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
312 },
313 lora_request=None,
314 token_chunk_size=1,
315 + state=state,
316 )
317
318 def _split_scoring_output(
vllm/spec_decode/spec_decode_worker.py
+18 -1
@@ -9,7 +9,7 @@ from vllm.distributed.communication_op import broadcast_tensor_dict
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11 from vllm.model_executor.layers.spec_decode_base_sampler import (
12 - SpecDecodeBaseSampler)
12 + SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
13 from vllm.model_executor.layers.typical_acceptance_sampler import (
14 TypicalAcceptanceSampler)
15 from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
@@ -521,11 +521,28 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
521 # Get proposed tokens.
522 proposal_token_ids = proposals.proposal_token_ids[spec_indices]
523
524 + # Sampler arguments
525 + sampler_extra_kwargs = {}
526 + if isinstance(self.spec_decode_sampler,
527 + SpecDecodeStochasticBaseSampler):
528 +
529 + # Get sequence group state
530 + generators = []
531 + for seq_group_metadata in seq_group_metadata_list:
532 + if (seq_group_metadata.state is not None
533 + and seq_group_metadata.state.generator is not None):
534 + generators.append(seq_group_metadata.state.generator)
535 + else:
536 + generators.append(None)
537 +
538 + sampler_extra_kwargs["generators"] = generators
539 +
540 accepted_token_ids = self.spec_decode_sampler(
541 target_probs=proposal_verifier_probs,
542 bonus_token_ids=bonus_token_ids,
543 draft_probs=proposal_probs,
544 draft_token_ids=proposal_token_ids,
545 + **sampler_extra_kwargs,
546 )
547
548 # Append output tokens from non-speculative sequences to