A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Make spec. decode respect per-request seed. (#6034)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Thomas Parnell committed
Jul 19, 2024 at 04:22 UTC
d4201e06d5ec3384e06b20816ad6b2f1d1fb1441
8 files changed
+293
-46
tests/samplers/test_rejection_sampler.py
+53
-3
@@ -150,9 +150,54 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
150
high=vocab_size,
151
size=(batch_size, k),
152
dtype=torch.int64)
153
+ generators = [None] * batch_size
154
155
rejection_sampler(target_probs, bonus_token_ids, draft_probs,
155
- draft_token_ids)
156
+ draft_token_ids, generators)
157
+
158
+
159
+@pytest.mark.parametrize("frac_seeded", [0.0, 0.25, 0.5, 1.0])
160
+@pytest.mark.parametrize("k", [1, 3, 6])
161
+@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
162
+@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
163
+@pytest.mark.parametrize("n_rep", [100])
164
+@pytest.mark.parametrize("device", CUDA_DEVICES)
165
+@torch.inference_mode()
166
+def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
167
+ frac_seeded: float, n_rep: int,
168
+ device: str):
169
+ torch.set_default_device(device)
170
+ rejection_sampler = RejectionSampler()
171
+ rejection_sampler.init_gpu_tensors(rank=0)
172
+
173
+ draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
174
+ target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
175
+ bonus_token_ids = torch.randint(low=0,
176
+ high=vocab_size,
177
+ size=(batch_size, 1),
178
+ dtype=torch.int64)
179
+ draft_token_ids = torch.randint(low=0,
180
+ high=vocab_size,
181
+ size=(batch_size, k),
182
+ dtype=torch.int64)
183
+
184
+ seeded_mask = torch.rand(batch_size, dtype=torch.float32) <= frac_seeded
185
+
186
+ results = []
187
+ for _ in range(n_rep):
188
+ generators = [
189
+ torch.Generator(
190
+ device=device).manual_seed(i) if seeded_mask[i] else None
191
+ for i in range(batch_size)
192
+ ]
193
+ results.append(
194
+ rejection_sampler(target_probs, bonus_token_ids, draft_probs,
195
+ draft_token_ids, generators))
196
+
197
+ for i in range(batch_size):
198
+ if seeded_mask[i]:
199
+ for j in range(1, n_rep):
200
+ assert torch.equal(results[j][i], results[0][i])
201
202
203
@pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
@@ -197,10 +242,11 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
242
raise AssertionError()
243
244
oob_token_ids[0][0] = rogue_token_id
245
+ generators = [None] * batch_size
246
247
with pytest.raises(AssertionError):
248
rejection_sampler(target_probs, bonus_token_ids, draft_probs,
203
- draft_token_ids)
249
+ draft_token_ids, generators)
250
251
252
@pytest.mark.parametrize("draft_and_target_probs_equal", [True, False])
@@ -371,11 +417,15 @@ class _CorrectnessTestHelper:
417
dtype=torch.int64,
418
device="cuda").repeat(num_samples, 1)
419
420
+ # unseeded
421
+ generators = [None]
422
+
423
# Get output tokens via rejection sampling.
424
output_token_ids = self.rejection_sampler(target_probs.to("cuda"),
425
bonus_token_ids.to("cuda"),
426
draft_probs.to("cuda"),
378
- draft_token_ids.to("cuda"))
427
+ draft_token_ids.to("cuda"),
428
+ generators)
429
430
# Remove bonus tokens
431
output_token_ids = output_token_ids[:, :-1].flatten()
tests/spec_decode/e2e/conftest.py
+44
-8
@@ -1,6 +1,6 @@
1
import asyncio
2
from itertools import cycle
3
-from typing import Dict, List, Optional, Tuple, Union
3
+from typing import Dict, List, Optional, Sequence, Tuple, Union
4
5
import pytest
6
import ray
@@ -128,7 +128,9 @@ class AsyncLLM:
128
try:
129
for i in range(num_requests):
130
prompt = prompts[i] if prompts is not None else None
131
- res = asyncio.run(get_output(prompt, sampling_params))
131
+ params = sampling_params[i] if isinstance(
132
+ sampling_params, Sequence) else sampling_params
133
+ res = asyncio.run(get_output(prompt, params))
134
outputs.append(res)
135
finally:
136
ray.shutdown()
@@ -267,7 +269,31 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
269
the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
270
the same when temperature is zero.
271
"""
270
- temperature = 0.0
272
+
273
+ run_equality_correctness_test(baseline_llm_generator,
274
+ test_llm_generator,
275
+ batch_size,
276
+ max_output_len,
277
+ force_output_len,
278
+ temperature=0.0,
279
+ seeded=False,
280
+ print_tokens=print_tokens,
281
+ ensure_all_accepted=ensure_all_accepted)
282
+
283
+
284
+def run_equality_correctness_test(baseline_llm_generator,
285
+ test_llm_generator,
286
+ batch_size,
287
+ max_output_len,
288
+ force_output_len: bool,
289
+ temperature: float,
290
+ seeded: bool,
291
+ print_tokens: bool = False,
292
+ ensure_all_accepted: bool = False):
293
+ """Helper method that compares the outputs of both the baseline LLM and
294
+ the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
295
+ the same when temperature is zero (or when temperature is > 0 and seeded).
296
+ """
297
298
prompts = [
299
"Hello, my name is",
@@ -286,11 +312,21 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
312
# sampling params to ignore eos token.
313
ignore_eos = force_output_len
314
289
- sampling_params = SamplingParams(
290
- max_tokens=max_output_len,
291
- ignore_eos=ignore_eos,
292
- temperature=temperature,
293
- )
315
+ if seeded:
316
+ sampling_params = [
317
+ SamplingParams(
318
+ max_tokens=max_output_len,
319
+ ignore_eos=ignore_eos,
320
+ temperature=temperature,
321
+ seed=i,
322
+ ) for i in range(len(prompts))
323
+ ]
324
+ else:
325
+ sampling_params = SamplingParams(
326
+ max_tokens=max_output_len,
327
+ ignore_eos=ignore_eos,
328
+ temperature=temperature,
329
+ )
330
331
(spec_batch_tokens, spec_batch_token_ids,
332
acceptance_rate) = get_output_from_llm_generator(test_llm_generator,
tests/spec_decode/e2e/test_seed.py
new
+44
@@ -0,0 +1,44 @@
1
+import pytest
2
+
3
+from .conftest import run_equality_correctness_test
4
+
5
+
6
+@pytest.mark.parametrize(
7
+ "common_llm_kwargs",
8
+ [{
9
+ "model": "JackFram/llama-68m",
10
+
11
+ # Skip cuda graph recording for fast test.
12
+ "enforce_eager": True,
13
+
14
+ # Required for spec decode.
15
+ "use_v2_block_manager": True,
16
+
17
+ # speculative model
18
+ "speculative_model": "JackFram/llama-160m",
19
+
20
+ # num speculative tokens
21
+ "num_speculative_tokens": 3,
22
+ }])
23
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
24
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
25
+@pytest.mark.parametrize("batch_size", [1, 8, 32])
26
+@pytest.mark.parametrize("temperature", [0.1, 1.0])
27
+@pytest.mark.parametrize(
28
+ "output_len",
29
+ [
30
+ # Use smaller output len for fast test.
31
+ 10,
32
+ ])
33
+@pytest.mark.parametrize("seed", [1])
34
+def test_seeded_consistency(baseline_llm_generator, batch_size: int,
35
+ temperature: float, output_len: int):
36
+ """Verify outputs are consistent across multiple runs with same seed
37
+ """
38
+ run_equality_correctness_test(baseline_llm_generator,
39
+ baseline_llm_generator,
40
+ batch_size,
41
+ max_output_len=output_len,
42
+ temperature=temperature,
43
+ seeded=True,
44
+ force_output_len=True)
vllm/model_executor/layers/rejection_sampler.py
+86
-20
@@ -1,14 +1,14 @@
1
from functools import cached_property
2
-from typing import Tuple
2
+from typing import List, Optional, Tuple
3
4
import torch
5
import torch.jit
6
7
from vllm.model_executor.layers.spec_decode_base_sampler import (
8
- SpecDecodeBaseSampler)
8
+ SpecDecodeStochasticBaseSampler)
9
10
11
-class RejectionSampler(SpecDecodeBaseSampler):
11
+class RejectionSampler(SpecDecodeStochasticBaseSampler):
12
"""Apply modified rejection sampling as described in "Accelerating Large
13
Language Model Decoding with Speculative Sampling"
14
https://arxiv.org/pdf/2302.01318.pdf.
@@ -36,6 +36,7 @@ class RejectionSampler(SpecDecodeBaseSampler):
36
bonus_token_ids: torch.Tensor,
37
draft_probs: torch.Tensor,
38
draft_token_ids: torch.Tensor,
39
+ generators: List[Optional[torch.Generator]],
40
) -> torch.Tensor:
41
"""Sample token ids using rejection sampling. This accepts or rejects
42
tokens proposed by the draft model using the probability of each token
@@ -82,6 +83,7 @@ class RejectionSampler(SpecDecodeBaseSampler):
83
target_probs,
84
draft_probs,
85
draft_token_ids,
86
+ generators,
87
))
88
89
output_token_ids = self._create_output(
@@ -94,10 +96,11 @@ class RejectionSampler(SpecDecodeBaseSampler):
96
return output_token_ids
97
98
def _batch_modified_rejection_sampling(
97
- self,
98
- target_probs: torch.Tensor, # [batch_size, k, vocab_size]
99
- draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
100
- draft_token_ids: torch.Tensor, # [batch_size, k]
99
+ self,
100
+ target_probs: torch.Tensor, # [batch_size, k, vocab_size]
101
+ draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
102
+ draft_token_ids: torch.Tensor, # [batch_size, k]
103
+ generators: List[Optional[torch.Generator]],
104
) -> Tuple[torch.Tensor, torch.Tensor]:
105
"""Perform modified rejection sampling on each sequence.
106
@@ -114,22 +117,33 @@ class RejectionSampler(SpecDecodeBaseSampler):
117
118
# shape [batch_size, k]
119
accepted = self._get_accepted(target_probs, draft_probs,
117
- draft_token_ids)
120
+ draft_token_ids, generators)
121
122
recovered_probs = self._get_recovered_probs(
123
target_probs, draft_probs).reshape(batch_size * k, vocab_size)
124
125
+ seed_indices, non_seed_indices = self._split_batch_by_seeded(
126
+ generators, k=k)
127
+
128
# NOTE: the recovered_probs are overwritten by this method.
123
- recovered_token_ids = _multinomial(recovered_probs,
124
- num_samples=1).reshape(
125
- batch_size, k)
129
+ recovered_token_ids = _multinomial(
130
+ recovered_probs,
131
+ num_samples=1,
132
+ k=k,
133
+ generators=generators,
134
+ seed_indices=seed_indices,
135
+ # this arg is unused when None but torch.jit requires a list
136
+ non_seed_indices=non_seed_indices or [],
137
+ ).reshape(batch_size, k)
138
+
139
return accepted, recovered_token_ids
140
141
def _get_accepted(
129
- self,
130
- target_probs: torch.Tensor, # [batch_size, k, vocab_size]
131
- draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
132
- draft_token_ids: torch.Tensor, # [batch_size, k]
142
+ self,
143
+ target_probs: torch.Tensor, # [batch_size, k, vocab_size]
144
+ draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
145
+ draft_token_ids: torch.Tensor, # [batch_size, k]
146
+ generators: List[Optional[torch.Generator]],
147
) -> torch.Tensor:
148
r"""Create bool matrix over the proposed draft tokens. If
149
True, then a token can be accepted, else it should be
@@ -164,10 +178,28 @@ class RejectionSampler(SpecDecodeBaseSampler):
178
selected_target_probs = target_probs[batch_indices, probs_indicies,
179
draft_token_ids]
180
167
- uniform_rand = torch.rand(batch_size,
168
- k,
169
- dtype=self.probs_dtype,
170
- device=target_probs.device)
181
+ seed_indices, non_seed_indices = self._split_batch_by_seeded(
182
+ generators)
183
+
184
+ if len(seed_indices) == 0:
185
+ uniform_rand = torch.rand_like(selected_target_probs)
186
+ else:
187
+ uniform_rand = torch.empty_like(selected_target_probs)
188
+
189
+ for idx in seed_indices:
190
+ uniform_rand[idx, :] = torch.rand(1,
191
+ k,
192
+ dtype=self.probs_dtype,
193
+ device=target_probs.device,
194
+ generator=generators[idx])
195
+
196
+ if non_seed_indices:
197
+ uniform_rand[non_seed_indices, :] = torch.rand(
198
+ len(non_seed_indices),
199
+ k,
200
+ dtype=self.probs_dtype,
201
+ device=target_probs.device)
202
+
203
capped_ratio = torch.minimum(
204
selected_target_probs / selected_draft_probs,
205
torch.full((1, ), 1, device=target_probs.device))
@@ -240,6 +272,27 @@ class RejectionSampler(SpecDecodeBaseSampler):
272
"""
273
return torch.finfo(self.probs_dtype).tiny
274
275
+ # partition batch into indices for which a generator is provided
276
+ # and indicies for which no generator is provided
277
+ @staticmethod
278
+ def _split_batch_by_seeded(
279
+ generators: List[Optional[torch.Generator]],
280
+ k: int = 1,
281
+ ) -> Tuple[List[int], Optional[List[int]]]:
282
+
283
+ if all(generator is None for generator in generators):
284
+ seed_indices: List[int] = []
285
+ non_seed_indices: Optional[List[int]] = None
286
+ else:
287
+ seed_indices, non_seed_indices = [], []
288
+ for i, generator in enumerate(generators):
289
+ if generator is None:
290
+ non_seed_indices.extend(range(k * i, k * (i + 1)))
291
+ else:
292
+ seed_indices.extend(range(k * i, k * (i + 1)))
293
+
294
+ return seed_indices, non_seed_indices
295
+
296
297
# torch.multinomial forces a GPU<->CPU sync.
298
# Therefore, we use an optimized implementation instead that skips the sync.
@@ -250,12 +303,25 @@ class RejectionSampler(SpecDecodeBaseSampler):
303
def _multinomial(
304
probs: torch.Tensor,
305
num_samples: int,
306
+ k: int,
307
+ generators: List[Optional[torch.Generator]],
308
+ seed_indices: List[int],
309
+ non_seed_indices: List[int],
310
) -> torch.Tensor:
311
+
312
if num_samples > 1:
313
# This is equivalent to torch.repeat_interleaved (which also
314
# forces a GPU<->CPU sync).
315
probs = probs[:, None, :].expand(probs.shape[0], num_samples,
316
probs.shape[1]).contiguous().view(
317
-1, probs.shape[1])
260
- q = torch.empty_like(probs).exponential_(1.0)
318
+
319
+ q = torch.empty_like(probs)
320
+ if len(seed_indices) == 0:
321
+ q.exponential_(1.0)
322
+ else:
323
+ q[non_seed_indices].exponential_(1.0)
324
+ for idx in seed_indices:
325
+ q[idx].exponential_(1.0, generator=generators[idx // k])
326
+
327
return probs.div_(q).argmax(dim=1).view(-1, num_samples)
vllm/model_executor/layers/spec_decode_base_sampler.py
+34
-11
@@ -1,5 +1,5 @@
1
from abc import abstractmethod
2
-from typing import Optional
2
+from typing import List, Optional
3
4
import torch
5
import torch.jit
@@ -54,16 +54,6 @@ class SpecDecodeBaseSampler(nn.Module):
54
def token_id_dtype(self):
55
return torch.int64
56
57
- @abstractmethod
58
- def forward(
59
- self,
60
- target_probs: torch.Tensor,
61
- bonus_token_ids: torch.Tensor,
62
- draft_probs: torch.Tensor,
63
- draft_token_ids: torch.Tensor,
64
- ) -> torch.Tensor:
65
- raise NotImplementedError
66
-
57
def _create_output(
58
self,
59
accepted: torch.Tensor, # [batch_size, k]
@@ -217,3 +207,36 @@ class SpecDecodeBaseSampler(nn.Module):
207
assert torch.all(bonus_token_ids >= 0)
208
assert torch.all(draft_token_ids < vocab_size)
209
assert torch.all(draft_token_ids >= 0)
210
+
211
+
212
+class SpecDecodeDeterministicBaseSampler(SpecDecodeBaseSampler):
213
+ """Base class for samplers used for Speculative Decoding verification
214
+ step which are deterministic.
215
+ """
216
+
217
+ @abstractmethod
218
+ def forward(
219
+ self,
220
+ target_probs: torch.Tensor,
221
+ bonus_token_ids: torch.Tensor,
222
+ draft_probs: torch.Tensor,
223
+ draft_token_ids: torch.Tensor,
224
+ ) -> torch.Tensor:
225
+ raise NotImplementedError
226
+
227
+
228
+class SpecDecodeStochasticBaseSampler(SpecDecodeBaseSampler):
229
+ """Base class for samplers used for Speculative Decoding verification
230
+ step which are stochastic
231
+ """
232
+
233
+ @abstractmethod
234
+ def forward(
235
+ self,
236
+ target_probs: torch.Tensor,
237
+ bonus_token_ids: torch.Tensor,
238
+ draft_probs: torch.Tensor,
239
+ draft_token_ids: torch.Tensor,
240
+ generators: List[Optional[torch.Generator]],
241
+ ) -> torch.Tensor:
242
+ raise NotImplementedError
vllm/model_executor/layers/typical_acceptance_sampler.py
+2
-2
@@ -2,10 +2,10 @@ import torch
2
import torch.jit
3
4
from vllm.model_executor.layers.spec_decode_base_sampler import (
5
- SpecDecodeBaseSampler)
5
+ SpecDecodeDeterministicBaseSampler)
6
7
8
-class TypicalAcceptanceSampler(SpecDecodeBaseSampler):
8
+class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
9
"""Apply typical acceptance sampling as described in section 3.3.1 in
10
"MEDUSA: Simple LLM Inference Acceleration Framework with
11
Multiple Decoding Heads"
vllm/spec_decode/batch_expansion.py
+12
-1
@@ -4,7 +4,8 @@ from typing import Iterator, List, Tuple
4
import torch
5
6
from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
7
- SequenceGroupMetadata, get_all_seq_ids)
7
+ SequenceGroupMetadata, SequenceGroupState,
8
+ get_all_seq_ids)
9
from vllm.spec_decode.interfaces import (SpeculativeProposals,
10
SpeculativeScorer, SpeculativeScores)
11
from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
@@ -292,6 +293,15 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
293
for data in new_seq_data_dict.values():
294
data.update_num_computed_tokens(data.get_len() - 1)
295
296
+ if (seq_group_metadata.state is not None
297
+ and seq_group_metadata.state.generator is not None):
298
+ generator = torch.Generator(
299
+ device=seq_group_metadata.state.generator.device)
300
+ generator.set_state(seq_group_metadata.state.generator.get_state())
301
+ state = SequenceGroupState(generator=generator)
302
+ else:
303
+ state = None
304
+
305
return SequenceGroupMetadata(
306
request_id=seq_group_metadata.request_id,
307
is_prompt=seq_group_metadata.is_prompt,
@@ -302,6 +312,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
312
},
313
lora_request=None,
314
token_chunk_size=1,
315
+ state=state,
316
)
317
318
def _split_scoring_output(
vllm/spec_decode/spec_decode_worker.py
+18
-1
@@ -9,7 +9,7 @@ from vllm.distributed.communication_op import broadcast_tensor_dict
9
from vllm.logger import init_logger
10
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11
from vllm.model_executor.layers.spec_decode_base_sampler import (
12
- SpecDecodeBaseSampler)
12
+ SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
13
from vllm.model_executor.layers.typical_acceptance_sampler import (
14
TypicalAcceptanceSampler)
15
from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
@@ -521,11 +521,28 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
521
# Get proposed tokens.
522
proposal_token_ids = proposals.proposal_token_ids[spec_indices]
523
524
+ # Sampler arguments
525
+ sampler_extra_kwargs = {}
526
+ if isinstance(self.spec_decode_sampler,
527
+ SpecDecodeStochasticBaseSampler):
528
+
529
+ # Get sequence group state
530
+ generators = []
531
+ for seq_group_metadata in seq_group_metadata_list:
532
+ if (seq_group_metadata.state is not None
533
+ and seq_group_metadata.state.generator is not None):
534
+ generators.append(seq_group_metadata.state.generator)
535
+ else:
536
+ generators.append(None)
537
+
538
+ sampler_extra_kwargs["generators"] = generators
539
+
540
accepted_token_ids = self.spec_decode_sampler(
541
target_probs=proposal_verifier_probs,
542
bonus_token_ids=bonus_token_ids,
543
draft_probs=proposal_probs,
544
draft_token_ids=proposal_token_ids,
545
+ **sampler_extra_kwargs,
546
)
547
548
# Append output tokens from non-speculative sequences to