A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[BugFix] Fix use of per-request seed with pipeline parallel (#6698)
Nick Hill committed
Jul 30, 2024 at 10:40 UTC
5cf9254a9cfd1611e5c2fcd1b5011b4bdb56947f
21 files changed
+222
-137
tests/samplers/test_rejection_sampler.py
+8
-15
@@ -150,10 +150,9 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
150
high=vocab_size,
151
size=(batch_size, k),
152
dtype=torch.int64)
153
- generators = [None] * batch_size
153
154
rejection_sampler(target_probs, bonus_token_ids, draft_probs,
156
- draft_token_ids, generators)
155
+ draft_token_ids)
156
157
158
@pytest.mark.parametrize("frac_seeded", [0.0, 0.25, 0.5, 1.0])
@@ -185,14 +184,13 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
184
185
results = []
186
for _ in range(n_rep):
188
- generators = [
189
- torch.Generator(
190
- device=device).manual_seed(i) if seeded_mask[i] else None
191
- for i in range(batch_size)
192
- ]
187
+ seeded_seqs = {
188
+ i: torch.Generator(device=device).manual_seed(i)
189
+ for i in range(batch_size) if seeded_mask[i]
190
+ }
191
results.append(
192
rejection_sampler(target_probs, bonus_token_ids, draft_probs,
195
- draft_token_ids, generators))
193
+ draft_token_ids, seeded_seqs))
194
195
for i in range(batch_size):
196
if seeded_mask[i]:
@@ -242,11 +240,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
240
raise AssertionError()
241
242
oob_token_ids[0][0] = rogue_token_id
245
- generators = [None] * batch_size
243
244
with pytest.raises(AssertionError):
245
rejection_sampler(target_probs, bonus_token_ids, draft_probs,
249
- draft_token_ids, generators)
246
+ draft_token_ids)
247
248
249
@pytest.mark.parametrize("draft_and_target_probs_equal", [True, False])
@@ -417,15 +414,11 @@ class _CorrectnessTestHelper:
414
dtype=torch.int64,
415
device="cuda").repeat(num_samples, 1)
416
420
- # unseeded
421
- generators = [None]
422
-
417
# Get output tokens via rejection sampling.
418
output_token_ids = self.rejection_sampler(target_probs.to("cuda"),
419
bonus_token_ids.to("cuda"),
420
draft_probs.to("cuda"),
427
- draft_token_ids.to("cuda"),
428
- generators)
421
+ draft_token_ids.to("cuda"))
422
423
# Remove bonus tokens
424
output_token_ids = output_token_ids[:, :-1].flatten()
tests/samplers/test_sampler.py
+4
-1
@@ -510,13 +510,16 @@ def test_sampler_mixed(seed: int, device: str):
510
))
511
seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
512
513
+ generators: Dict[str, torch.Generator] = {}
514
+
515
def test_sampling():
516
sampling_metadata = SamplingMetadata.prepare(
517
seq_group_metadata_list,
518
seq_lens,
519
query_lens=seq_lens,
520
device=device,
519
- pin_memory=is_pin_memory_available())
521
+ pin_memory=is_pin_memory_available(),
522
+ generators=generators)
523
sampler_output = sampler(logits=fake_logits,
524
sampling_metadata=sampling_metadata)
525
tests/spec_decode/e2e/test_mlp_correctness.py
+53
-1
@@ -21,7 +21,8 @@ correctess for the target model outputs.
21
22
import pytest
23
24
-from .conftest import run_greedy_equality_correctness_test
24
+from .conftest import (run_equality_correctness_test,
25
+ run_greedy_equality_correctness_test)
26
27
# main model
28
MAIN_MODEL = "JackFram/llama-160m"
@@ -77,6 +78,57 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
78
force_output_len=True)
79
80
81
+@pytest.mark.parametrize(
82
+ "common_llm_kwargs",
83
+ [{
84
+ # Skip cuda graph recording for fast test.
85
+ "enforce_eager": True,
86
+
87
+ # Required for spec decode.
88
+ "use_v2_block_manager": True,
89
+
90
+ # Print spec metrics.
91
+ "disable_log_stats": False,
92
+
93
+ # Precision
94
+ "dtype": PRECISION,
95
+
96
+ # Main model
97
+ "model": MAIN_MODEL,
98
+
99
+ # Speculative model
100
+ "speculative_model": SPEC_MODEL,
101
+ }])
102
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
103
+@pytest.mark.parametrize("baseline_llm_kwargs", [{"seed": 1}])
104
+@pytest.mark.parametrize("test_llm_kwargs", [{"seed": 5}])
105
+@pytest.mark.parametrize("output_len", [64])
106
+@pytest.mark.parametrize("batch_size", [1, 32])
107
+@pytest.mark.parametrize("temperature", [0.1, 1.0])
108
+@pytest.mark.parametrize("seed", [None])
109
+def test_mlp_e2e_seeded_correctness(baseline_llm_generator, test_llm_generator,
110
+ batch_size: int, output_len: int,
111
+ temperature: float):
112
+ """Verify seeded runs produce the same output."""
113
+ run_equality_correctness_test(baseline_llm_generator,
114
+ test_llm_generator,
115
+ batch_size,
116
+ max_output_len=output_len,
117
+ temperature=temperature,
118
+ seeded=True,
119
+ force_output_len=True)
120
+
121
+ # Ensure this same test does fail if we _don't_ include per-request seeds
122
+ with pytest.raises(AssertionError):
123
+ run_equality_correctness_test(baseline_llm_generator,
124
+ test_llm_generator,
125
+ batch_size,
126
+ max_output_len=output_len,
127
+ temperature=temperature,
128
+ seeded=False,
129
+ force_output_len=True)
130
+
131
+
132
@pytest.mark.parametrize(
133
"common_llm_kwargs",
134
[{
tests/spec_decode/e2e/test_seed.py
+1
-1
@@ -29,7 +29,7 @@ from .conftest import run_equality_correctness_test
29
"output_len",
30
[
31
# Use smaller output len for fast test.
32
- 10,
32
+ 20,
33
])
34
@pytest.mark.parametrize("seed", [None])
35
def test_seeded_consistency(baseline_llm_generator, test_llm_generator,
tests/spec_decode/test_batch_expansion.py
+1
@@ -86,6 +86,7 @@ def test_create_single_target_seq_group_metadata(k: int):
86
input_seq_id,
87
target_seq_id,
88
token_ids,
89
+ input_seq_group_metadata.sampling_params,
90
)
91
92
assert output.request_id == input_seq_group_metadata.request_id
tests/utils.py
+31
@@ -178,6 +178,37 @@ def compare_two_settings(model: str, arg1: List[str], arg2: List[str]):
178
"usage": completion.usage,
179
})
180
181
+ # test seeded random sampling
182
+ completion = client.completions.create(model=model,
183
+ prompt=prompt,
184
+ max_tokens=5,
185
+ seed=33,
186
+ temperature=1.0)
187
+
188
+ results.append({
189
+ "test": "seeded_sampling",
190
+ "text": completion.choices[0].text,
191
+ "finish_reason": completion.choices[0].finish_reason,
192
+ "usage": completion.usage,
193
+ })
194
+
195
+ # test seeded random sampling with multiple prompts
196
+ completion = client.completions.create(model=model,
197
+ prompt=[prompt, prompt],
198
+ max_tokens=5,
199
+ seed=33,
200
+ temperature=1.0)
201
+
202
+ results.append({
203
+ "test":
204
+ "seeded_sampling",
205
+ "text": [choice.text for choice in completion.choices],
206
+ "finish_reason":
207
+ [choice.finish_reason for choice in completion.choices],
208
+ "usage":
209
+ completion.usage,
210
+ })
211
+
212
# test simple list
213
batch = client.completions.create(
214
model=model,
vllm/core/scheduler.py
-1
@@ -1029,7 +1029,6 @@ class Scheduler:
1029
token_chunk_size=token_chunk_size,
1030
lora_request=seq_group.lora_request,
1031
computed_block_nums=common_computed_block_nums,
1032
- state=seq_group.state,
1032
# `multi_modal_data` will only be present for the 1st comm
1033
# between engine and worker.
1034
# the subsequent comms can still use delta, but
vllm/model_executor/layers/rejection_sampler.py
+39
-56
@@ -1,5 +1,5 @@
1
from functools import cached_property
2
-from typing import List, Optional, Tuple
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
import torch.jit
@@ -36,7 +36,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
36
bonus_token_ids: torch.Tensor,
37
draft_probs: torch.Tensor,
38
draft_token_ids: torch.Tensor,
39
- generators: List[Optional[torch.Generator]],
39
+ seeded_seqs: Optional[Dict[int, torch.Generator]] = None,
40
) -> torch.Tensor:
41
"""Sample token ids using rejection sampling. This accepts or rejects
42
tokens proposed by the draft model using the probability of each token
@@ -66,6 +66,9 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
66
probabilities.
67
shape = [batch_size, num_speculative_tokens]
68
69
+ seeded_seqs: Dict of batch row index to torch generator, for
70
+ sequences using seeded generation.
71
+
72
Returns:
73
output_token_ids: The token ids sampled via rejection sampling,
74
or -1 if unable to sample a token because the previous token
@@ -83,7 +86,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
86
target_probs,
87
draft_probs,
88
draft_token_ids,
86
- generators,
89
+ seeded_seqs,
90
))
91
92
output_token_ids = self._create_output(
@@ -100,7 +103,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
103
target_probs: torch.Tensor, # [batch_size, k, vocab_size]
104
draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
105
draft_token_ids: torch.Tensor, # [batch_size, k]
103
- generators: List[Optional[torch.Generator]],
106
+ seeded_seqs: Optional[Dict[int, torch.Generator]],
107
) -> Tuple[torch.Tensor, torch.Tensor]:
108
"""Perform modified rejection sampling on each sequence.
109
@@ -117,23 +120,17 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
120
121
# shape [batch_size, k]
122
accepted = self._get_accepted(target_probs, draft_probs,
120
- draft_token_ids, generators)
123
+ draft_token_ids, seeded_seqs)
124
125
recovered_probs = self._get_recovered_probs(
126
target_probs, draft_probs).reshape(batch_size * k, vocab_size)
127
125
- seed_indices, non_seed_indices = self._split_batch_by_seeded(
126
- generators, k=k)
127
-
128
# NOTE: the recovered_probs are overwritten by this method.
129
recovered_token_ids = _multinomial(
130
recovered_probs,
131
num_samples=1,
132
k=k,
133
- generators=generators,
134
- seed_indices=seed_indices,
135
- # this arg is unused when None but torch.jit requires a list
136
- non_seed_indices=non_seed_indices or [],
133
+ seeded_seqs=seeded_seqs or {},
134
).reshape(batch_size, k)
135
136
return accepted, recovered_token_ids
@@ -143,7 +140,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
140
target_probs: torch.Tensor, # [batch_size, k, vocab_size]
141
draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
142
draft_token_ids: torch.Tensor, # [batch_size, k]
146
- generators: List[Optional[torch.Generator]],
143
+ seeded_seqs: Optional[Dict[int, torch.Generator]],
144
) -> torch.Tensor:
145
r"""Create bool matrix over the proposed draft tokens. If
146
True, then a token can be accepted, else it should be
@@ -178,24 +175,26 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
175
selected_target_probs = target_probs[batch_indices, probs_indicies,
176
draft_token_ids]
177
181
- seed_indices, non_seed_indices = self._split_batch_by_seeded(
182
- generators)
183
-
184
- if len(seed_indices) == 0:
178
+ if not seeded_seqs:
179
uniform_rand = torch.rand_like(selected_target_probs)
180
else:
181
uniform_rand = torch.empty_like(selected_target_probs)
182
189
- for idx in seed_indices:
190
- uniform_rand[idx, :] = torch.rand(1,
191
- k,
192
- dtype=self.probs_dtype,
193
- device=target_probs.device,
194
- generator=generators[idx])
195
-
196
- if non_seed_indices:
197
- uniform_rand[non_seed_indices, :] = torch.rand(
198
- len(non_seed_indices),
183
+ non_seeded_indices = []
184
+ for idx in range(batch_size):
185
+ generator = seeded_seqs.get(idx)
186
+ if generator is None:
187
+ non_seeded_indices.append(idx)
188
+ else:
189
+ uniform_rand[idx, :] = torch.rand(
190
+ 1,
191
+ k,
192
+ dtype=self.probs_dtype,
193
+ device=target_probs.device,
194
+ generator=generator)
195
+ if non_seeded_indices:
196
+ uniform_rand[non_seeded_indices, :] = torch.rand(
197
+ len(non_seeded_indices),
198
k,
199
dtype=self.probs_dtype,
200
device=target_probs.device)
@@ -272,27 +271,6 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
271
"""
272
return torch.finfo(self.probs_dtype).tiny
273
275
- # partition batch into indices for which a generator is provided
276
- # and indicies for which no generator is provided
277
- @staticmethod
278
- def _split_batch_by_seeded(
279
- generators: List[Optional[torch.Generator]],
280
- k: int = 1,
281
- ) -> Tuple[List[int], Optional[List[int]]]:
282
-
283
- if all(generator is None for generator in generators):
284
- seed_indices: List[int] = []
285
- non_seed_indices: Optional[List[int]] = None
286
- else:
287
- seed_indices, non_seed_indices = [], []
288
- for i, generator in enumerate(generators):
289
- if generator is None:
290
- non_seed_indices.extend(range(k * i, k * (i + 1)))
291
- else:
292
- seed_indices.extend(range(k * i, k * (i + 1)))
293
-
294
- return seed_indices, non_seed_indices
295
-
274
275
# torch.multinomial forces a GPU<->CPU sync.
276
# Therefore, we use an optimized implementation instead that skips the sync.
@@ -304,9 +282,7 @@ def _multinomial(
282
probs: torch.Tensor,
283
num_samples: int,
284
k: int,
307
- generators: List[Optional[torch.Generator]],
308
- seed_indices: List[int],
309
- non_seed_indices: List[int],
285
+ seeded_seqs: Dict[int, torch.Generator],
286
) -> torch.Tensor:
287
288
if num_samples > 1:
@@ -315,13 +291,20 @@ def _multinomial(
291
probs = probs[:, None, :].expand(probs.shape[0], num_samples,
292
probs.shape[1]).contiguous().view(
293
-1, probs.shape[1])
318
-
294
q = torch.empty_like(probs)
320
- if len(seed_indices) == 0:
295
+ if not seeded_seqs:
296
q.exponential_(1.0)
297
else:
323
- q[non_seed_indices].exponential_(1.0)
324
- for idx in seed_indices:
325
- q[idx].exponential_(1.0, generator=generators[idx // k])
298
+ non_seeded_indices: List[int] = []
299
+ start = 0
300
+ for idx in range(len(q) // k):
301
+ end = start + k
302
+ generator = seeded_seqs.get(idx)
303
+ if generator is None:
304
+ non_seeded_indices.extend(list(range(start, end)))
305
+ else:
306
+ q[start:end].exponential_(1.0, generator=generator)
307
+ start = end
308
+ q[non_seeded_indices].exponential_(1.0)
309
310
return probs.div_(q).argmax(dim=1).view(-1, num_samples)
vllm/model_executor/layers/spec_decode_base_sampler.py
+2
-2
@@ -1,5 +1,5 @@
1
from abc import abstractmethod
2
-from typing import List, Optional
2
+from typing import Dict, Optional
3
4
import torch
5
import torch.jit
@@ -237,6 +237,6 @@ class SpecDecodeStochasticBaseSampler(SpecDecodeBaseSampler):
237
bonus_token_ids: torch.Tensor,
238
draft_probs: torch.Tensor,
239
draft_token_ids: torch.Tensor,
240
- generators: List[Optional[torch.Generator]],
240
+ seeded_seqs: Optional[Dict[int, torch.Generator]] = None,
241
) -> torch.Tensor:
242
raise NotImplementedError
vllm/model_executor/sampling_metadata.py
+13
-7
@@ -118,6 +118,7 @@ class SamplingMetadata:
118
query_lens: Optional[List[int]],
119
device: str,
120
pin_memory: bool,
121
+ generators: Optional[Dict[str, torch.Generator]] = None,
122
) -> "SamplingMetadata":
123
(
124
seq_groups,
@@ -125,7 +126,7 @@ class SamplingMetadata:
126
categorized_sample_indices,
127
num_prompts,
128
) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
128
- device)
129
+ device, generators)
130
selected_token_indices = async_tensor_h2d(selected_token_indices,
131
dtype=torch.long,
132
target_device=device,
@@ -160,6 +161,7 @@ def _prepare_seq_groups(
161
seq_lens: List[int],
162
query_lens: Optional[List[int]],
163
device: str,
164
+ generators: Optional[Dict[str, torch.Generator]] = None,
165
) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
166
SamplingType, List[Tuple[int, int]]], int]:
167
"""Prepare sequence groups and indices for sampling.
@@ -170,8 +172,10 @@ def _prepare_seq_groups(
172
Index of prompt len should match with seq_group_metadata_list.
173
query_lens: A list of query lengths. Prompt lens include the length
174
of entire prompt tokens, and it could be shorter.
173
- device: A device to use for random number generator,
175
+ device: A device to use for random number generators,
176
`SequenceGroupToSample.generator`.
177
+ generators: A store of per-request random number generators used
178
+ for seeded requests.
179
180
Returns:
181
seq_groups: A list of sequence group to sample.
@@ -217,8 +221,10 @@ def _prepare_seq_groups(
221
222
if seq_group_metadata.is_prompt:
223
if sampling_params.seed is not None:
220
- seq_group_metadata.state.generator = torch.Generator(
221
- device=device).manual_seed(sampling_params.seed)
224
+ generator = torch.Generator(device=device).manual_seed(
225
+ sampling_params.seed)
226
+ if generators is not None:
227
+ generators[seq_group_metadata.request_id] = generator
228
229
num_prompts += 1
230
num_prefill_sample = len(seq_ids)
@@ -235,6 +241,9 @@ def _prepare_seq_groups(
241
prompt_logprob_len = 0
242
sample_len = len(seq_ids) if do_sample else 0
243
244
+ if sampling_params.seed is not None and generators is not None:
245
+ generator = generators.get(seq_group_metadata.request_id)
246
+
247
# Update indices to select from the model output.
248
"""
249
This blocks computes selected_token_indices which is used in the
@@ -279,9 +288,6 @@ def _prepare_seq_groups(
288
logit_idx += sample_len
289
sample_idx += sample_len
290
282
- if sampling_params.seed is not None:
283
- generator = seq_group_metadata.state.generator
284
-
291
seq_groups.append(
292
SequenceGroupToSample(
293
seq_ids=seq_ids,
vllm/sequence.py
-12
@@ -411,14 +411,6 @@ class Sequence:
411
f"num_blocks={self.n_blocks}, ")
412
413
414
-@dataclass
415
-class SequenceGroupState:
416
- """Mutable state tied to a specific sequence group"""
417
-
418
- # torch.Generator used in seeded sampling
419
- generator: Optional = None # type: ignore
420
-
421
-
414
class SequenceGroup:
415
"""A group of sequences that are generated from the same prompt.
416
@@ -461,7 +453,6 @@ class SequenceGroup:
453
time_in_queue=None)
454
self.lora_request = lora_request
455
self.prompt_logprobs: Optional[PromptLogprobs] = None
464
- self.state = SequenceGroupState()
456
self.embeddings = embeddings
457
self.pooling_params = pooling_params
458
self.prompt_adapter_request = prompt_adapter_request
@@ -648,7 +639,6 @@ class SequenceGroupMetadata:
639
lora_request: LoRA request.
640
computed_block_nums: The block numbers that are already computed,
641
used in prefix caching.
651
- state: Internal state tied to this sequence group.
642
multi_modal_data: Multi modal data.
643
encoder_seq_data: Optional sequence data for encoder prompt
644
(SequenceGroup.encoder_seq). Should be None
@@ -674,7 +664,6 @@ class SequenceGroupMetadata:
664
token_chunk_size: Optional[int] = None,
665
lora_request: Optional[LoRARequest] = None,
666
computed_block_nums: Optional[List[int]] = None,
677
- state: Optional[SequenceGroupState] = None,
667
multi_modal_data: Optional["MultiModalDataDict"] = None,
668
encoder_seq_data: Optional[SequenceData] = None,
669
cross_block_table: Optional[List[int]] = None,
@@ -690,7 +679,6 @@ class SequenceGroupMetadata:
679
self.prompt_adapter_request = prompt_adapter_request
680
self.computed_block_nums = computed_block_nums
681
self.multi_modal_data = multi_modal_data
693
- self.state = SequenceGroupState() if state is None else state
682
self.encoder_seq_data = encoder_seq_data
683
self.cross_block_table = cross_block_table
684
self._token_chunk_size = token_chunk_size
vllm/spec_decode/batch_expansion.py
+22
-15
@@ -3,9 +3,9 @@ from typing import Iterator, List, Tuple
3
4
import torch
5
6
+from vllm import SamplingParams
7
from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
7
- SequenceGroupMetadata, SequenceGroupState,
8
- get_all_seq_ids)
8
+ SequenceGroupMetadata, get_all_seq_ids)
9
from vllm.spec_decode.interfaces import (SpeculativeProposals,
10
SpeculativeScorer, SpeculativeScores)
11
from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
@@ -16,6 +16,8 @@ SeqId = int
16
TargetSeqId = int
17
TokenId = int
18
19
+DEFAULT_SIMPLE_SAMPLING_PARAMS = SamplingParams()
20
+
21
22
class BatchExpansionTop1Scorer(SpeculativeScorer):
23
"""Implements a speculative scorer that uses batch expansion to get
@@ -247,24 +249,39 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
249
token_ids_to_score = self._get_token_ids_to_score(
250
proposal_token_ids[batch_index])
251
252
+ # Use simpler sampling parameters apart from for final token
253
+ # (in particular don't do seeded sampling) since those sampled tokens
254
+ # aren't used.
255
+ # We don't replace the sampling_params in the greedy case because
256
+ # this also controls whether the probs get modified in the sampler
257
+ # (see use of _modify_greedy_probs_inplace there).
258
+ sampling_params = input_seq_group_metadata.sampling_params
259
+ non_bonus_sampling_params = DEFAULT_SIMPLE_SAMPLING_PARAMS \
260
+ if sampling_params.temperature else sampling_params
261
+
262
target_seq_group_metadata_list: List[SequenceGroupMetadata] = []
251
- for token_ids in token_ids_to_score:
263
+ last_index = len(token_ids_to_score) - 1
264
+ for i, token_ids in enumerate(token_ids_to_score):
265
+ target_sampling_params = sampling_params if i == last_index \
266
+ else non_bonus_sampling_params
267
target_seq_group_metadata_list.append(
268
self._create_single_target_seq_group_metadata(
269
input_seq_group_metadata,
270
input_seq_id,
271
next(target_seq_ids_iter),
272
token_ids,
273
+ sampling_params=target_sampling_params,
274
))
275
276
return target_seq_group_metadata_list
277
278
+ @staticmethod
279
def _create_single_target_seq_group_metadata(
263
- self,
280
seq_group_metadata: SequenceGroupMetadata,
281
seq_id: SeqId,
282
target_seq_id: TargetSeqId,
283
token_ids: List[TokenId],
284
+ sampling_params: SamplingParams,
285
) -> SequenceGroupMetadata:
286
"""Create a single target SequenceGroupMetadata.
287
@@ -293,26 +310,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
310
for data in new_seq_data_dict.values():
311
data.update_num_computed_tokens(data.get_len() - 1)
312
296
- if (seq_group_metadata.state is not None
297
- and seq_group_metadata.state.generator is not None):
298
- generator = torch.Generator(
299
- device=seq_group_metadata.state.generator.device)
300
- generator.set_state(seq_group_metadata.state.generator.get_state())
301
- state = SequenceGroupState(generator=generator)
302
- else:
303
- state = None
304
-
313
return SequenceGroupMetadata(
314
request_id=seq_group_metadata.request_id,
315
is_prompt=seq_group_metadata.is_prompt,
316
seq_data=new_seq_data_dict,
309
- sampling_params=seq_group_metadata.sampling_params,
317
+ sampling_params=sampling_params,
318
block_tables={
319
target_seq_id: seq_group_metadata.block_tables[seq_id],
320
},
321
lora_request=None,
322
token_chunk_size=1,
315
- state=state,
323
)
324
325
def _split_scoring_output(
vllm/spec_decode/medusa_worker.py
+3
-1
@@ -57,9 +57,11 @@ class MedusaWorker(NonLLMProposerWorkerBase, Worker):
57
seq_lens, query_lens = self._prepare_input_tensors(
58
seq_group_metadata_list)
59
60
+ generators = self.model_runner.get_generators(
61
+ execute_model_req.finished_requests_ids)
62
sampling_metadata = SamplingMetadata.prepare(
63
seq_group_metadata_list, seq_lens, query_lens, self.device,
62
- self.model_runner.pin_memory)
64
+ self.model_runner.pin_memory, generators)
65
66
model_outputs = self.model_runner.model.generate_proposals(
67
previous_hidden_states=execute_model_req.previous_hidden_states.
vllm/spec_decode/mlp_speculator_worker.py
+3
-1
@@ -38,9 +38,11 @@ class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
38
(input_tokens, seq_lens,
39
query_lens) = self._prepare_input_tensors(seq_group_metadata_list)
40
41
+ generators = self.model_runner.get_generators(
42
+ execute_model_req.finished_requests_ids)
43
sampling_metadata = SamplingMetadata.prepare(
44
seq_group_metadata_list, seq_lens, query_lens, self.device,
43
- self.model_runner.pin_memory)
45
+ self.model_runner.pin_memory, generators)
46
47
model_outputs = self.model_runner.model.generate_proposals(
48
input_ids=input_tokens,
vllm/spec_decode/ngram_worker.py
+1
-2
@@ -7,10 +7,9 @@ from vllm.sequence import ExecuteModelRequest, SamplerOutput
7
from vllm.spec_decode.interfaces import SpeculativeProposals
8
from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
9
from vllm.spec_decode.top1_proposer import Top1Proposer
10
-from vllm.worker.worker_base import LoraNotSupportedWorkerBase
10
11
13
-class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
12
+class NGramWorker(NonLLMProposerWorkerBase):
13
"""NGramWorker provides a light drafter without need for model.
14
15
Current NGramWorker only implements prompt lookup decoding,
vllm/spec_decode/spec_decode_worker.py
+11
-14
@@ -213,6 +213,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
213
"""
214
self.proposer_worker = proposer_worker
215
self.scorer_worker = scorer_worker
216
+ scorer_runner = getattr(self.scorer_worker, "model_runner", None)
217
+ self.generators = scorer_runner.get_generators(
218
+ ) if scorer_runner else None
219
self.disable_by_batch_size = disable_by_batch_size or float("inf")
220
self.spec_decode_sampler = spec_decode_sampler
221
self._allow_zero_draft_token_step = allow_zero_draft_token_step
@@ -591,20 +594,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
594
proposal_token_ids = proposals.proposal_token_ids[spec_indices]
595
596
# Sampler arguments
594
- sampler_extra_kwargs = {}
595
- if isinstance(self.spec_decode_sampler,
596
- SpecDecodeStochasticBaseSampler):
597
-
598
- # Get sequence group state
599
- generators = []
600
- for seq_group_metadata in seq_group_metadata_list:
601
- if (seq_group_metadata.state is not None
602
- and seq_group_metadata.state.generator is not None):
603
- generators.append(seq_group_metadata.state.generator)
604
- else:
605
- generators.append(None)
606
-
607
- sampler_extra_kwargs["generators"] = generators
597
+ sampler_extra_kwargs: Dict[str, Any] = {}
598
+ if self.generators and isinstance(self.spec_decode_sampler,
599
+ SpecDecodeStochasticBaseSampler):
600
+ sampler_extra_kwargs["seeded_seqs"] = {
601
+ idx: self.generators[sgm.request_id]
602
+ for idx, sgm in enumerate(seq_group_metadata_list)
603
+ if sgm.sampling_params.seed is not None
604
+ }
605
606
accepted_token_ids = self.spec_decode_sampler(
607
target_probs=proposal_verifier_probs,
vllm/worker/cpu_model_runner.py
+2
-1
@@ -337,7 +337,8 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
337
# just use seq_lens instead.
338
seq_lens,
339
self.device,
340
- pin_memory=False)
340
+ pin_memory=False,
341
+ generators=self.get_generators(finished_requests_ids))
342
return CPUModelInput(
343
input_tokens=input_tokens,
344
input_positions=input_positions,
vllm/worker/model_runner.py
+9
-5
@@ -1264,11 +1264,15 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1264
"""
1265
model_input = self._prepare_model_input_tensors(
1266
seq_group_metadata_list, finished_requests_ids)
1267
- sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
1268
- model_input.seq_lens,
1269
- model_input.query_lens,
1270
- self.device,
1271
- self.pin_memory)
1267
+ if get_pp_group().is_last_rank:
1268
+ # Sampling metadata is only required for the final pp group
1269
+ generators = self.get_generators(finished_requests_ids)
1270
+ sampling_metadata = SamplingMetadata.prepare(
1271
+ seq_group_metadata_list, model_input.seq_lens,
1272
+ model_input.query_lens, self.device, self.pin_memory,
1273
+ generators)
1274
+ else:
1275
+ sampling_metadata = None
1276
is_prompt = (seq_group_metadata_list[0].is_prompt
1277
if seq_group_metadata_list else None)
1278
return dataclasses.replace(model_input,
vllm/worker/model_runner_base.py
+15
@@ -139,6 +139,9 @@ class ModelRunnerBase(ABC, Generic[T]):
139
ModelRunnerInputBase subclass.
140
"""
141
142
+ # Map of request_id -> generator used for seeded random sampling
143
+ generators: Dict[str, torch.Generator] = {}
144
+
145
@abstractmethod
146
def make_model_input_from_broadcasted_tensor_dict(
147
self,
@@ -176,3 +179,15 @@ class ModelRunnerBase(ABC, Generic[T]):
179
Execute the model on the given input.
180
"""
181
raise NotImplementedError
182
+
183
+ def get_generators(self, finished_request_ids: Optional[List[str]] = None):
184
+ """
185
+ Return dict of per-request generators used for random sampling.
186
+ """
187
+
188
+ # Clean up generators from completed requests
189
+ if finished_request_ids:
190
+ for request_id in finished_request_ids:
191
+ self.generators.pop(request_id, None)
192
+
193
+ return self.generators
vllm/worker/neuron_model_runner.py
+2
-1
@@ -219,7 +219,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
219
# just use seq_lens instead.
220
seq_lens,
221
self.device,
222
- self.pin_memory)
222
+ self.pin_memory,
223
+ generators=self.get_generators(finished_requests_ids))
224
225
return ModelInputForNeuron(input_tokens=input_tokens,
226
input_positions=input_positions,
vllm/worker/xpu_model_runner.py
+2
-1
@@ -246,7 +246,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
246
# just use seq_lens instead.
247
seq_lens,
248
self.device,
249
- pin_memory=False)
249
+ pin_memory=False,
250
+ generators=self.get_generators(finished_requests_ids))
251
# Broadcast the metadata.
252
metadata_dict = {
253
"input_tokens": input_tokens,