@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5cf9254a9c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[BugFix] Fix use of per-request seed with pipeline parallel (#6698)

Nick Hill committed Jul 30, 2024 at 10:40 UTC 5cf9254a9cfd1611e5c2fcd1b5011b4bdb56947f
21 files changed +222 -137
tests/samplers/test_rejection_sampler.py
+8 -15
@@ -150,10 +150,9 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
150 high=vocab_size,
151 size=(batch_size, k),
152 dtype=torch.int64)
153 - generators = [None] * batch_size
153
154 rejection_sampler(target_probs, bonus_token_ids, draft_probs,
156 - draft_token_ids, generators)
155 + draft_token_ids)
156
157
158 @pytest.mark.parametrize("frac_seeded", [0.0, 0.25, 0.5, 1.0])
@@ -185,14 +184,13 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
184
185 results = []
186 for _ in range(n_rep):
188 - generators = [
189 - torch.Generator(
190 - device=device).manual_seed(i) if seeded_mask[i] else None
191 - for i in range(batch_size)
192 - ]
187 + seeded_seqs = {
188 + i: torch.Generator(device=device).manual_seed(i)
189 + for i in range(batch_size) if seeded_mask[i]
190 + }
191 results.append(
192 rejection_sampler(target_probs, bonus_token_ids, draft_probs,
195 - draft_token_ids, generators))
193 + draft_token_ids, seeded_seqs))
194
195 for i in range(batch_size):
196 if seeded_mask[i]:
@@ -242,11 +240,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
240 raise AssertionError()
241
242 oob_token_ids[0][0] = rogue_token_id
245 - generators = [None] * batch_size
243
244 with pytest.raises(AssertionError):
245 rejection_sampler(target_probs, bonus_token_ids, draft_probs,
249 - draft_token_ids, generators)
246 + draft_token_ids)
247
248
249 @pytest.mark.parametrize("draft_and_target_probs_equal", [True, False])
@@ -417,15 +414,11 @@ class _CorrectnessTestHelper:
414 dtype=torch.int64,
415 device="cuda").repeat(num_samples, 1)
416
420 - # unseeded
421 - generators = [None]
422 -
417 # Get output tokens via rejection sampling.
418 output_token_ids = self.rejection_sampler(target_probs.to("cuda"),
419 bonus_token_ids.to("cuda"),
420 draft_probs.to("cuda"),
427 - draft_token_ids.to("cuda"),
428 - generators)
421 + draft_token_ids.to("cuda"))
422
423 # Remove bonus tokens
424 output_token_ids = output_token_ids[:, :-1].flatten()
tests/samplers/test_sampler.py
+4 -1
@@ -510,13 +510,16 @@ def test_sampler_mixed(seed: int, device: str):
510 ))
511 seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
512
513 + generators: Dict[str, torch.Generator] = {}
514 +
515 def test_sampling():
516 sampling_metadata = SamplingMetadata.prepare(
517 seq_group_metadata_list,
518 seq_lens,
519 query_lens=seq_lens,
520 device=device,
519 - pin_memory=is_pin_memory_available())
521 + pin_memory=is_pin_memory_available(),
522 + generators=generators)
523 sampler_output = sampler(logits=fake_logits,
524 sampling_metadata=sampling_metadata)
525
tests/spec_decode/e2e/test_mlp_correctness.py
+53 -1
@@ -21,7 +21,8 @@ correctess for the target model outputs.
21
22 import pytest
23
24 -from .conftest import run_greedy_equality_correctness_test
24 +from .conftest import (run_equality_correctness_test,
25 + run_greedy_equality_correctness_test)
26
27 # main model
28 MAIN_MODEL = "JackFram/llama-160m"
@@ -77,6 +78,57 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
78 force_output_len=True)
79
80
81 +@pytest.mark.parametrize(
82 + "common_llm_kwargs",
83 + [{
84 + # Skip cuda graph recording for fast test.
85 + "enforce_eager": True,
86 +
87 + # Required for spec decode.
88 + "use_v2_block_manager": True,
89 +
90 + # Print spec metrics.
91 + "disable_log_stats": False,
92 +
93 + # Precision
94 + "dtype": PRECISION,
95 +
96 + # Main model
97 + "model": MAIN_MODEL,
98 +
99 + # Speculative model
100 + "speculative_model": SPEC_MODEL,
101 + }])
102 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
103 +@pytest.mark.parametrize("baseline_llm_kwargs", [{"seed": 1}])
104 +@pytest.mark.parametrize("test_llm_kwargs", [{"seed": 5}])
105 +@pytest.mark.parametrize("output_len", [64])
106 +@pytest.mark.parametrize("batch_size", [1, 32])
107 +@pytest.mark.parametrize("temperature", [0.1, 1.0])
108 +@pytest.mark.parametrize("seed", [None])
109 +def test_mlp_e2e_seeded_correctness(baseline_llm_generator, test_llm_generator,
110 + batch_size: int, output_len: int,
111 + temperature: float):
112 + """Verify seeded runs produce the same output."""
113 + run_equality_correctness_test(baseline_llm_generator,
114 + test_llm_generator,
115 + batch_size,
116 + max_output_len=output_len,
117 + temperature=temperature,
118 + seeded=True,
119 + force_output_len=True)
120 +
121 + # Ensure this same test does fail if we _don't_ include per-request seeds
122 + with pytest.raises(AssertionError):
123 + run_equality_correctness_test(baseline_llm_generator,
124 + test_llm_generator,
125 + batch_size,
126 + max_output_len=output_len,
127 + temperature=temperature,
128 + seeded=False,
129 + force_output_len=True)
130 +
131 +
132 @pytest.mark.parametrize(
133 "common_llm_kwargs",
134 [{
tests/spec_decode/e2e/test_seed.py
+1 -1
@@ -29,7 +29,7 @@ from .conftest import run_equality_correctness_test
29 "output_len",
30 [
31 # Use smaller output len for fast test.
32 - 10,
32 + 20,
33 ])
34 @pytest.mark.parametrize("seed", [None])
35 def test_seeded_consistency(baseline_llm_generator, test_llm_generator,
tests/spec_decode/test_batch_expansion.py
+1
@@ -86,6 +86,7 @@ def test_create_single_target_seq_group_metadata(k: int):
86 input_seq_id,
87 target_seq_id,
88 token_ids,
89 + input_seq_group_metadata.sampling_params,
90 )
91
92 assert output.request_id == input_seq_group_metadata.request_id
tests/utils.py
+31
@@ -178,6 +178,37 @@ def compare_two_settings(model: str, arg1: List[str], arg2: List[str]):
178 "usage": completion.usage,
179 })
180
181 + # test seeded random sampling
182 + completion = client.completions.create(model=model,
183 + prompt=prompt,
184 + max_tokens=5,
185 + seed=33,
186 + temperature=1.0)
187 +
188 + results.append({
189 + "test": "seeded_sampling",
190 + "text": completion.choices[0].text,
191 + "finish_reason": completion.choices[0].finish_reason,
192 + "usage": completion.usage,
193 + })
194 +
195 + # test seeded random sampling with multiple prompts
196 + completion = client.completions.create(model=model,
197 + prompt=[prompt, prompt],
198 + max_tokens=5,
199 + seed=33,
200 + temperature=1.0)
201 +
202 + results.append({
203 + "test":
204 + "seeded_sampling",
205 + "text": [choice.text for choice in completion.choices],
206 + "finish_reason":
207 + [choice.finish_reason for choice in completion.choices],
208 + "usage":
209 + completion.usage,
210 + })
211 +
212 # test simple list
213 batch = client.completions.create(
214 model=model,
vllm/core/scheduler.py
-1
@@ -1029,7 +1029,6 @@ class Scheduler:
1029 token_chunk_size=token_chunk_size,
1030 lora_request=seq_group.lora_request,
1031 computed_block_nums=common_computed_block_nums,
1032 - state=seq_group.state,
1032 # `multi_modal_data` will only be present for the 1st comm
1033 # between engine and worker.
1034 # the subsequent comms can still use delta, but
vllm/model_executor/layers/rejection_sampler.py
+39 -56
@@ -1,5 +1,5 @@
1 from functools import cached_property
2 -from typing import List, Optional, Tuple
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5 import torch.jit
@@ -36,7 +36,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
36 bonus_token_ids: torch.Tensor,
37 draft_probs: torch.Tensor,
38 draft_token_ids: torch.Tensor,
39 - generators: List[Optional[torch.Generator]],
39 + seeded_seqs: Optional[Dict[int, torch.Generator]] = None,
40 ) -> torch.Tensor:
41 """Sample token ids using rejection sampling. This accepts or rejects
42 tokens proposed by the draft model using the probability of each token
@@ -66,6 +66,9 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
66 probabilities.
67 shape = [batch_size, num_speculative_tokens]
68
69 + seeded_seqs: Dict of batch row index to torch generator, for
70 + sequences using seeded generation.
71 +
72 Returns:
73 output_token_ids: The token ids sampled via rejection sampling,
74 or -1 if unable to sample a token because the previous token
@@ -83,7 +86,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
86 target_probs,
87 draft_probs,
88 draft_token_ids,
86 - generators,
89 + seeded_seqs,
90 ))
91
92 output_token_ids = self._create_output(
@@ -100,7 +103,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
103 target_probs: torch.Tensor, # [batch_size, k, vocab_size]
104 draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
105 draft_token_ids: torch.Tensor, # [batch_size, k]
103 - generators: List[Optional[torch.Generator]],
106 + seeded_seqs: Optional[Dict[int, torch.Generator]],
107 ) -> Tuple[torch.Tensor, torch.Tensor]:
108 """Perform modified rejection sampling on each sequence.
109
@@ -117,23 +120,17 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
120
121 # shape [batch_size, k]
122 accepted = self._get_accepted(target_probs, draft_probs,
120 - draft_token_ids, generators)
123 + draft_token_ids, seeded_seqs)
124
125 recovered_probs = self._get_recovered_probs(
126 target_probs, draft_probs).reshape(batch_size * k, vocab_size)
127
125 - seed_indices, non_seed_indices = self._split_batch_by_seeded(
126 - generators, k=k)
127 -
128 # NOTE: the recovered_probs are overwritten by this method.
129 recovered_token_ids = _multinomial(
130 recovered_probs,
131 num_samples=1,
132 k=k,
133 - generators=generators,
134 - seed_indices=seed_indices,
135 - # this arg is unused when None but torch.jit requires a list
136 - non_seed_indices=non_seed_indices or [],
133 + seeded_seqs=seeded_seqs or {},
134 ).reshape(batch_size, k)
135
136 return accepted, recovered_token_ids
@@ -143,7 +140,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
140 target_probs: torch.Tensor, # [batch_size, k, vocab_size]
141 draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
142 draft_token_ids: torch.Tensor, # [batch_size, k]
146 - generators: List[Optional[torch.Generator]],
143 + seeded_seqs: Optional[Dict[int, torch.Generator]],
144 ) -> torch.Tensor:
145 r"""Create bool matrix over the proposed draft tokens. If
146 True, then a token can be accepted, else it should be
@@ -178,24 +175,26 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
175 selected_target_probs = target_probs[batch_indices, probs_indicies,
176 draft_token_ids]
177
181 - seed_indices, non_seed_indices = self._split_batch_by_seeded(
182 - generators)
183 -
184 - if len(seed_indices) == 0:
178 + if not seeded_seqs:
179 uniform_rand = torch.rand_like(selected_target_probs)
180 else:
181 uniform_rand = torch.empty_like(selected_target_probs)
182
189 - for idx in seed_indices:
190 - uniform_rand[idx, :] = torch.rand(1,
191 - k,
192 - dtype=self.probs_dtype,
193 - device=target_probs.device,
194 - generator=generators[idx])
195 -
196 - if non_seed_indices:
197 - uniform_rand[non_seed_indices, :] = torch.rand(
198 - len(non_seed_indices),
183 + non_seeded_indices = []
184 + for idx in range(batch_size):
185 + generator = seeded_seqs.get(idx)
186 + if generator is None:
187 + non_seeded_indices.append(idx)
188 + else:
189 + uniform_rand[idx, :] = torch.rand(
190 + 1,
191 + k,
192 + dtype=self.probs_dtype,
193 + device=target_probs.device,
194 + generator=generator)
195 + if non_seeded_indices:
196 + uniform_rand[non_seeded_indices, :] = torch.rand(
197 + len(non_seeded_indices),
198 k,
199 dtype=self.probs_dtype,
200 device=target_probs.device)
@@ -272,27 +271,6 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
271 """
272 return torch.finfo(self.probs_dtype).tiny
273
275 - # partition batch into indices for which a generator is provided
276 - # and indicies for which no generator is provided
277 - @staticmethod
278 - def _split_batch_by_seeded(
279 - generators: List[Optional[torch.Generator]],
280 - k: int = 1,
281 - ) -> Tuple[List[int], Optional[List[int]]]:
282 -
283 - if all(generator is None for generator in generators):
284 - seed_indices: List[int] = []
285 - non_seed_indices: Optional[List[int]] = None
286 - else:
287 - seed_indices, non_seed_indices = [], []
288 - for i, generator in enumerate(generators):
289 - if generator is None:
290 - non_seed_indices.extend(range(k * i, k * (i + 1)))
291 - else:
292 - seed_indices.extend(range(k * i, k * (i + 1)))
293 -
294 - return seed_indices, non_seed_indices
295 -
274
275 # torch.multinomial forces a GPU<->CPU sync.
276 # Therefore, we use an optimized implementation instead that skips the sync.
@@ -304,9 +282,7 @@ def _multinomial(
282 probs: torch.Tensor,
283 num_samples: int,
284 k: int,
307 - generators: List[Optional[torch.Generator]],
308 - seed_indices: List[int],
309 - non_seed_indices: List[int],
285 + seeded_seqs: Dict[int, torch.Generator],
286 ) -> torch.Tensor:
287
288 if num_samples > 1:
@@ -315,13 +291,20 @@ def _multinomial(
291 probs = probs[:, None, :].expand(probs.shape[0], num_samples,
292 probs.shape[1]).contiguous().view(
293 -1, probs.shape[1])
318 -
294 q = torch.empty_like(probs)
320 - if len(seed_indices) == 0:
295 + if not seeded_seqs:
296 q.exponential_(1.0)
297 else:
323 - q[non_seed_indices].exponential_(1.0)
324 - for idx in seed_indices:
325 - q[idx].exponential_(1.0, generator=generators[idx // k])
298 + non_seeded_indices: List[int] = []
299 + start = 0
300 + for idx in range(len(q) // k):
301 + end = start + k
302 + generator = seeded_seqs.get(idx)
303 + if generator is None:
304 + non_seeded_indices.extend(list(range(start, end)))
305 + else:
306 + q[start:end].exponential_(1.0, generator=generator)
307 + start = end
308 + q[non_seeded_indices].exponential_(1.0)
309
310 return probs.div_(q).argmax(dim=1).view(-1, num_samples)
vllm/model_executor/layers/spec_decode_base_sampler.py
+2 -2
@@ -1,5 +1,5 @@
1 from abc import abstractmethod
2 -from typing import List, Optional
2 +from typing import Dict, Optional
3
4 import torch
5 import torch.jit
@@ -237,6 +237,6 @@ class SpecDecodeStochasticBaseSampler(SpecDecodeBaseSampler):
237 bonus_token_ids: torch.Tensor,
238 draft_probs: torch.Tensor,
239 draft_token_ids: torch.Tensor,
240 - generators: List[Optional[torch.Generator]],
240 + seeded_seqs: Optional[Dict[int, torch.Generator]] = None,
241 ) -> torch.Tensor:
242 raise NotImplementedError
vllm/model_executor/sampling_metadata.py
+13 -7
@@ -118,6 +118,7 @@ class SamplingMetadata:
118 query_lens: Optional[List[int]],
119 device: str,
120 pin_memory: bool,
121 + generators: Optional[Dict[str, torch.Generator]] = None,
122 ) -> "SamplingMetadata":
123 (
124 seq_groups,
@@ -125,7 +126,7 @@ class SamplingMetadata:
126 categorized_sample_indices,
127 num_prompts,
128 ) = _prepare_seq_groups(seq_group_metadata_list, seq_lens, query_lens,
128 - device)
129 + device, generators)
130 selected_token_indices = async_tensor_h2d(selected_token_indices,
131 dtype=torch.long,
132 target_device=device,
@@ -160,6 +161,7 @@ def _prepare_seq_groups(
161 seq_lens: List[int],
162 query_lens: Optional[List[int]],
163 device: str,
164 + generators: Optional[Dict[str, torch.Generator]] = None,
165 ) -> Tuple[List[SequenceGroupToSample], List[int], Dict[
166 SamplingType, List[Tuple[int, int]]], int]:
167 """Prepare sequence groups and indices for sampling.
@@ -170,8 +172,10 @@ def _prepare_seq_groups(
172 Index of prompt len should match with seq_group_metadata_list.
173 query_lens: A list of query lengths. Prompt lens include the length
174 of entire prompt tokens, and it could be shorter.
173 - device: A device to use for random number generator,
175 + device: A device to use for random number generators,
176 `SequenceGroupToSample.generator`.
177 + generators: A store of per-request random number generators used
178 + for seeded requests.
179
180 Returns:
181 seq_groups: A list of sequence group to sample.
@@ -217,8 +221,10 @@ def _prepare_seq_groups(
221
222 if seq_group_metadata.is_prompt:
223 if sampling_params.seed is not None:
220 - seq_group_metadata.state.generator = torch.Generator(
221 - device=device).manual_seed(sampling_params.seed)
224 + generator = torch.Generator(device=device).manual_seed(
225 + sampling_params.seed)
226 + if generators is not None:
227 + generators[seq_group_metadata.request_id] = generator
228
229 num_prompts += 1
230 num_prefill_sample = len(seq_ids)
@@ -235,6 +241,9 @@ def _prepare_seq_groups(
241 prompt_logprob_len = 0
242 sample_len = len(seq_ids) if do_sample else 0
243
244 + if sampling_params.seed is not None and generators is not None:
245 + generator = generators.get(seq_group_metadata.request_id)
246 +
247 # Update indices to select from the model output.
248 """
249 This blocks computes selected_token_indices which is used in the
@@ -279,9 +288,6 @@ def _prepare_seq_groups(
288 logit_idx += sample_len
289 sample_idx += sample_len
290
282 - if sampling_params.seed is not None:
283 - generator = seq_group_metadata.state.generator
284 -
291 seq_groups.append(
292 SequenceGroupToSample(
293 seq_ids=seq_ids,
vllm/sequence.py
-12
@@ -411,14 +411,6 @@ class Sequence:
411 f"num_blocks={self.n_blocks}, ")
412
413
414 -@dataclass
415 -class SequenceGroupState:
416 - """Mutable state tied to a specific sequence group"""
417 -
418 - # torch.Generator used in seeded sampling
419 - generator: Optional = None # type: ignore
420 -
421 -
414 class SequenceGroup:
415 """A group of sequences that are generated from the same prompt.
416
@@ -461,7 +453,6 @@ class SequenceGroup:
453 time_in_queue=None)
454 self.lora_request = lora_request
455 self.prompt_logprobs: Optional[PromptLogprobs] = None
464 - self.state = SequenceGroupState()
456 self.embeddings = embeddings
457 self.pooling_params = pooling_params
458 self.prompt_adapter_request = prompt_adapter_request
@@ -648,7 +639,6 @@ class SequenceGroupMetadata:
639 lora_request: LoRA request.
640 computed_block_nums: The block numbers that are already computed,
641 used in prefix caching.
651 - state: Internal state tied to this sequence group.
642 multi_modal_data: Multi modal data.
643 encoder_seq_data: Optional sequence data for encoder prompt
644 (SequenceGroup.encoder_seq). Should be None
@@ -674,7 +664,6 @@ class SequenceGroupMetadata:
664 token_chunk_size: Optional[int] = None,
665 lora_request: Optional[LoRARequest] = None,
666 computed_block_nums: Optional[List[int]] = None,
677 - state: Optional[SequenceGroupState] = None,
667 multi_modal_data: Optional["MultiModalDataDict"] = None,
668 encoder_seq_data: Optional[SequenceData] = None,
669 cross_block_table: Optional[List[int]] = None,
@@ -690,7 +679,6 @@ class SequenceGroupMetadata:
679 self.prompt_adapter_request = prompt_adapter_request
680 self.computed_block_nums = computed_block_nums
681 self.multi_modal_data = multi_modal_data
693 - self.state = SequenceGroupState() if state is None else state
682 self.encoder_seq_data = encoder_seq_data
683 self.cross_block_table = cross_block_table
684 self._token_chunk_size = token_chunk_size
vllm/spec_decode/batch_expansion.py
+22 -15
@@ -3,9 +3,9 @@ from typing import Iterator, List, Tuple
3
4 import torch
5
6 +from vllm import SamplingParams
7 from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
7 - SequenceGroupMetadata, SequenceGroupState,
8 - get_all_seq_ids)
8 + SequenceGroupMetadata, get_all_seq_ids)
9 from vllm.spec_decode.interfaces import (SpeculativeProposals,
10 SpeculativeScorer, SpeculativeScores)
11 from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
@@ -16,6 +16,8 @@ SeqId = int
16 TargetSeqId = int
17 TokenId = int
18
19 +DEFAULT_SIMPLE_SAMPLING_PARAMS = SamplingParams()
20 +
21
22 class BatchExpansionTop1Scorer(SpeculativeScorer):
23 """Implements a speculative scorer that uses batch expansion to get
@@ -247,24 +249,39 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
249 token_ids_to_score = self._get_token_ids_to_score(
250 proposal_token_ids[batch_index])
251
252 + # Use simpler sampling parameters apart from for final token
253 + # (in particular don't do seeded sampling) since those sampled tokens
254 + # aren't used.
255 + # We don't replace the sampling_params in the greedy case because
256 + # this also controls whether the probs get modified in the sampler
257 + # (see use of _modify_greedy_probs_inplace there).
258 + sampling_params = input_seq_group_metadata.sampling_params
259 + non_bonus_sampling_params = DEFAULT_SIMPLE_SAMPLING_PARAMS \
260 + if sampling_params.temperature else sampling_params
261 +
262 target_seq_group_metadata_list: List[SequenceGroupMetadata] = []
251 - for token_ids in token_ids_to_score:
263 + last_index = len(token_ids_to_score) - 1
264 + for i, token_ids in enumerate(token_ids_to_score):
265 + target_sampling_params = sampling_params if i == last_index \
266 + else non_bonus_sampling_params
267 target_seq_group_metadata_list.append(
268 self._create_single_target_seq_group_metadata(
269 input_seq_group_metadata,
270 input_seq_id,
271 next(target_seq_ids_iter),
272 token_ids,
273 + sampling_params=target_sampling_params,
274 ))
275
276 return target_seq_group_metadata_list
277
278 + @staticmethod
279 def _create_single_target_seq_group_metadata(
263 - self,
280 seq_group_metadata: SequenceGroupMetadata,
281 seq_id: SeqId,
282 target_seq_id: TargetSeqId,
283 token_ids: List[TokenId],
284 + sampling_params: SamplingParams,
285 ) -> SequenceGroupMetadata:
286 """Create a single target SequenceGroupMetadata.
287
@@ -293,26 +310,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
310 for data in new_seq_data_dict.values():
311 data.update_num_computed_tokens(data.get_len() - 1)
312
296 - if (seq_group_metadata.state is not None
297 - and seq_group_metadata.state.generator is not None):
298 - generator = torch.Generator(
299 - device=seq_group_metadata.state.generator.device)
300 - generator.set_state(seq_group_metadata.state.generator.get_state())
301 - state = SequenceGroupState(generator=generator)
302 - else:
303 - state = None
304 -
313 return SequenceGroupMetadata(
314 request_id=seq_group_metadata.request_id,
315 is_prompt=seq_group_metadata.is_prompt,
316 seq_data=new_seq_data_dict,
309 - sampling_params=seq_group_metadata.sampling_params,
317 + sampling_params=sampling_params,
318 block_tables={
319 target_seq_id: seq_group_metadata.block_tables[seq_id],
320 },
321 lora_request=None,
322 token_chunk_size=1,
315 - state=state,
323 )
324
325 def _split_scoring_output(
vllm/spec_decode/medusa_worker.py
+3 -1
@@ -57,9 +57,11 @@ class MedusaWorker(NonLLMProposerWorkerBase, Worker):
57 seq_lens, query_lens = self._prepare_input_tensors(
58 seq_group_metadata_list)
59
60 + generators = self.model_runner.get_generators(
61 + execute_model_req.finished_requests_ids)
62 sampling_metadata = SamplingMetadata.prepare(
63 seq_group_metadata_list, seq_lens, query_lens, self.device,
62 - self.model_runner.pin_memory)
64 + self.model_runner.pin_memory, generators)
65
66 model_outputs = self.model_runner.model.generate_proposals(
67 previous_hidden_states=execute_model_req.previous_hidden_states.
vllm/spec_decode/mlp_speculator_worker.py
+3 -1
@@ -38,9 +38,11 @@ class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
38 (input_tokens, seq_lens,
39 query_lens) = self._prepare_input_tensors(seq_group_metadata_list)
40
41 + generators = self.model_runner.get_generators(
42 + execute_model_req.finished_requests_ids)
43 sampling_metadata = SamplingMetadata.prepare(
44 seq_group_metadata_list, seq_lens, query_lens, self.device,
43 - self.model_runner.pin_memory)
45 + self.model_runner.pin_memory, generators)
46
47 model_outputs = self.model_runner.model.generate_proposals(
48 input_ids=input_tokens,
vllm/spec_decode/ngram_worker.py
+1 -2
@@ -7,10 +7,9 @@ from vllm.sequence import ExecuteModelRequest, SamplerOutput
7 from vllm.spec_decode.interfaces import SpeculativeProposals
8 from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
9 from vllm.spec_decode.top1_proposer import Top1Proposer
10 -from vllm.worker.worker_base import LoraNotSupportedWorkerBase
10
11
13 -class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
12 +class NGramWorker(NonLLMProposerWorkerBase):
13 """NGramWorker provides a light drafter without need for model.
14
15 Current NGramWorker only implements prompt lookup decoding,
vllm/spec_decode/spec_decode_worker.py
+11 -14
@@ -213,6 +213,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
213 """
214 self.proposer_worker = proposer_worker
215 self.scorer_worker = scorer_worker
216 + scorer_runner = getattr(self.scorer_worker, "model_runner", None)
217 + self.generators = scorer_runner.get_generators(
218 + ) if scorer_runner else None
219 self.disable_by_batch_size = disable_by_batch_size or float("inf")
220 self.spec_decode_sampler = spec_decode_sampler
221 self._allow_zero_draft_token_step = allow_zero_draft_token_step
@@ -591,20 +594,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
594 proposal_token_ids = proposals.proposal_token_ids[spec_indices]
595
596 # Sampler arguments
594 - sampler_extra_kwargs = {}
595 - if isinstance(self.spec_decode_sampler,
596 - SpecDecodeStochasticBaseSampler):
597 -
598 - # Get sequence group state
599 - generators = []
600 - for seq_group_metadata in seq_group_metadata_list:
601 - if (seq_group_metadata.state is not None
602 - and seq_group_metadata.state.generator is not None):
603 - generators.append(seq_group_metadata.state.generator)
604 - else:
605 - generators.append(None)
606 -
607 - sampler_extra_kwargs["generators"] = generators
597 + sampler_extra_kwargs: Dict[str, Any] = {}
598 + if self.generators and isinstance(self.spec_decode_sampler,
599 + SpecDecodeStochasticBaseSampler):
600 + sampler_extra_kwargs["seeded_seqs"] = {
601 + idx: self.generators[sgm.request_id]
602 + for idx, sgm in enumerate(seq_group_metadata_list)
603 + if sgm.sampling_params.seed is not None
604 + }
605
606 accepted_token_ids = self.spec_decode_sampler(
607 target_probs=proposal_verifier_probs,
vllm/worker/cpu_model_runner.py
+2 -1
@@ -337,7 +337,8 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
337 # just use seq_lens instead.
338 seq_lens,
339 self.device,
340 - pin_memory=False)
340 + pin_memory=False,
341 + generators=self.get_generators(finished_requests_ids))
342 return CPUModelInput(
343 input_tokens=input_tokens,
344 input_positions=input_positions,
vllm/worker/model_runner.py
+9 -5
@@ -1264,11 +1264,15 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1264 """
1265 model_input = self._prepare_model_input_tensors(
1266 seq_group_metadata_list, finished_requests_ids)
1267 - sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
1268 - model_input.seq_lens,
1269 - model_input.query_lens,
1270 - self.device,
1271 - self.pin_memory)
1267 + if get_pp_group().is_last_rank:
1268 + # Sampling metadata is only required for the final pp group
1269 + generators = self.get_generators(finished_requests_ids)
1270 + sampling_metadata = SamplingMetadata.prepare(
1271 + seq_group_metadata_list, model_input.seq_lens,
1272 + model_input.query_lens, self.device, self.pin_memory,
1273 + generators)
1274 + else:
1275 + sampling_metadata = None
1276 is_prompt = (seq_group_metadata_list[0].is_prompt
1277 if seq_group_metadata_list else None)
1278 return dataclasses.replace(model_input,
vllm/worker/model_runner_base.py
+15
@@ -139,6 +139,9 @@ class ModelRunnerBase(ABC, Generic[T]):
139 ModelRunnerInputBase subclass.
140 """
141
142 + # Map of request_id -> generator used for seeded random sampling
143 + generators: Dict[str, torch.Generator] = {}
144 +
145 @abstractmethod
146 def make_model_input_from_broadcasted_tensor_dict(
147 self,
@@ -176,3 +179,15 @@ class ModelRunnerBase(ABC, Generic[T]):
179 Execute the model on the given input.
180 """
181 raise NotImplementedError
182 +
183 + def get_generators(self, finished_request_ids: Optional[List[str]] = None):
184 + """
185 + Return dict of per-request generators used for random sampling.
186 + """
187 +
188 + # Clean up generators from completed requests
189 + if finished_request_ids:
190 + for request_id in finished_request_ids:
191 + self.generators.pop(request_id, None)
192 +
193 + return self.generators
vllm/worker/neuron_model_runner.py
+2 -1
@@ -219,7 +219,8 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
219 # just use seq_lens instead.
220 seq_lens,
221 self.device,
222 - self.pin_memory)
222 + self.pin_memory,
223 + generators=self.get_generators(finished_requests_ids))
224
225 return ModelInputForNeuron(input_tokens=input_tokens,
226 input_positions=input_positions,
vllm/worker/xpu_model_runner.py
+2 -1
@@ -246,7 +246,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
246 # just use seq_lens instead.
247 seq_lens,
248 self.device,
249 - pin_memory=False)
249 + pin_memory=False,
250 + generators=self.get_generators(finished_requests_ids))
251 # Broadcast the metadata.
252 metadata_dict = {
253 "input_tokens": input_tokens,