@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7d2dcce175
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Support per-request seed (#2514)

Nick Hill committed Feb 21, 2024 at 11:47 UTC 7d2dcce175cec00bc1d127d6b3a5f1ef73a6ba3c
10 files changed +289 -84
tests/samplers/test_sampler.py
+147 -75
@@ -1,10 +1,11 @@
1 import random
2 -from typing import Tuple
2 +from typing import Tuple, List
3 from unittest.mock import patch
4
5 import pytest
6 import torch
7 from transformers import GenerationConfig, GenerationMixin
8 +from typing import Optional
9
10 from vllm.model_executor.layers.sampler import Sampler
11 from vllm.model_executor.utils import set_random_seed
@@ -46,15 +47,13 @@ CUDA_DEVICES = [
47 ]
48
49
49 -@pytest.mark.parametrize("seed", RANDOM_SEEDS)
50 -@pytest.mark.parametrize("device", CUDA_DEVICES)
51 -def test_sampler_all_greedy(seed: int, device: str):
52 - set_random_seed(seed)
53 - torch.set_default_device(device)
54 - batch_size = random.randint(1, 256)
55 - input_tensor, fake_logits, sampler, model_runner = _prepare_test(
56 - batch_size)
57 -
50 +def _do_sample(
51 + batch_size: int,
52 + input_tensor: torch.Tensor,
53 + sampler: MockLogitsSampler,
54 + model_runner: ModelRunner,
55 + sampling_params: SamplingParams,
56 +):
57 seq_group_metadata_list = []
58 prompt_lens = []
59 for i in range(batch_size):
@@ -63,7 +62,7 @@ def test_sampler_all_greedy(seed: int, device: str):
62 request_id=f"test_{i}",
63 is_prompt=True,
64 seq_data={0: SequenceData([1, 2, 3])},
66 - sampling_params=SamplingParams(temperature=0, ),
65 + sampling_params=sampling_params,
66 block_tables={0: [1]},
67 ))
68 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
@@ -71,9 +70,23 @@ def test_sampler_all_greedy(seed: int, device: str):
70 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
71 prompt_lens,
72 subquery_lens=prompt_lens)
74 - sampler_output = sampler(embedding=None,
75 - hidden_states=input_tensor,
76 - sampling_metadata=sampling_metadata)
73 + return sampler(embedding=None,
74 + hidden_states=input_tensor,
75 + sampling_metadata=sampling_metadata)
76 +
77 +
78 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
79 +@pytest.mark.parametrize("device", CUDA_DEVICES)
80 +def test_sampler_all_greedy(seed: int, device: str):
81 + set_random_seed(seed)
82 + torch.set_default_device(device)
83 + batch_size = random.randint(1, 256)
84 + input_tensor, fake_logits, sampler, model_runner = _prepare_test(
85 + batch_size)
86 +
87 + sampling_params = SamplingParams(temperature=0)
88 + sampler_output = _do_sample(batch_size, input_tensor, sampler,
89 + model_runner, sampling_params)
90 expected = torch.argmax(fake_logits, dim=-1)
91 for i, sequence_output in enumerate(sampler_output):
92 for nth_output in sequence_output.samples:
@@ -94,28 +107,40 @@ def test_sampler_all_random(seed: int, device: str):
107 for i in range(batch_size):
108 fake_logits[i, i] = 1e2
109
97 - seq_group_metadata_list = []
98 - prompt_lens = []
110 + sampling_params = SamplingParams(
111 + temperature=1.0,
112 + n=random.randint(1, 10),
113 + )
114 + sampler_output = _do_sample(batch_size, input_tensor, sampler,
115 + model_runner, sampling_params)
116 +
117 + for i, sequence_output in enumerate(sampler_output):
118 + for nth_output in sequence_output.samples:
119 + assert nth_output.output_token == i
120 +
121 + del model_runner
122 +
123 +
124 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
125 +@pytest.mark.parametrize("device", CUDA_DEVICES)
126 +def test_sampler_all_random_seed(seed: int, device: str):
127 + set_random_seed(seed)
128 + torch.set_default_device(device)
129 + batch_size = random.randint(1, 256)
130 + input_tensor, fake_logits, sampler, model_runner = _prepare_test(
131 + batch_size)
132 +
133 for i in range(batch_size):
100 - seq_group_metadata_list.append(
101 - SequenceGroupMetadata(
102 - request_id=f"test_{i}",
103 - is_prompt=True,
104 - seq_data={0: SequenceData([1, 2, 3])},
105 - sampling_params=SamplingParams(
106 - temperature=1.0,
107 - n=random.randint(1, 10),
108 - ),
109 - block_tables={0: [1]},
110 - ))
111 - prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
134 + fake_logits[i, i] = 1e2
135 +
136 + sampling_params = SamplingParams(
137 + temperature=1.0,
138 + n=random.randint(1, 10),
139 + seed=random.randint(0, 10000),
140 + )
141 + sampler_output = _do_sample(batch_size, input_tensor, sampler,
142 + model_runner, sampling_params)
143
113 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
114 - prompt_lens,
115 - subquery_lens=prompt_lens)
116 - sampler_output = sampler(embedding=None,
117 - hidden_states=input_tensor,
118 - sampling_metadata=sampling_metadata)
144 for i, sequence_output in enumerate(sampler_output):
145 for nth_output in sequence_output.samples:
146 assert nth_output.output_token == i
@@ -123,6 +148,31 @@ def test_sampler_all_random(seed: int, device: str):
148 del model_runner
149
150
151 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
152 +@pytest.mark.parametrize("device", CUDA_DEVICES)
153 +def test_sampler_all_random_seed_deterministic(seed: int, device: str):
154 + set_random_seed(seed)
155 + torch.set_default_device(device)
156 + batch_size = random.randint(1, 256)
157 + input_tensor, fake_logits, sampler, model_runner = _prepare_test(
158 + batch_size)
159 +
160 + sampling_params = SamplingParams(
161 + temperature=1.0,
162 + n=random.randint(1, 10),
163 + seed=random.randint(0, 10000),
164 + )
165 + first_sampler_output = _do_sample(batch_size, input_tensor, sampler,
166 + model_runner, sampling_params)
167 +
168 + second_sampler_output = _do_sample(batch_size, input_tensor, sampler,
169 + model_runner, sampling_params)
170 +
171 + assert first_sampler_output == second_sampler_output
172 +
173 + del model_runner
174 +
175 +
176 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
177 @pytest.mark.parametrize("device", CUDA_DEVICES)
178 def test_sampler_all_beam(seed: int, device: str):
@@ -131,29 +181,13 @@ def test_sampler_all_beam(seed: int, device: str):
181 batch_size = random.randint(1, 256)
182 input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
183
134 - seq_group_metadata_list = []
135 - prompt_lens = []
136 - for i in range(batch_size):
137 - seq_group_metadata_list.append(
138 - SequenceGroupMetadata(
139 - request_id=f"test_{i}",
140 - is_prompt=True,
141 - seq_data={0: SequenceData([1, 2, 3])},
142 - sampling_params=SamplingParams(
143 - temperature=0,
144 - best_of=2,
145 - use_beam_search=True,
146 - ),
147 - block_tables={0: [1]},
148 - ))
149 - prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
150 -
151 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
152 - prompt_lens,
153 - subquery_lens=prompt_lens)
154 - sampler(embedding=None,
155 - hidden_states=input_tensor,
156 - sampling_metadata=sampling_metadata)
184 + sampling_params = SamplingParams(
185 + temperature=0,
186 + best_of=2,
187 + use_beam_search=True,
188 + )
189 + _do_sample(batch_size, input_tensor, sampler, model_runner,
190 + sampling_params)
191 # no assertion here as I am not sure how to determine whether
192 # the outputs are expected - in other words, this just tests
193 # whether there are no exceptions in the sampler
@@ -171,14 +205,15 @@ def test_sampler_mixed(seed: int, device: str):
205 batch_size)
206
207 seq_group_metadata_list = []
174 - expected_tokens = []
208 + expected_tokens: List[Optional[List[int]]] = []
209 prompt_lens = []
210 for i in range(batch_size):
177 - n = 1
178 - sampling_type = random.randint(0, 2)
211 + expected: Optional[List[int]] = None
212 + sampling_type = random.randint(0, 3)
213 if sampling_type == 0:
214 sampling_params = SamplingParams(temperature=0)
181 - elif sampling_type == 1:
215 + expected = [torch.argmax(fake_logits[i], dim=-1).item()]
216 + elif sampling_type in (1, 2):
217 n = random.randint(1, 10)
218 sampling_params = SamplingParams(
219 temperature=random.random() + 0.1,
@@ -187,13 +222,17 @@ def test_sampler_mixed(seed: int, device: str):
222 n=n,
223 presence_penalty=random.randint(0, 1),
224 )
225 + if sampling_type == 2:
226 + sampling_params.seed = random.randint(0, 10000)
227 + else:
228 + for idx in range(n):
229 + fake_logits[i, i + idx] = 1e2
230 + expected = list(range(i, i + n))
231 else:
232 sampling_params = SamplingParams(temperature=0,
233 use_beam_search=True,
234 best_of=2)
194 - for idx in range(n):
195 - fake_logits[i, i + idx] = 1e2
196 - expected_tokens.append(i + idx)
235 + expected_tokens.append(expected)
236 seq_group_metadata_list.append(
237 SequenceGroupMetadata(
238 request_id=f"test_{i}",
@@ -204,17 +243,50 @@ def test_sampler_mixed(seed: int, device: str):
243 ))
244 prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
245
207 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
208 - prompt_lens,
209 - subquery_lens=prompt_lens)
210 - sampler_output = sampler(embedding=None,
211 - hidden_states=input_tensor,
212 - sampling_metadata=sampling_metadata)
213 - for i, sequence_output in enumerate(sampler_output):
214 - if seq_group_metadata_list[i].sampling_params.use_beam_search:
215 - continue
216 - for nth_output in sequence_output.samples:
217 - assert nth_output.output_token in expected_tokens
246 + def test_sampling(model_runner: ModelRunner):
247 + sampling_metadata = model_runner._prepare_sample(
248 + seq_group_metadata_list, prompt_lens, subquery_lens=prompt_lens)
249 + sampler_output = sampler(embedding=None,
250 + hidden_states=input_tensor,
251 + sampling_metadata=sampling_metadata)
252 +
253 + for i, (sequence_output, metadata) in enumerate(
254 + zip(sampler_output, seq_group_metadata_list)):
255 + if metadata.sampling_params.use_beam_search:
256 + continue
257 +
258 + if metadata.sampling_params.seed is not None \
259 + and expected_tokens[i] is None:
260 + # Record seeded random result to compare with results of second invocation
261 + expected_tokens[i] = [
262 + nth_output.output_token
263 + for nth_output in sequence_output.samples
264 + ]
265 + continue
266 +
267 + for n, nth_output in enumerate(sequence_output.samples):
268 + if metadata.sampling_params.temperature == 0 or metadata.sampling_params.seed is not None:
269 + # Ensure exact matches for greedy or random with seed
270 + assert nth_output.output_token == expected_tokens[i][n]
271 + else:
272 + # For non-seeded random check that one of the high-logit tokens were chosen
273 + assert nth_output.output_token in expected_tokens[i]
274 +
275 + # Test batch
276 + test_sampling(model_runner)
277 +
278 + # Shuffle the batch and resample
279 + target_index = list(range(batch_size))
280 + for list_to_shuffle in (target_index, seq_group_metadata_list,
281 + expected_tokens, prompt_lens):
282 + random.Random(seed).shuffle(list_to_shuffle)
283 + target_index = torch.tensor(target_index)
284 + input_tensor.data = input_tensor.index_select(0, target_index)
285 + fake_logits.data = fake_logits.index_select(0, target_index)
286 +
287 + # This time, results of seeded random samples will be compared with the corresponding
288 + # sample in the pre-shuffled batch
289 + test_sampling(model_runner)
290
291 del model_runner
292
tests/samplers/test_seeded_generate.py new
+82
@@ -0,0 +1,82 @@
1 +"""Verify that seeded random sampling is deterministic.
2 +
3 +Run `pytest tests/samplers/test_seeded_generate.py --forked`.
4 +"""
5 +import copy
6 +import random
7 +from itertools import combinations
8 +
9 +import pytest
10 +
11 +from vllm.model_executor.utils import set_random_seed
12 +from vllm import SamplingParams
13 +
14 +MODEL = "facebook/opt-125m"
15 +RANDOM_SEEDS = list(range(5))
16 +
17 +
18 +@pytest.fixture
19 +def vllm_model(vllm_runner):
20 + vllm_model = vllm_runner(MODEL, dtype="half")
21 + yield vllm_model
22 + del vllm_model
23 +
24 +
25 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
26 +def test_random_sample_with_seed(
27 + vllm_model,
28 + example_prompts,
29 + seed: int,
30 +) -> None:
31 + set_random_seed(seed)
32 +
33 + sampling_params = SamplingParams(
34 + # Parameters to ensure sufficient randomness
35 + temperature=2.0,
36 + top_p=min(random.random() + 0.3, 1),
37 + top_k=random.randint(5, 20),
38 + n=random.randint(1, 10),
39 + presence_penalty=random.randint(0, 1),
40 + max_tokens=8,
41 + ignore_eos=True,
42 + )
43 +
44 + sampling_params_seed_1 = copy.deepcopy(sampling_params)
45 + sampling_params_seed_1.seed = 100
46 + sampling_params_seed_2 = copy.deepcopy(sampling_params)
47 + sampling_params_seed_2.seed = 200
48 +
49 + llm = vllm_model.model
50 +
51 + for prompt in example_prompts:
52 + for params in (
53 + sampling_params,
54 + sampling_params_seed_1,
55 + sampling_params_seed_2,
56 + sampling_params,
57 + sampling_params_seed_1,
58 + sampling_params_seed_2,
59 + ):
60 + llm._add_request(
61 + prompt=prompt,
62 + prompt_token_ids=None,
63 + sampling_params=params,
64 + )
65 +
66 + results = llm._run_engine(use_tqdm=False)
67 + all_outputs = [[out.token_ids for out in output.outputs]
68 + for output in results]
69 +
70 + for i in range(0, len(example_prompts), 6):
71 + outputs = all_outputs[i:i + 6]
72 +
73 + # verify all non-seeded requests differ
74 + for output_a, output_b in combinations(
75 + (outputs[0], outputs[1], outputs[2], outputs[3]),
76 + 2,
77 + ):
78 + assert output_a != output_b
79 +
80 + # verify requests with the same seed match
81 + assert outputs[1] == outputs[4]
82 + assert outputs[2] == outputs[5]
vllm/core/scheduler.py
+1
@@ -387,6 +387,7 @@ class Scheduler:
387 block_tables=block_tables,
388 lora_request=seq_group.lora_request,
389 prefix=seq_group.prefix,
390 + state=seq_group.state,
391 )
392 seq_group_metadata_list.append(seq_group_metadata)
393 return seq_group_metadata_list, scheduler_outputs
vllm/engine/arg_utils.py
-1
@@ -173,7 +173,6 @@ class EngineArgs:
173 default=EngineArgs.block_size,
174 choices=[8, 16, 32],
175 help='token block size')
176 - # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
176 parser.add_argument('--seed',
177 type=int,
178 default=EngineArgs.seed,
vllm/entrypoints/openai/protocol.py
+4
@@ -60,6 +60,7 @@ class ChatCompletionRequest(BaseModel):
60 top_p: Optional[float] = 1.0
61 n: Optional[int] = 1
62 max_tokens: Optional[int] = None
63 + seed: Optional[int] = None
64 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
65 stream: Optional[bool] = False
66 presence_penalty: Optional[float] = 0.0
@@ -90,6 +91,7 @@ class ChatCompletionRequest(BaseModel):
91 temperature=self.temperature,
92 top_p=self.top_p,
93 min_p=self.min_p,
94 + seed=self.seed,
95 stop=self.stop,
96 stop_token_ids=self.stop_token_ids,
97 max_tokens=self.max_tokens,
@@ -117,6 +119,7 @@ class CompletionRequest(BaseModel):
119 logprobs: Optional[int] = None
120 echo: Optional[bool] = False
121 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
122 + seed: Optional[int] = None
123 presence_penalty: Optional[float] = 0.0
124 frequency_penalty: Optional[float] = 0.0
125 best_of: Optional[int] = None
@@ -147,6 +150,7 @@ class CompletionRequest(BaseModel):
150 top_p=self.top_p,
151 top_k=self.top_k,
152 min_p=self.min_p,
153 + seed=self.seed,
154 stop=self.stop,
155 stop_token_ids=self.stop_token_ids,
156 ignore_eos=self.ignore_eos,
vllm/model_executor/layers/sampler.py
+22 -7
@@ -342,7 +342,9 @@ def _beam_search_sample(
342 def _multinomial(
343 probs: torch.Tensor,
344 num_samples: int,
345 -):
345 + seq_groups: Optional[List[Tuple[List[int], SamplingParams]]] = None,
346 + generators: Optional[List[torch.Generator]] = None,
347 +) -> torch.Tensor:
348 if num_samples > 1:
349 # This is equivalent to torch.repeat_interleaved (which also
350 # forces a GPU<->CPU sync).
@@ -352,7 +354,15 @@ def _multinomial(
354 probs = probs[:, None, :].expand(probs.shape[0], num_samples,
355 probs.shape[1]).contiguous().view(
356 -1, probs.shape[1])
355 - q = torch.empty_like(probs).exponential_(1)
357 + q = torch.empty_like(probs)
358 + if seq_groups is None:
359 + q.exponential_()
360 + else:
361 + sample_idx = 0
362 + for (seq_ids, _), generator in zip(seq_groups, generators):
363 + next_sample_idx = sample_idx + len(seq_ids) * num_samples
364 + q[sample_idx:next_sample_idx].exponential_(generator=generator)
365 + sample_idx = next_sample_idx
366 return probs.div_(q).argmax(dim=1).view(-1, num_samples)
367
368
@@ -370,6 +380,7 @@ def _sample(
380
381 sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
382 sample_metadata = {}
383 + multinomial_samples = {}
384
385 # Counterintiutively, having two loops here is actually faster.
386 # The first loop can run without waiting on GPU<->CPU sync.
@@ -385,14 +396,18 @@ def _sample(
396 is_prompts, sample_indices)
397 if sampling_type == SamplingType.GREEDY:
398 greedy_samples = torch.argmax(logprobs[sample_indices], dim=-1)
388 - elif sampling_type == SamplingType.RANDOM:
399 + elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
400 max_best_of = 1
401 for seq_group, is_prompt in zip(seq_groups, is_prompts):
402 if is_prompt:
403 _, sampling_params = seq_group
404 max_best_of = max(max_best_of, sampling_params.best_of)
394 - multinomial_samples = _multinomial(probs[sample_indices],
395 - max_best_of)
405 + seeded_args = {} if sampling_type == SamplingType.RANDOM else {
406 + "seq_groups": seq_groups,
407 + "generators": sampling_metadata.generators,
408 + }
409 + multinomial_samples[sampling_type] = _multinomial(
410 + probs[sample_indices], max_best_of, **seeded_args)
411 elif sampling_type == SamplingType.BEAM:
412 beam_search_logprobs = logprobs[sample_indices]
413 else:
@@ -407,9 +422,9 @@ def _sample(
422 sampling_type]
423 if sampling_type == SamplingType.GREEDY:
424 sample_results = _greedy_sample(seq_groups, greedy_samples)
410 - elif sampling_type == SamplingType.RANDOM:
425 + elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
426 sample_results = _random_sample(seq_groups, is_prompts,
412 - multinomial_samples)
427 + multinomial_samples[sampling_type])
428 elif sampling_type == SamplingType.BEAM:
429 sample_results = _beam_search_sample(seq_groups, is_prompts,
430 sampling_metadata.seq_data,
vllm/model_executor/sampling_metadata.py
+3
@@ -19,6 +19,7 @@ class SamplingMetadata:
19 prompt_lens: Lengths of prompts.
20 selected_token_indices: Token indices selected for sampling.
21 categorized_sample_indices: SamplingType -> token indices to sample.
22 + generators: List of torch.Generators to use for seeded sampling
23 perform_sampling: Whether to perform sampling. This option is used to
24 make the sampling only happens in the driver worker, and disable
25 sampling in other worker processes.
@@ -31,6 +32,7 @@ class SamplingMetadata:
32 prompt_lens: Optional[List[int]],
33 selected_token_indices: torch.Tensor,
34 categorized_sample_indices: Optional[Dict[SamplingType, torch.Tensor]],
35 + generators: Optional[List[torch.Generator]] = None,
36 perform_sampling: bool = True,
37 ) -> None:
38 self.seq_groups = seq_groups
@@ -38,6 +40,7 @@ class SamplingMetadata:
40 self.prompt_lens = prompt_lens
41 self.selected_token_indices = selected_token_indices
42 self.categorized_sample_indices = categorized_sample_indices
43 + self.generators = generators
44 self.perform_sampling = perform_sampling
45
46 self.num_prompts = len(prompt_lens) if prompt_lens is not None else 0
vllm/sampling_params.py
+8 -1
@@ -11,7 +11,8 @@ _SAMPLING_EPS = 1e-5
11 class SamplingType(IntEnum):
12 GREEDY = 0
13 RANDOM = 1
14 - BEAM = 2
14 + RANDOM_SEED = 2
15 + BEAM = 3
16
17
18 LogitsProcessor = Callable[[List[int], torch.Tensor], torch.Tensor]
@@ -56,6 +57,7 @@ class SamplingParams:
57 min_p: Float that represents the minimum probability for a token to be
58 considered, relative to the probability of the most likely token.
59 Must be in [0, 1]. Set to 0 to disable this.
60 + seed: Random seed to use for the generation.
61 use_beam_search: Whether to use beam search instead of sampling.
62 length_penalty: Float that penalizes sequences based on their length.
63 Used in beam search.
@@ -101,6 +103,7 @@ class SamplingParams:
103 top_p: float = 1.0,
104 top_k: int = -1,
105 min_p: float = 0.0,
106 + seed: Optional[int] = None,
107 use_beam_search: bool = False,
108 length_penalty: float = 1.0,
109 early_stopping: Union[bool, str] = False,
@@ -124,6 +127,7 @@ class SamplingParams:
127 self.top_p = top_p
128 self.top_k = top_k
129 self.min_p = min_p
130 + self.seed = seed
131 self.use_beam_search = use_beam_search
132 self.length_penalty = length_penalty
133 self.early_stopping = early_stopping
@@ -229,6 +233,8 @@ class SamplingParams:
233 return SamplingType.BEAM
234 if self.temperature < _SAMPLING_EPS:
235 return SamplingType.GREEDY
236 + if self.seed is not None:
237 + return SamplingType.RANDOM_SEED
238 return SamplingType.RANDOM
239
240 def __repr__(self) -> str:
@@ -242,6 +248,7 @@ class SamplingParams:
248 f"top_p={self.top_p}, "
249 f"top_k={self.top_k}, "
250 f"min_p={self.min_p}, "
251 + f"seed={self.seed}, "
252 f"use_beam_search={self.use_beam_search}, "
253 f"length_penalty={self.length_penalty}, "
254 f"early_stopping={self.early_stopping}, "
vllm/sequence.py
+12
@@ -248,6 +248,14 @@ class Sequence:
248 f"num_blocks={len(self.logical_token_blocks)})")
249
250
251 +@dataclass
252 +class SequenceGroupState:
253 + """Mutable state tied to a specific sequence group"""
254 +
255 + # torch.Generator used in seeded sampling
256 + generator: Optional = None
257 +
258 +
259 class SequenceGroup:
260 """A group of sequences that are generated from the same prompt.
261
@@ -280,6 +288,7 @@ class SequenceGroup:
288 self.lora_request = lora_request
289 self.prefix: Optional[Prefix] = prefix
290 self.prompt_logprobs: Optional[PromptLogprobs] = None
291 + self.state = SequenceGroupState()
292
293 @property
294 def prompt(self) -> str:
@@ -397,6 +406,7 @@ class SequenceGroupMetadata:
406 sampling_params: The sampling parameters used to generate the outputs.
407 block_tables: The block tables. (Seq id -> list of physical block
408 numbers)
409 + state: Internal state tied to this sequence group.
410 lora_request: LoRA request.
411 prefix: The prefix of the prompt of the sequence group.
412 """
@@ -410,6 +420,7 @@ class SequenceGroupMetadata:
420 block_tables: Dict[int, List[int]],
421 lora_request: Optional[LoRARequest] = None,
422 prefix: Optional[Prefix] = None,
423 + state: Optional[SequenceGroupState] = None,
424 ) -> None:
425 self.request_id = request_id
426 self.is_prompt = is_prompt
@@ -418,6 +429,7 @@ class SequenceGroupMetadata:
429 self.block_tables = block_tables
430 self.lora_request = lora_request
431 self.prefix = prefix
432 + self.state = SequenceGroupState() if state is None else state
433
434 @property
435 def lora_int_id(self) -> int:
vllm/worker/model_runner.py
+10
@@ -389,6 +389,7 @@ class ModelRunner:
389 ) -> SamplingMetadata:
390 seq_groups: List[Tuple[List[int], SamplingParams]] = []
391 selected_token_indices: List[int] = []
392 + generators: List[torch.Generator] = []
393 selected_token_start_idx = 0
394 categorized_sample_indices = {t: [] for t in SamplingType}
395 categorized_sample_indices_start_idx = 0
@@ -419,6 +420,10 @@ class ModelRunner:
420 selected_token_indices.append(selected_token_start_idx +
421 subquery_len - 1)
422 selected_token_start_idx += max_subquery_len
423 +
424 + if sampling_params.seed is not None:
425 + seq_group_metadata.state.generator = torch.Generator(
426 + device="cuda").manual_seed(sampling_params.seed)
427 else:
428 num_seqs = len(seq_ids)
429 selected_token_indices.extend(
@@ -432,6 +437,9 @@ class ModelRunner:
437 categorized_sample_indices_start_idx + num_seqs))
438 categorized_sample_indices_start_idx += num_seqs
439
440 + if sampling_params.seed is not None:
441 + generators.append(seq_group_metadata.state.generator)
442 +
443 selected_token_indices = _async_h2d(selected_token_indices,
444 dtype=torch.long,
445 target_device=self.device,
@@ -454,6 +462,7 @@ class ModelRunner:
462 prompt_lens=prompt_lens,
463 selected_token_indices=selected_token_indices,
464 categorized_sample_indices=categorized_sample_indices,
465 + generators=generators,
466 )
467 return sampling_metadata
468
@@ -536,6 +545,7 @@ class ModelRunner:
545 prompt_lens=None,
546 selected_token_indices=metadata_dict["selected_token_indices"],
547 categorized_sample_indices=None,
548 + generators=None,
549 perform_sampling=False,
550 )
551