A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Support per-request seed (#2514)
Nick Hill committed
Feb 21, 2024 at 11:47 UTC
7d2dcce175cec00bc1d127d6b3a5f1ef73a6ba3c
10 files changed
+289
-84
tests/samplers/test_sampler.py
+147
-75
@@ -1,10 +1,11 @@
1
import random
2
-from typing import Tuple
2
+from typing import Tuple, List
3
from unittest.mock import patch
4
5
import pytest
6
import torch
7
from transformers import GenerationConfig, GenerationMixin
8
+from typing import Optional
9
10
from vllm.model_executor.layers.sampler import Sampler
11
from vllm.model_executor.utils import set_random_seed
@@ -46,15 +47,13 @@ CUDA_DEVICES = [
47
]
48
49
49
-@pytest.mark.parametrize("seed", RANDOM_SEEDS)
50
-@pytest.mark.parametrize("device", CUDA_DEVICES)
51
-def test_sampler_all_greedy(seed: int, device: str):
52
- set_random_seed(seed)
53
- torch.set_default_device(device)
54
- batch_size = random.randint(1, 256)
55
- input_tensor, fake_logits, sampler, model_runner = _prepare_test(
56
- batch_size)
57
-
50
+def _do_sample(
51
+ batch_size: int,
52
+ input_tensor: torch.Tensor,
53
+ sampler: MockLogitsSampler,
54
+ model_runner: ModelRunner,
55
+ sampling_params: SamplingParams,
56
+):
57
seq_group_metadata_list = []
58
prompt_lens = []
59
for i in range(batch_size):
@@ -63,7 +62,7 @@ def test_sampler_all_greedy(seed: int, device: str):
62
request_id=f"test_{i}",
63
is_prompt=True,
64
seq_data={0: SequenceData([1, 2, 3])},
66
- sampling_params=SamplingParams(temperature=0, ),
65
+ sampling_params=sampling_params,
66
block_tables={0: [1]},
67
))
68
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
@@ -71,9 +70,23 @@ def test_sampler_all_greedy(seed: int, device: str):
70
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
71
prompt_lens,
72
subquery_lens=prompt_lens)
74
- sampler_output = sampler(embedding=None,
75
- hidden_states=input_tensor,
76
- sampling_metadata=sampling_metadata)
73
+ return sampler(embedding=None,
74
+ hidden_states=input_tensor,
75
+ sampling_metadata=sampling_metadata)
76
+
77
+
78
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
79
+@pytest.mark.parametrize("device", CUDA_DEVICES)
80
+def test_sampler_all_greedy(seed: int, device: str):
81
+ set_random_seed(seed)
82
+ torch.set_default_device(device)
83
+ batch_size = random.randint(1, 256)
84
+ input_tensor, fake_logits, sampler, model_runner = _prepare_test(
85
+ batch_size)
86
+
87
+ sampling_params = SamplingParams(temperature=0)
88
+ sampler_output = _do_sample(batch_size, input_tensor, sampler,
89
+ model_runner, sampling_params)
90
expected = torch.argmax(fake_logits, dim=-1)
91
for i, sequence_output in enumerate(sampler_output):
92
for nth_output in sequence_output.samples:
@@ -94,28 +107,40 @@ def test_sampler_all_random(seed: int, device: str):
107
for i in range(batch_size):
108
fake_logits[i, i] = 1e2
109
97
- seq_group_metadata_list = []
98
- prompt_lens = []
110
+ sampling_params = SamplingParams(
111
+ temperature=1.0,
112
+ n=random.randint(1, 10),
113
+ )
114
+ sampler_output = _do_sample(batch_size, input_tensor, sampler,
115
+ model_runner, sampling_params)
116
+
117
+ for i, sequence_output in enumerate(sampler_output):
118
+ for nth_output in sequence_output.samples:
119
+ assert nth_output.output_token == i
120
+
121
+ del model_runner
122
+
123
+
124
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
125
+@pytest.mark.parametrize("device", CUDA_DEVICES)
126
+def test_sampler_all_random_seed(seed: int, device: str):
127
+ set_random_seed(seed)
128
+ torch.set_default_device(device)
129
+ batch_size = random.randint(1, 256)
130
+ input_tensor, fake_logits, sampler, model_runner = _prepare_test(
131
+ batch_size)
132
+
133
for i in range(batch_size):
100
- seq_group_metadata_list.append(
101
- SequenceGroupMetadata(
102
- request_id=f"test_{i}",
103
- is_prompt=True,
104
- seq_data={0: SequenceData([1, 2, 3])},
105
- sampling_params=SamplingParams(
106
- temperature=1.0,
107
- n=random.randint(1, 10),
108
- ),
109
- block_tables={0: [1]},
110
- ))
111
- prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
134
+ fake_logits[i, i] = 1e2
135
+
136
+ sampling_params = SamplingParams(
137
+ temperature=1.0,
138
+ n=random.randint(1, 10),
139
+ seed=random.randint(0, 10000),
140
+ )
141
+ sampler_output = _do_sample(batch_size, input_tensor, sampler,
142
+ model_runner, sampling_params)
143
113
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
114
- prompt_lens,
115
- subquery_lens=prompt_lens)
116
- sampler_output = sampler(embedding=None,
117
- hidden_states=input_tensor,
118
- sampling_metadata=sampling_metadata)
144
for i, sequence_output in enumerate(sampler_output):
145
for nth_output in sequence_output.samples:
146
assert nth_output.output_token == i
@@ -123,6 +148,31 @@ def test_sampler_all_random(seed: int, device: str):
148
del model_runner
149
150
151
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
152
+@pytest.mark.parametrize("device", CUDA_DEVICES)
153
+def test_sampler_all_random_seed_deterministic(seed: int, device: str):
154
+ set_random_seed(seed)
155
+ torch.set_default_device(device)
156
+ batch_size = random.randint(1, 256)
157
+ input_tensor, fake_logits, sampler, model_runner = _prepare_test(
158
+ batch_size)
159
+
160
+ sampling_params = SamplingParams(
161
+ temperature=1.0,
162
+ n=random.randint(1, 10),
163
+ seed=random.randint(0, 10000),
164
+ )
165
+ first_sampler_output = _do_sample(batch_size, input_tensor, sampler,
166
+ model_runner, sampling_params)
167
+
168
+ second_sampler_output = _do_sample(batch_size, input_tensor, sampler,
169
+ model_runner, sampling_params)
170
+
171
+ assert first_sampler_output == second_sampler_output
172
+
173
+ del model_runner
174
+
175
+
176
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
177
@pytest.mark.parametrize("device", CUDA_DEVICES)
178
def test_sampler_all_beam(seed: int, device: str):
@@ -131,29 +181,13 @@ def test_sampler_all_beam(seed: int, device: str):
181
batch_size = random.randint(1, 256)
182
input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
183
134
- seq_group_metadata_list = []
135
- prompt_lens = []
136
- for i in range(batch_size):
137
- seq_group_metadata_list.append(
138
- SequenceGroupMetadata(
139
- request_id=f"test_{i}",
140
- is_prompt=True,
141
- seq_data={0: SequenceData([1, 2, 3])},
142
- sampling_params=SamplingParams(
143
- temperature=0,
144
- best_of=2,
145
- use_beam_search=True,
146
- ),
147
- block_tables={0: [1]},
148
- ))
149
- prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
150
-
151
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
152
- prompt_lens,
153
- subquery_lens=prompt_lens)
154
- sampler(embedding=None,
155
- hidden_states=input_tensor,
156
- sampling_metadata=sampling_metadata)
184
+ sampling_params = SamplingParams(
185
+ temperature=0,
186
+ best_of=2,
187
+ use_beam_search=True,
188
+ )
189
+ _do_sample(batch_size, input_tensor, sampler, model_runner,
190
+ sampling_params)
191
# no assertion here as I am not sure how to determine whether
192
# the outputs are expected - in other words, this just tests
193
# whether there are no exceptions in the sampler
@@ -171,14 +205,15 @@ def test_sampler_mixed(seed: int, device: str):
205
batch_size)
206
207
seq_group_metadata_list = []
174
- expected_tokens = []
208
+ expected_tokens: List[Optional[List[int]]] = []
209
prompt_lens = []
210
for i in range(batch_size):
177
- n = 1
178
- sampling_type = random.randint(0, 2)
211
+ expected: Optional[List[int]] = None
212
+ sampling_type = random.randint(0, 3)
213
if sampling_type == 0:
214
sampling_params = SamplingParams(temperature=0)
181
- elif sampling_type == 1:
215
+ expected = [torch.argmax(fake_logits[i], dim=-1).item()]
216
+ elif sampling_type in (1, 2):
217
n = random.randint(1, 10)
218
sampling_params = SamplingParams(
219
temperature=random.random() + 0.1,
@@ -187,13 +222,17 @@ def test_sampler_mixed(seed: int, device: str):
222
n=n,
223
presence_penalty=random.randint(0, 1),
224
)
225
+ if sampling_type == 2:
226
+ sampling_params.seed = random.randint(0, 10000)
227
+ else:
228
+ for idx in range(n):
229
+ fake_logits[i, i + idx] = 1e2
230
+ expected = list(range(i, i + n))
231
else:
232
sampling_params = SamplingParams(temperature=0,
233
use_beam_search=True,
234
best_of=2)
194
- for idx in range(n):
195
- fake_logits[i, i + idx] = 1e2
196
- expected_tokens.append(i + idx)
235
+ expected_tokens.append(expected)
236
seq_group_metadata_list.append(
237
SequenceGroupMetadata(
238
request_id=f"test_{i}",
@@ -204,17 +243,50 @@ def test_sampler_mixed(seed: int, device: str):
243
))
244
prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
245
207
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
208
- prompt_lens,
209
- subquery_lens=prompt_lens)
210
- sampler_output = sampler(embedding=None,
211
- hidden_states=input_tensor,
212
- sampling_metadata=sampling_metadata)
213
- for i, sequence_output in enumerate(sampler_output):
214
- if seq_group_metadata_list[i].sampling_params.use_beam_search:
215
- continue
216
- for nth_output in sequence_output.samples:
217
- assert nth_output.output_token in expected_tokens
246
+ def test_sampling(model_runner: ModelRunner):
247
+ sampling_metadata = model_runner._prepare_sample(
248
+ seq_group_metadata_list, prompt_lens, subquery_lens=prompt_lens)
249
+ sampler_output = sampler(embedding=None,
250
+ hidden_states=input_tensor,
251
+ sampling_metadata=sampling_metadata)
252
+
253
+ for i, (sequence_output, metadata) in enumerate(
254
+ zip(sampler_output, seq_group_metadata_list)):
255
+ if metadata.sampling_params.use_beam_search:
256
+ continue
257
+
258
+ if metadata.sampling_params.seed is not None \
259
+ and expected_tokens[i] is None:
260
+ # Record seeded random result to compare with results of second invocation
261
+ expected_tokens[i] = [
262
+ nth_output.output_token
263
+ for nth_output in sequence_output.samples
264
+ ]
265
+ continue
266
+
267
+ for n, nth_output in enumerate(sequence_output.samples):
268
+ if metadata.sampling_params.temperature == 0 or metadata.sampling_params.seed is not None:
269
+ # Ensure exact matches for greedy or random with seed
270
+ assert nth_output.output_token == expected_tokens[i][n]
271
+ else:
272
+ # For non-seeded random check that one of the high-logit tokens were chosen
273
+ assert nth_output.output_token in expected_tokens[i]
274
+
275
+ # Test batch
276
+ test_sampling(model_runner)
277
+
278
+ # Shuffle the batch and resample
279
+ target_index = list(range(batch_size))
280
+ for list_to_shuffle in (target_index, seq_group_metadata_list,
281
+ expected_tokens, prompt_lens):
282
+ random.Random(seed).shuffle(list_to_shuffle)
283
+ target_index = torch.tensor(target_index)
284
+ input_tensor.data = input_tensor.index_select(0, target_index)
285
+ fake_logits.data = fake_logits.index_select(0, target_index)
286
+
287
+ # This time, results of seeded random samples will be compared with the corresponding
288
+ # sample in the pre-shuffled batch
289
+ test_sampling(model_runner)
290
291
del model_runner
292
tests/samplers/test_seeded_generate.py
new
+82
@@ -0,0 +1,82 @@
1
+"""Verify that seeded random sampling is deterministic.
2
+
3
+Run `pytest tests/samplers/test_seeded_generate.py --forked`.
4
+"""
5
+import copy
6
+import random
7
+from itertools import combinations
8
+
9
+import pytest
10
+
11
+from vllm.model_executor.utils import set_random_seed
12
+from vllm import SamplingParams
13
+
14
+MODEL = "facebook/opt-125m"
15
+RANDOM_SEEDS = list(range(5))
16
+
17
+
18
+@pytest.fixture
19
+def vllm_model(vllm_runner):
20
+ vllm_model = vllm_runner(MODEL, dtype="half")
21
+ yield vllm_model
22
+ del vllm_model
23
+
24
+
25
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
26
+def test_random_sample_with_seed(
27
+ vllm_model,
28
+ example_prompts,
29
+ seed: int,
30
+) -> None:
31
+ set_random_seed(seed)
32
+
33
+ sampling_params = SamplingParams(
34
+ # Parameters to ensure sufficient randomness
35
+ temperature=2.0,
36
+ top_p=min(random.random() + 0.3, 1),
37
+ top_k=random.randint(5, 20),
38
+ n=random.randint(1, 10),
39
+ presence_penalty=random.randint(0, 1),
40
+ max_tokens=8,
41
+ ignore_eos=True,
42
+ )
43
+
44
+ sampling_params_seed_1 = copy.deepcopy(sampling_params)
45
+ sampling_params_seed_1.seed = 100
46
+ sampling_params_seed_2 = copy.deepcopy(sampling_params)
47
+ sampling_params_seed_2.seed = 200
48
+
49
+ llm = vllm_model.model
50
+
51
+ for prompt in example_prompts:
52
+ for params in (
53
+ sampling_params,
54
+ sampling_params_seed_1,
55
+ sampling_params_seed_2,
56
+ sampling_params,
57
+ sampling_params_seed_1,
58
+ sampling_params_seed_2,
59
+ ):
60
+ llm._add_request(
61
+ prompt=prompt,
62
+ prompt_token_ids=None,
63
+ sampling_params=params,
64
+ )
65
+
66
+ results = llm._run_engine(use_tqdm=False)
67
+ all_outputs = [[out.token_ids for out in output.outputs]
68
+ for output in results]
69
+
70
+ for i in range(0, len(example_prompts), 6):
71
+ outputs = all_outputs[i:i + 6]
72
+
73
+ # verify all non-seeded requests differ
74
+ for output_a, output_b in combinations(
75
+ (outputs[0], outputs[1], outputs[2], outputs[3]),
76
+ 2,
77
+ ):
78
+ assert output_a != output_b
79
+
80
+ # verify requests with the same seed match
81
+ assert outputs[1] == outputs[4]
82
+ assert outputs[2] == outputs[5]
vllm/core/scheduler.py
+1
@@ -387,6 +387,7 @@ class Scheduler:
387
block_tables=block_tables,
388
lora_request=seq_group.lora_request,
389
prefix=seq_group.prefix,
390
+ state=seq_group.state,
391
)
392
seq_group_metadata_list.append(seq_group_metadata)
393
return seq_group_metadata_list, scheduler_outputs
vllm/engine/arg_utils.py
-1
@@ -173,7 +173,6 @@ class EngineArgs:
173
default=EngineArgs.block_size,
174
choices=[8, 16, 32],
175
help='token block size')
176
- # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
176
parser.add_argument('--seed',
177
type=int,
178
default=EngineArgs.seed,
vllm/entrypoints/openai/protocol.py
+4
@@ -60,6 +60,7 @@ class ChatCompletionRequest(BaseModel):
60
top_p: Optional[float] = 1.0
61
n: Optional[int] = 1
62
max_tokens: Optional[int] = None
63
+ seed: Optional[int] = None
64
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
65
stream: Optional[bool] = False
66
presence_penalty: Optional[float] = 0.0
@@ -90,6 +91,7 @@ class ChatCompletionRequest(BaseModel):
91
temperature=self.temperature,
92
top_p=self.top_p,
93
min_p=self.min_p,
94
+ seed=self.seed,
95
stop=self.stop,
96
stop_token_ids=self.stop_token_ids,
97
max_tokens=self.max_tokens,
@@ -117,6 +119,7 @@ class CompletionRequest(BaseModel):
119
logprobs: Optional[int] = None
120
echo: Optional[bool] = False
121
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
122
+ seed: Optional[int] = None
123
presence_penalty: Optional[float] = 0.0
124
frequency_penalty: Optional[float] = 0.0
125
best_of: Optional[int] = None
@@ -147,6 +150,7 @@ class CompletionRequest(BaseModel):
150
top_p=self.top_p,
151
top_k=self.top_k,
152
min_p=self.min_p,
153
+ seed=self.seed,
154
stop=self.stop,
155
stop_token_ids=self.stop_token_ids,
156
ignore_eos=self.ignore_eos,
vllm/model_executor/layers/sampler.py
+22
-7
@@ -342,7 +342,9 @@ def _beam_search_sample(
342
def _multinomial(
343
probs: torch.Tensor,
344
num_samples: int,
345
-):
345
+ seq_groups: Optional[List[Tuple[List[int], SamplingParams]]] = None,
346
+ generators: Optional[List[torch.Generator]] = None,
347
+) -> torch.Tensor:
348
if num_samples > 1:
349
# This is equivalent to torch.repeat_interleaved (which also
350
# forces a GPU<->CPU sync).
@@ -352,7 +354,15 @@ def _multinomial(
354
probs = probs[:, None, :].expand(probs.shape[0], num_samples,
355
probs.shape[1]).contiguous().view(
356
-1, probs.shape[1])
355
- q = torch.empty_like(probs).exponential_(1)
357
+ q = torch.empty_like(probs)
358
+ if seq_groups is None:
359
+ q.exponential_()
360
+ else:
361
+ sample_idx = 0
362
+ for (seq_ids, _), generator in zip(seq_groups, generators):
363
+ next_sample_idx = sample_idx + len(seq_ids) * num_samples
364
+ q[sample_idx:next_sample_idx].exponential_(generator=generator)
365
+ sample_idx = next_sample_idx
366
return probs.div_(q).argmax(dim=1).view(-1, num_samples)
367
368
@@ -370,6 +380,7 @@ def _sample(
380
381
sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
382
sample_metadata = {}
383
+ multinomial_samples = {}
384
385
# Counterintiutively, having two loops here is actually faster.
386
# The first loop can run without waiting on GPU<->CPU sync.
@@ -385,14 +396,18 @@ def _sample(
396
is_prompts, sample_indices)
397
if sampling_type == SamplingType.GREEDY:
398
greedy_samples = torch.argmax(logprobs[sample_indices], dim=-1)
388
- elif sampling_type == SamplingType.RANDOM:
399
+ elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
400
max_best_of = 1
401
for seq_group, is_prompt in zip(seq_groups, is_prompts):
402
if is_prompt:
403
_, sampling_params = seq_group
404
max_best_of = max(max_best_of, sampling_params.best_of)
394
- multinomial_samples = _multinomial(probs[sample_indices],
395
- max_best_of)
405
+ seeded_args = {} if sampling_type == SamplingType.RANDOM else {
406
+ "seq_groups": seq_groups,
407
+ "generators": sampling_metadata.generators,
408
+ }
409
+ multinomial_samples[sampling_type] = _multinomial(
410
+ probs[sample_indices], max_best_of, **seeded_args)
411
elif sampling_type == SamplingType.BEAM:
412
beam_search_logprobs = logprobs[sample_indices]
413
else:
@@ -407,9 +422,9 @@ def _sample(
422
sampling_type]
423
if sampling_type == SamplingType.GREEDY:
424
sample_results = _greedy_sample(seq_groups, greedy_samples)
410
- elif sampling_type == SamplingType.RANDOM:
425
+ elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
426
sample_results = _random_sample(seq_groups, is_prompts,
412
- multinomial_samples)
427
+ multinomial_samples[sampling_type])
428
elif sampling_type == SamplingType.BEAM:
429
sample_results = _beam_search_sample(seq_groups, is_prompts,
430
sampling_metadata.seq_data,
vllm/model_executor/sampling_metadata.py
+3
@@ -19,6 +19,7 @@ class SamplingMetadata:
19
prompt_lens: Lengths of prompts.
20
selected_token_indices: Token indices selected for sampling.
21
categorized_sample_indices: SamplingType -> token indices to sample.
22
+ generators: List of torch.Generators to use for seeded sampling
23
perform_sampling: Whether to perform sampling. This option is used to
24
make the sampling only happens in the driver worker, and disable
25
sampling in other worker processes.
@@ -31,6 +32,7 @@ class SamplingMetadata:
32
prompt_lens: Optional[List[int]],
33
selected_token_indices: torch.Tensor,
34
categorized_sample_indices: Optional[Dict[SamplingType, torch.Tensor]],
35
+ generators: Optional[List[torch.Generator]] = None,
36
perform_sampling: bool = True,
37
) -> None:
38
self.seq_groups = seq_groups
@@ -38,6 +40,7 @@ class SamplingMetadata:
40
self.prompt_lens = prompt_lens
41
self.selected_token_indices = selected_token_indices
42
self.categorized_sample_indices = categorized_sample_indices
43
+ self.generators = generators
44
self.perform_sampling = perform_sampling
45
46
self.num_prompts = len(prompt_lens) if prompt_lens is not None else 0
vllm/sampling_params.py
+8
-1
@@ -11,7 +11,8 @@ _SAMPLING_EPS = 1e-5
11
class SamplingType(IntEnum):
12
GREEDY = 0
13
RANDOM = 1
14
- BEAM = 2
14
+ RANDOM_SEED = 2
15
+ BEAM = 3
16
17
18
LogitsProcessor = Callable[[List[int], torch.Tensor], torch.Tensor]
@@ -56,6 +57,7 @@ class SamplingParams:
57
min_p: Float that represents the minimum probability for a token to be
58
considered, relative to the probability of the most likely token.
59
Must be in [0, 1]. Set to 0 to disable this.
60
+ seed: Random seed to use for the generation.
61
use_beam_search: Whether to use beam search instead of sampling.
62
length_penalty: Float that penalizes sequences based on their length.
63
Used in beam search.
@@ -101,6 +103,7 @@ class SamplingParams:
103
top_p: float = 1.0,
104
top_k: int = -1,
105
min_p: float = 0.0,
106
+ seed: Optional[int] = None,
107
use_beam_search: bool = False,
108
length_penalty: float = 1.0,
109
early_stopping: Union[bool, str] = False,
@@ -124,6 +127,7 @@ class SamplingParams:
127
self.top_p = top_p
128
self.top_k = top_k
129
self.min_p = min_p
130
+ self.seed = seed
131
self.use_beam_search = use_beam_search
132
self.length_penalty = length_penalty
133
self.early_stopping = early_stopping
@@ -229,6 +233,8 @@ class SamplingParams:
233
return SamplingType.BEAM
234
if self.temperature < _SAMPLING_EPS:
235
return SamplingType.GREEDY
236
+ if self.seed is not None:
237
+ return SamplingType.RANDOM_SEED
238
return SamplingType.RANDOM
239
240
def __repr__(self) -> str:
@@ -242,6 +248,7 @@ class SamplingParams:
248
f"top_p={self.top_p}, "
249
f"top_k={self.top_k}, "
250
f"min_p={self.min_p}, "
251
+ f"seed={self.seed}, "
252
f"use_beam_search={self.use_beam_search}, "
253
f"length_penalty={self.length_penalty}, "
254
f"early_stopping={self.early_stopping}, "
vllm/sequence.py
+12
@@ -248,6 +248,14 @@ class Sequence:
248
f"num_blocks={len(self.logical_token_blocks)})")
249
250
251
+@dataclass
252
+class SequenceGroupState:
253
+ """Mutable state tied to a specific sequence group"""
254
+
255
+ # torch.Generator used in seeded sampling
256
+ generator: Optional = None
257
+
258
+
259
class SequenceGroup:
260
"""A group of sequences that are generated from the same prompt.
261
@@ -280,6 +288,7 @@ class SequenceGroup:
288
self.lora_request = lora_request
289
self.prefix: Optional[Prefix] = prefix
290
self.prompt_logprobs: Optional[PromptLogprobs] = None
291
+ self.state = SequenceGroupState()
292
293
@property
294
def prompt(self) -> str:
@@ -397,6 +406,7 @@ class SequenceGroupMetadata:
406
sampling_params: The sampling parameters used to generate the outputs.
407
block_tables: The block tables. (Seq id -> list of physical block
408
numbers)
409
+ state: Internal state tied to this sequence group.
410
lora_request: LoRA request.
411
prefix: The prefix of the prompt of the sequence group.
412
"""
@@ -410,6 +420,7 @@ class SequenceGroupMetadata:
420
block_tables: Dict[int, List[int]],
421
lora_request: Optional[LoRARequest] = None,
422
prefix: Optional[Prefix] = None,
423
+ state: Optional[SequenceGroupState] = None,
424
) -> None:
425
self.request_id = request_id
426
self.is_prompt = is_prompt
@@ -418,6 +429,7 @@ class SequenceGroupMetadata:
429
self.block_tables = block_tables
430
self.lora_request = lora_request
431
self.prefix = prefix
432
+ self.state = SequenceGroupState() if state is None else state
433
434
@property
435
def lora_int_id(self) -> int:
vllm/worker/model_runner.py
+10
@@ -389,6 +389,7 @@ class ModelRunner:
389
) -> SamplingMetadata:
390
seq_groups: List[Tuple[List[int], SamplingParams]] = []
391
selected_token_indices: List[int] = []
392
+ generators: List[torch.Generator] = []
393
selected_token_start_idx = 0
394
categorized_sample_indices = {t: [] for t in SamplingType}
395
categorized_sample_indices_start_idx = 0
@@ -419,6 +420,10 @@ class ModelRunner:
420
selected_token_indices.append(selected_token_start_idx +
421
subquery_len - 1)
422
selected_token_start_idx += max_subquery_len
423
+
424
+ if sampling_params.seed is not None:
425
+ seq_group_metadata.state.generator = torch.Generator(
426
+ device="cuda").manual_seed(sampling_params.seed)
427
else:
428
num_seqs = len(seq_ids)
429
selected_token_indices.extend(
@@ -432,6 +437,9 @@ class ModelRunner:
437
categorized_sample_indices_start_idx + num_seqs))
438
categorized_sample_indices_start_idx += num_seqs
439
440
+ if sampling_params.seed is not None:
441
+ generators.append(seq_group_metadata.state.generator)
442
+
443
selected_token_indices = _async_h2d(selected_token_indices,
444
dtype=torch.long,
445
target_device=self.device,
@@ -454,6 +462,7 @@ class ModelRunner:
462
prompt_lens=prompt_lens,
463
selected_token_indices=selected_token_indices,
464
categorized_sample_indices=categorized_sample_indices,
465
+ generators=generators,
466
)
467
return sampling_metadata
468
@@ -536,6 +545,7 @@ class ModelRunner:
545
prompt_lens=None,
546
selected_token_indices=metadata_dict["selected_token_indices"],
547
categorized_sample_indices=None,
548
+ generators=None,
549
perform_sampling=False,
550
)
551