A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix broken sampler tests (#1896)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Woosuk Kwon committed
Dec 2, 2023 at 16:06 UTC
5f09cbdb63a5821d365b72318bcd92b8ffd03fb4
2 files changed
+41
-21
tests/samplers/test_sampler.py
+37
-20
@@ -8,7 +8,7 @@ import torch
8
from vllm.model_executor.layers.sampler import Sampler
9
from vllm.model_executor.utils import set_random_seed
10
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11
-from vllm.worker.worker import Worker
11
+from vllm.worker.model_runner import ModelRunner
12
13
14
class MockLogitsSampler(Sampler):
@@ -27,7 +27,7 @@ class MockLogitsSampler(Sampler):
27
28
def _prepare_test(
29
batch_size: int
30
-) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, Worker]:
30
+) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, ModelRunner]:
31
vocab_size = 32000
32
input_tensor = torch.rand((batch_size, 1024),
33
device="cuda",
@@ -37,9 +37,8 @@ def _prepare_test(
37
device=input_tensor.device,
38
dtype=input_tensor.dtype)
39
sampler = MockLogitsSampler(32000, fake_logits)
40
- worker = Worker(None, None, None)
41
- worker.block_size = 16
42
- return input_tensor, fake_logits, sampler, worker
40
+ model_runner = ModelRunner(None, None, None)
41
+ return input_tensor, fake_logits, sampler, model_runner
42
43
44
RANDOM_SEEDS = list(range(128))
@@ -49,9 +48,11 @@ RANDOM_SEEDS = list(range(128))
48
def test_sampler_all_greedy(seed: int):
49
set_random_seed(seed)
50
batch_size = random.randint(1, 256)
52
- input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
51
+ input_tensor, fake_logits, sampler, model_runner = _prepare_test(
52
+ batch_size)
53
54
seq_group_metadata_list = []
55
+ prompt_lens = []
56
for i in range(batch_size):
57
seq_group_metadata_list.append(
58
SequenceGroupMetadata(
@@ -61,11 +62,13 @@ def test_sampler_all_greedy(seed: int):
62
sampling_params=SamplingParams(temperature=0, ),
63
block_tables={0: [1]},
64
))
65
+ prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
66
65
- _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
67
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
68
+ prompt_lens)
69
sampler_output = sampler(embedding=None,
70
hidden_states=input_tensor,
68
- input_metadata=input_metadata)
71
+ sampling_metadata=sampling_metadata)
72
expected = torch.argmax(fake_logits, dim=-1)
73
for i, sequence_output in enumerate(sampler_output):
74
for nth_output in sequence_output.samples:
@@ -76,12 +79,14 @@ def test_sampler_all_greedy(seed: int):
79
def test_sampler_all_random(seed: int):
80
set_random_seed(seed)
81
batch_size = random.randint(1, 256)
79
- input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
82
+ input_tensor, fake_logits, sampler, model_runner = _prepare_test(
83
+ batch_size)
84
85
for i in range(batch_size):
86
fake_logits[i, i] = 1e2
87
88
seq_group_metadata_list = []
89
+ prompt_lens = []
90
for i in range(batch_size):
91
seq_group_metadata_list.append(
92
SequenceGroupMetadata(
@@ -94,11 +99,13 @@ def test_sampler_all_random(seed: int):
99
),
100
block_tables={0: [1]},
101
))
102
+ prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
103
98
- _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
104
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
105
+ prompt_lens)
106
sampler_output = sampler(embedding=None,
107
hidden_states=input_tensor,
101
- input_metadata=input_metadata)
108
+ sampling_metadata=sampling_metadata)
109
for i, sequence_output in enumerate(sampler_output):
110
for nth_output in sequence_output.samples:
111
assert nth_output.output_token == i
@@ -108,9 +115,10 @@ def test_sampler_all_random(seed: int):
115
def test_sampler_all_beam(seed: int):
116
set_random_seed(seed)
117
batch_size = random.randint(1, 256)
111
- input_tensor, _, sampler, worker = _prepare_test(batch_size)
118
+ input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
119
120
seq_group_metadata_list = []
121
+ prompt_lens = []
122
for i in range(batch_size):
123
seq_group_metadata_list.append(
124
SequenceGroupMetadata(
@@ -124,11 +132,13 @@ def test_sampler_all_beam(seed: int):
132
),
133
block_tables={0: [1]},
134
))
135
+ prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
136
128
- _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
137
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
138
+ prompt_lens)
139
sampler(embedding=None,
140
hidden_states=input_tensor,
131
- input_metadata=input_metadata)
141
+ sampling_metadata=sampling_metadata)
142
# no assertion here as I am not sure how to determine whether
143
# the outputs are expected - in other words, this just tests
144
# whether there are no exceptions in the sampler
@@ -139,10 +149,12 @@ def test_sampler_all_beam(seed: int):
149
def test_sampler_mixed(seed: int):
150
set_random_seed(seed)
151
batch_size = random.randint(1, 256)
142
- input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
152
+ input_tensor, fake_logits, sampler, model_runner = _prepare_test(
153
+ batch_size)
154
155
seq_group_metadata_list = []
156
expected_tokens = []
157
+ prompt_lens = []
158
for i in range(batch_size):
159
n = 1
160
sampling_type = random.randint(0, 2)
@@ -172,11 +184,13 @@ def test_sampler_mixed(seed: int):
184
sampling_params=sampling_params,
185
block_tables={0: [1]},
186
))
187
+ prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
188
176
- _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
189
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
190
+ prompt_lens)
191
sampler_output = sampler(embedding=None,
192
hidden_states=input_tensor,
179
- input_metadata=input_metadata)
193
+ sampling_metadata=sampling_metadata)
194
for i, sequence_output in enumerate(sampler_output):
195
if seq_group_metadata_list[i].sampling_params.use_beam_search:
196
continue
@@ -188,7 +202,7 @@ def test_sampler_mixed(seed: int):
202
def test_sampler_logits_processors(seed: int):
203
set_random_seed(seed)
204
batch_size = random.randint(1, 256)
191
- input_tensor, _, sampler, worker = _prepare_test(batch_size)
205
+ input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
206
207
# This sample logits processor gives infinite score to the i-th token,
208
# where i is the length of the input sequence.
@@ -198,6 +212,7 @@ def test_sampler_logits_processors(seed: int):
212
return logits
213
214
seq_group_metadata_list = []
215
+ prompt_lens = []
216
for i in range(batch_size):
217
seq_group_metadata_list.append(
218
SequenceGroupMetadata(
@@ -208,11 +223,13 @@ def test_sampler_logits_processors(seed: int):
223
logits_processors=[pick_ith]),
224
block_tables={0: [1]},
225
))
226
+ prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
227
212
- _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
228
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
229
+ prompt_lens)
230
sampler_output = sampler(embedding=None,
231
hidden_states=input_tensor,
215
- input_metadata=input_metadata)
232
+ sampling_metadata=sampling_metadata)
233
for _, sequence_output in enumerate(sampler_output):
234
for idx, nth_output in enumerate(sequence_output.samples):
235
assert nth_output.output_token == idx
vllm/worker/model_runner.py
+4
-1
@@ -25,7 +25,10 @@ class ModelRunner:
25
self.parallel_config = parallel_config
26
self.scheduler_config = scheduler_config
27
28
- self.sliding_window = model_config.get_sliding_window()
28
+ # model_config can be None in tests/samplers/test_sampler.py.
29
+ # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
30
+ self.sliding_window = (model_config.get_sliding_window()
31
+ if model_config is not None else None)
32
self.model = None
33
self.block_size = None # Set after initial profiling.
34