@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5f09cbdb63
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix broken sampler tests (#1896)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Woosuk Kwon committed Dec 2, 2023 at 16:06 UTC 5f09cbdb63a5821d365b72318bcd92b8ffd03fb4
2 files changed +41 -21
tests/samplers/test_sampler.py
+37 -20
@@ -8,7 +8,7 @@ import torch
8 from vllm.model_executor.layers.sampler import Sampler
9 from vllm.model_executor.utils import set_random_seed
10 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11 -from vllm.worker.worker import Worker
11 +from vllm.worker.model_runner import ModelRunner
12
13
14 class MockLogitsSampler(Sampler):
@@ -27,7 +27,7 @@ class MockLogitsSampler(Sampler):
27
28 def _prepare_test(
29 batch_size: int
30 -) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, Worker]:
30 +) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, ModelRunner]:
31 vocab_size = 32000
32 input_tensor = torch.rand((batch_size, 1024),
33 device="cuda",
@@ -37,9 +37,8 @@ def _prepare_test(
37 device=input_tensor.device,
38 dtype=input_tensor.dtype)
39 sampler = MockLogitsSampler(32000, fake_logits)
40 - worker = Worker(None, None, None)
41 - worker.block_size = 16
42 - return input_tensor, fake_logits, sampler, worker
40 + model_runner = ModelRunner(None, None, None)
41 + return input_tensor, fake_logits, sampler, model_runner
42
43
44 RANDOM_SEEDS = list(range(128))
@@ -49,9 +48,11 @@ RANDOM_SEEDS = list(range(128))
48 def test_sampler_all_greedy(seed: int):
49 set_random_seed(seed)
50 batch_size = random.randint(1, 256)
52 - input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
51 + input_tensor, fake_logits, sampler, model_runner = _prepare_test(
52 + batch_size)
53
54 seq_group_metadata_list = []
55 + prompt_lens = []
56 for i in range(batch_size):
57 seq_group_metadata_list.append(
58 SequenceGroupMetadata(
@@ -61,11 +62,13 @@ def test_sampler_all_greedy(seed: int):
62 sampling_params=SamplingParams(temperature=0, ),
63 block_tables={0: [1]},
64 ))
65 + prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
66
65 - _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
67 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
68 + prompt_lens)
69 sampler_output = sampler(embedding=None,
70 hidden_states=input_tensor,
68 - input_metadata=input_metadata)
71 + sampling_metadata=sampling_metadata)
72 expected = torch.argmax(fake_logits, dim=-1)
73 for i, sequence_output in enumerate(sampler_output):
74 for nth_output in sequence_output.samples:
@@ -76,12 +79,14 @@ def test_sampler_all_greedy(seed: int):
79 def test_sampler_all_random(seed: int):
80 set_random_seed(seed)
81 batch_size = random.randint(1, 256)
79 - input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
82 + input_tensor, fake_logits, sampler, model_runner = _prepare_test(
83 + batch_size)
84
85 for i in range(batch_size):
86 fake_logits[i, i] = 1e2
87
88 seq_group_metadata_list = []
89 + prompt_lens = []
90 for i in range(batch_size):
91 seq_group_metadata_list.append(
92 SequenceGroupMetadata(
@@ -94,11 +99,13 @@ def test_sampler_all_random(seed: int):
99 ),
100 block_tables={0: [1]},
101 ))
102 + prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
103
98 - _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
104 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
105 + prompt_lens)
106 sampler_output = sampler(embedding=None,
107 hidden_states=input_tensor,
101 - input_metadata=input_metadata)
108 + sampling_metadata=sampling_metadata)
109 for i, sequence_output in enumerate(sampler_output):
110 for nth_output in sequence_output.samples:
111 assert nth_output.output_token == i
@@ -108,9 +115,10 @@ def test_sampler_all_random(seed: int):
115 def test_sampler_all_beam(seed: int):
116 set_random_seed(seed)
117 batch_size = random.randint(1, 256)
111 - input_tensor, _, sampler, worker = _prepare_test(batch_size)
118 + input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
119
120 seq_group_metadata_list = []
121 + prompt_lens = []
122 for i in range(batch_size):
123 seq_group_metadata_list.append(
124 SequenceGroupMetadata(
@@ -124,11 +132,13 @@ def test_sampler_all_beam(seed: int):
132 ),
133 block_tables={0: [1]},
134 ))
135 + prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
136
128 - _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
137 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
138 + prompt_lens)
139 sampler(embedding=None,
140 hidden_states=input_tensor,
131 - input_metadata=input_metadata)
141 + sampling_metadata=sampling_metadata)
142 # no assertion here as I am not sure how to determine whether
143 # the outputs are expected - in other words, this just tests
144 # whether there are no exceptions in the sampler
@@ -139,10 +149,12 @@ def test_sampler_all_beam(seed: int):
149 def test_sampler_mixed(seed: int):
150 set_random_seed(seed)
151 batch_size = random.randint(1, 256)
142 - input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
152 + input_tensor, fake_logits, sampler, model_runner = _prepare_test(
153 + batch_size)
154
155 seq_group_metadata_list = []
156 expected_tokens = []
157 + prompt_lens = []
158 for i in range(batch_size):
159 n = 1
160 sampling_type = random.randint(0, 2)
@@ -172,11 +184,13 @@ def test_sampler_mixed(seed: int):
184 sampling_params=sampling_params,
185 block_tables={0: [1]},
186 ))
187 + prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
188
176 - _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
189 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
190 + prompt_lens)
191 sampler_output = sampler(embedding=None,
192 hidden_states=input_tensor,
179 - input_metadata=input_metadata)
193 + sampling_metadata=sampling_metadata)
194 for i, sequence_output in enumerate(sampler_output):
195 if seq_group_metadata_list[i].sampling_params.use_beam_search:
196 continue
@@ -188,7 +202,7 @@ def test_sampler_mixed(seed: int):
202 def test_sampler_logits_processors(seed: int):
203 set_random_seed(seed)
204 batch_size = random.randint(1, 256)
191 - input_tensor, _, sampler, worker = _prepare_test(batch_size)
205 + input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
206
207 # This sample logits processor gives infinite score to the i-th token,
208 # where i is the length of the input sequence.
@@ -198,6 +212,7 @@ def test_sampler_logits_processors(seed: int):
212 return logits
213
214 seq_group_metadata_list = []
215 + prompt_lens = []
216 for i in range(batch_size):
217 seq_group_metadata_list.append(
218 SequenceGroupMetadata(
@@ -208,11 +223,13 @@ def test_sampler_logits_processors(seed: int):
223 logits_processors=[pick_ith]),
224 block_tables={0: [1]},
225 ))
226 + prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
227
212 - _, _, input_metadata = worker._prepare_inputs(seq_group_metadata_list)
228 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
229 + prompt_lens)
230 sampler_output = sampler(embedding=None,
231 hidden_states=input_tensor,
215 - input_metadata=input_metadata)
232 + sampling_metadata=sampling_metadata)
233 for _, sequence_output in enumerate(sampler_output):
234 for idx, nth_output in enumerate(sequence_output.samples):
235 assert nth_output.output_token == idx
vllm/worker/model_runner.py
+4 -1
@@ -25,7 +25,10 @@ class ModelRunner:
25 self.parallel_config = parallel_config
26 self.scheduler_config = scheduler_config
27
28 - self.sliding_window = model_config.get_sliding_window()
28 + # model_config can be None in tests/samplers/test_sampler.py.
29 + # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
30 + self.sliding_window = (model_config.get_sliding_window()
31 + if model_config is not None else None)
32 self.model = None
33 self.block_size = None # Set after initial profiling.
34