A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Remove unnecessary ModelRunner imports (#4703)
Woosuk Kwon committed
May 9, 2024 at 00:17 UTC
190bc838e17196733526896bf2861f8d05bd3f43
2 files changed
+31
-73
tests/samplers/test_sampler.py
+24
-57
@@ -11,8 +11,7 @@ from vllm.model_executor.layers.sampler import Sampler
11
from vllm.model_executor.sampling_metadata import SamplingMetadata
12
from vllm.model_executor.utils import set_random_seed
13
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
14
-from vllm.utils import Counter
15
-from vllm.worker.model_runner import ModelRunner
14
+from vllm.utils import Counter, is_pin_memory_available
15
16
17
class MockLogitsSampler(Sampler):
@@ -26,20 +25,14 @@ class MockLogitsSampler(Sampler):
25
26
27
def _prepare_test(
29
- batch_size: int
30
-) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, ModelRunner]:
28
+ batch_size: int
29
+) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler]:
30
input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
31
fake_logits = torch.full((batch_size, VOCAB_SIZE),
32
1e-2,
33
dtype=input_tensor.dtype)
34
sampler = MockLogitsSampler(fake_logits)
36
- model_runner = ModelRunner(model_config=None,
37
- parallel_config=None,
38
- scheduler_config=None,
39
- device_config=None,
40
- load_config=None,
41
- lora_config=None)
42
- return input_tensor, fake_logits, sampler, model_runner
35
+ return input_tensor, fake_logits, sampler
36
37
38
VOCAB_SIZE = 32000
@@ -53,7 +46,6 @@ def _do_sample(
46
batch_size: int,
47
input_tensor: torch.Tensor,
48
sampler: MockLogitsSampler,
56
- model_runner: ModelRunner,
49
sampling_params: SamplingParams,
50
device: str,
51
):
@@ -75,7 +67,7 @@ def _do_sample(
67
seq_lens,
68
query_lens=seq_lens,
69
device=device,
78
- pin_memory=model_runner.pin_memory)
70
+ pin_memory=is_pin_memory_available())
71
return sampler(logits=input_tensor, sampling_metadata=sampling_metadata)
72
73
@@ -85,19 +77,16 @@ def test_sampler_all_greedy(seed: int, device: str):
77
set_random_seed(seed)
78
torch.set_default_device(device)
79
batch_size = random.randint(1, 256)
88
- input_tensor, fake_logits, sampler, model_runner = _prepare_test(
89
- batch_size)
80
+ input_tensor, fake_logits, sampler = _prepare_test(batch_size)
81
82
sampling_params = SamplingParams(temperature=0)
92
- sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
83
+ sampler_output = _do_sample(batch_size, fake_logits, sampler,
84
sampling_params, device)
85
expected = torch.argmax(fake_logits, dim=-1)
86
for i, sequence_output in enumerate(sampler_output):
87
for nth_output in sequence_output.samples:
88
assert nth_output.output_token == expected[i].item()
89
99
- del model_runner
100
-
90
91
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
92
@pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -105,8 +94,7 @@ def test_sampler_all_random(seed: int, device: str):
94
set_random_seed(seed)
95
torch.set_default_device(device)
96
batch_size = random.randint(1, 256)
108
- input_tensor, fake_logits, sampler, model_runner = _prepare_test(
109
- batch_size)
97
+ _, fake_logits, sampler = _prepare_test(batch_size)
98
99
for i in range(batch_size):
100
fake_logits[i, i] = 1e2
@@ -115,15 +103,13 @@ def test_sampler_all_random(seed: int, device: str):
103
temperature=1.0,
104
n=random.randint(1, 10),
105
)
118
- sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
106
+ sampler_output = _do_sample(batch_size, fake_logits, sampler,
107
sampling_params, device)
108
109
for i, sequence_output in enumerate(sampler_output):
110
for nth_output in sequence_output.samples:
111
assert nth_output.output_token == i
112
125
- del model_runner
126
-
113
114
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
115
@pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -131,7 +117,7 @@ def test_sampler_all_random_seed(seed: int, device: str):
117
set_random_seed(seed)
118
torch.set_default_device(device)
119
batch_size = random.randint(1, 256)
134
- _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
120
+ _, fake_logits, sampler = _prepare_test(batch_size)
121
122
for i in range(batch_size):
123
fake_logits[i, i] = 1e2
@@ -141,15 +127,13 @@ def test_sampler_all_random_seed(seed: int, device: str):
127
n=random.randint(1, 10),
128
seed=random.randint(0, 10000),
129
)
144
- sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
130
+ sampler_output = _do_sample(batch_size, fake_logits, sampler,
131
sampling_params, device)
132
133
for i, sequence_output in enumerate(sampler_output):
134
for nth_output in sequence_output.samples:
135
assert nth_output.output_token == i
136
151
- del model_runner
152
-
137
138
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
139
@pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -157,7 +141,7 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
141
set_random_seed(seed)
142
torch.set_default_device(device)
143
batch_size = random.randint(1, 256)
160
- _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
144
+ _, fake_logits, sampler = _prepare_test(batch_size)
145
146
sampling_params = SamplingParams(
147
temperature=1.0,
@@ -165,15 +149,13 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
149
seed=random.randint(0, 10000),
150
)
151
first_sampler_output = _do_sample(batch_size, fake_logits, sampler,
168
- model_runner, sampling_params, device)
152
+ sampling_params, device)
153
154
second_sampler_output = _do_sample(batch_size, fake_logits, sampler,
171
- model_runner, sampling_params, device)
155
+ sampling_params, device)
156
157
assert first_sampler_output == second_sampler_output
158
175
- del model_runner
176
-
159
160
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
161
@pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -181,20 +163,18 @@ def test_sampler_all_beam(seed: int, device: str):
163
set_random_seed(seed)
164
torch.set_default_device(device)
165
batch_size = random.randint(1, 256)
184
- _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
166
+ _, fake_logits, sampler = _prepare_test(batch_size)
167
168
sampling_params = SamplingParams(
169
temperature=0,
170
best_of=2,
171
use_beam_search=True,
172
)
191
- _do_sample(batch_size, fake_logits, sampler, model_runner, sampling_params,
192
- device)
173
+ _do_sample(batch_size, fake_logits, sampler, sampling_params, device)
174
# no assertion here as I am not sure how to determine whether
175
# the outputs are expected - in other words, this just tests
176
# whether there are no exceptions in the sampler
177
# when handling an all-beam search case.
197
- del model_runner
178
179
180
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
@@ -448,13 +428,13 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
428
("Invalid test case, expected_penalization does not match computed"
429
"batch size")
430
451
- _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
431
+ _, fake_logits, sampler = _prepare_test(batch_size)
432
sampling_metadata = SamplingMetadata.prepare(
433
seq_group_metadata_list,
434
seq_lens=seq_lens if seq_lens else None,
435
query_lens=seq_lens if seq_lens else None,
436
device=device,
457
- pin_memory=model_runner.pin_memory)
437
+ pin_memory=is_pin_memory_available())
438
# the logits tensor is modified in-place by the sampler
439
_ = sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
440
@@ -480,8 +460,6 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
460
fake_logits[logits_idx, :] ==
461
-float('inf')) == 0, "No tokens should have been penalized"
462
483
- del model_runner
484
-
463
for test_case in test_cases:
464
run_test_case(**test_case)
465
@@ -492,8 +470,7 @@ def test_sampler_mixed(seed: int, device: str):
470
set_random_seed(seed)
471
torch.set_default_device(device)
472
batch_size = random.randint(1, 256)
495
- input_tensor, fake_logits, sampler, model_runner = _prepare_test(
496
- batch_size)
473
+ input_tensor, fake_logits, sampler = _prepare_test(batch_size)
474
475
seq_group_metadata_list = []
476
expected_tokens: List[Optional[List[int]]] = []
@@ -534,13 +511,13 @@ def test_sampler_mixed(seed: int, device: str):
511
))
512
seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
513
537
- def test_sampling(model_runner: ModelRunner):
514
+ def test_sampling():
515
sampling_metadata = SamplingMetadata.prepare(
516
seq_group_metadata_list,
517
seq_lens,
518
query_lens=seq_lens,
519
device=device,
543
- pin_memory=model_runner.pin_memory)
520
+ pin_memory=is_pin_memory_available())
521
sampler_output = sampler(logits=fake_logits,
522
sampling_metadata=sampling_metadata)
523
@@ -570,7 +547,7 @@ def test_sampler_mixed(seed: int, device: str):
547
assert nth_output.output_token in expected_tokens[i]
548
549
# Test batch
573
- test_sampling(model_runner)
550
+ test_sampling()
551
552
# Shuffle the batch and resample
553
target_index = list(range(batch_size))
@@ -583,9 +560,7 @@ def test_sampler_mixed(seed: int, device: str):
560
561
# This time, results of seeded random samples will be compared with
562
# the corresponding sample in the pre-shuffled batch
586
- test_sampling(model_runner)
587
-
588
- del model_runner
563
+ test_sampling()
564
565
566
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
@@ -605,12 +580,6 @@ def test_sampler_top_k_top_p(seed: int, device: str):
580
device=input_tensor.device,
581
dtype=input_tensor.dtype)
582
sampler = MockLogitsSampler(fake_logits)
608
- model_runner = ModelRunner(model_config=None,
609
- parallel_config=None,
610
- scheduler_config=None,
611
- device_config=None,
612
- load_config=None,
613
- lora_config=None)
583
584
generation_model = GenerationMixin()
585
generation_config = GenerationConfig(top_k=top_k,
@@ -641,7 +610,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
610
seq_lens,
611
query_lens=seq_lens,
612
device=device,
644
- pin_memory=model_runner.pin_memory)
613
+ pin_memory=is_pin_memory_available())
614
615
sample_probs = None
616
@@ -657,5 +626,3 @@ def test_sampler_top_k_top_p(seed: int, device: str):
626
hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
627
assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
628
assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
660
-
661
- del model_runner
tests/test_logits_processor.py
+7
-16
@@ -9,7 +9,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
from vllm.model_executor.sampling_metadata import SamplingMetadata
10
from vllm.model_executor.utils import set_random_seed
11
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12
-from vllm.worker.model_runner import ModelRunner
12
+from vllm.utils import is_pin_memory_available
13
14
15
class MockLogitsProcessor(LogitsProcessor):
@@ -30,21 +30,15 @@ class MockLogitsProcessor(LogitsProcessor):
30
31
32
def _prepare_test(
33
- batch_size: int
34
-) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor, ModelRunner]:
33
+ batch_size: int
34
+) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor]:
35
vocab_size = 32000
36
input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
37
fake_logits = torch.full((batch_size, vocab_size),
38
1e-2,
39
dtype=input_tensor.dtype)
40
logits_processor = MockLogitsProcessor(32000, 0.5, fake_logits)
41
- model_runner = ModelRunner(model_config=None,
42
- parallel_config=None,
43
- scheduler_config=None,
44
- device_config=None,
45
- load_config=None,
46
- lora_config=None)
47
- return input_tensor, fake_logits, logits_processor, model_runner
41
+ return input_tensor, fake_logits, logits_processor
42
43
44
RANDOM_SEEDS = list(range(128))
@@ -59,8 +53,7 @@ def test_logits_processors(seed: int, device: str):
53
set_random_seed(seed)
54
torch.set_default_device(device)
55
batch_size = random.randint(1, 256)
62
- input_tensor, fake_logits, logits_processor, model_runner = _prepare_test(
63
- batch_size)
56
+ input_tensor, fake_logits, logits_processor = _prepare_test(batch_size)
57
58
# This sample logits processor gives infinite score to the i-th token,
59
# where i is the length of the input sequence.
@@ -87,8 +80,8 @@ def test_logits_processors(seed: int, device: str):
80
seq_group_metadata_list,
81
seq_lens,
82
query_lens=seq_lens,
90
- device=model_runner.device,
91
- pin_memory=model_runner.pin_memory)
83
+ device=device,
84
+ pin_memory=is_pin_memory_available())
85
logits_processor_output = logits_processor(
86
embedding=None,
87
hidden_states=input_tensor,
@@ -99,5 +92,3 @@ def test_logits_processors(seed: int, device: str):
92
fake_logits *= logits_processor.scale
93
assert torch.allclose(logits_processor_output[:, 1], fake_logits[:, 1],
94
1e-4)
102
-
103
- del model_runner