@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 190bc838e1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Remove unnecessary ModelRunner imports (#4703)

Woosuk Kwon committed May 9, 2024 at 00:17 UTC 190bc838e17196733526896bf2861f8d05bd3f43
2 files changed +31 -73
tests/samplers/test_sampler.py
+24 -57
@@ -11,8 +11,7 @@ from vllm.model_executor.layers.sampler import Sampler
11 from vllm.model_executor.sampling_metadata import SamplingMetadata
12 from vllm.model_executor.utils import set_random_seed
13 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
14 -from vllm.utils import Counter
15 -from vllm.worker.model_runner import ModelRunner
14 +from vllm.utils import Counter, is_pin_memory_available
15
16
17 class MockLogitsSampler(Sampler):
@@ -26,20 +25,14 @@ class MockLogitsSampler(Sampler):
25
26
27 def _prepare_test(
29 - batch_size: int
30 -) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler, ModelRunner]:
28 + batch_size: int
29 +) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsSampler]:
30 input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
31 fake_logits = torch.full((batch_size, VOCAB_SIZE),
32 1e-2,
33 dtype=input_tensor.dtype)
34 sampler = MockLogitsSampler(fake_logits)
36 - model_runner = ModelRunner(model_config=None,
37 - parallel_config=None,
38 - scheduler_config=None,
39 - device_config=None,
40 - load_config=None,
41 - lora_config=None)
42 - return input_tensor, fake_logits, sampler, model_runner
35 + return input_tensor, fake_logits, sampler
36
37
38 VOCAB_SIZE = 32000
@@ -53,7 +46,6 @@ def _do_sample(
46 batch_size: int,
47 input_tensor: torch.Tensor,
48 sampler: MockLogitsSampler,
56 - model_runner: ModelRunner,
49 sampling_params: SamplingParams,
50 device: str,
51 ):
@@ -75,7 +67,7 @@ def _do_sample(
67 seq_lens,
68 query_lens=seq_lens,
69 device=device,
78 - pin_memory=model_runner.pin_memory)
70 + pin_memory=is_pin_memory_available())
71 return sampler(logits=input_tensor, sampling_metadata=sampling_metadata)
72
73
@@ -85,19 +77,16 @@ def test_sampler_all_greedy(seed: int, device: str):
77 set_random_seed(seed)
78 torch.set_default_device(device)
79 batch_size = random.randint(1, 256)
88 - input_tensor, fake_logits, sampler, model_runner = _prepare_test(
89 - batch_size)
80 + input_tensor, fake_logits, sampler = _prepare_test(batch_size)
81
82 sampling_params = SamplingParams(temperature=0)
92 - sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
83 + sampler_output = _do_sample(batch_size, fake_logits, sampler,
84 sampling_params, device)
85 expected = torch.argmax(fake_logits, dim=-1)
86 for i, sequence_output in enumerate(sampler_output):
87 for nth_output in sequence_output.samples:
88 assert nth_output.output_token == expected[i].item()
89
99 - del model_runner
100 -
90
91 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
92 @pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -105,8 +94,7 @@ def test_sampler_all_random(seed: int, device: str):
94 set_random_seed(seed)
95 torch.set_default_device(device)
96 batch_size = random.randint(1, 256)
108 - input_tensor, fake_logits, sampler, model_runner = _prepare_test(
109 - batch_size)
97 + _, fake_logits, sampler = _prepare_test(batch_size)
98
99 for i in range(batch_size):
100 fake_logits[i, i] = 1e2
@@ -115,15 +103,13 @@ def test_sampler_all_random(seed: int, device: str):
103 temperature=1.0,
104 n=random.randint(1, 10),
105 )
118 - sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
106 + sampler_output = _do_sample(batch_size, fake_logits, sampler,
107 sampling_params, device)
108
109 for i, sequence_output in enumerate(sampler_output):
110 for nth_output in sequence_output.samples:
111 assert nth_output.output_token == i
112
125 - del model_runner
126 -
113
114 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
115 @pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -131,7 +117,7 @@ def test_sampler_all_random_seed(seed: int, device: str):
117 set_random_seed(seed)
118 torch.set_default_device(device)
119 batch_size = random.randint(1, 256)
134 - _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
120 + _, fake_logits, sampler = _prepare_test(batch_size)
121
122 for i in range(batch_size):
123 fake_logits[i, i] = 1e2
@@ -141,15 +127,13 @@ def test_sampler_all_random_seed(seed: int, device: str):
127 n=random.randint(1, 10),
128 seed=random.randint(0, 10000),
129 )
144 - sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
130 + sampler_output = _do_sample(batch_size, fake_logits, sampler,
131 sampling_params, device)
132
133 for i, sequence_output in enumerate(sampler_output):
134 for nth_output in sequence_output.samples:
135 assert nth_output.output_token == i
136
151 - del model_runner
152 -
137
138 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
139 @pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -157,7 +141,7 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
141 set_random_seed(seed)
142 torch.set_default_device(device)
143 batch_size = random.randint(1, 256)
160 - _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
144 + _, fake_logits, sampler = _prepare_test(batch_size)
145
146 sampling_params = SamplingParams(
147 temperature=1.0,
@@ -165,15 +149,13 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
149 seed=random.randint(0, 10000),
150 )
151 first_sampler_output = _do_sample(batch_size, fake_logits, sampler,
168 - model_runner, sampling_params, device)
152 + sampling_params, device)
153
154 second_sampler_output = _do_sample(batch_size, fake_logits, sampler,
171 - model_runner, sampling_params, device)
155 + sampling_params, device)
156
157 assert first_sampler_output == second_sampler_output
158
175 - del model_runner
176 -
159
160 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
161 @pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -181,20 +163,18 @@ def test_sampler_all_beam(seed: int, device: str):
163 set_random_seed(seed)
164 torch.set_default_device(device)
165 batch_size = random.randint(1, 256)
184 - _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
166 + _, fake_logits, sampler = _prepare_test(batch_size)
167
168 sampling_params = SamplingParams(
169 temperature=0,
170 best_of=2,
171 use_beam_search=True,
172 )
191 - _do_sample(batch_size, fake_logits, sampler, model_runner, sampling_params,
192 - device)
173 + _do_sample(batch_size, fake_logits, sampler, sampling_params, device)
174 # no assertion here as I am not sure how to determine whether
175 # the outputs are expected - in other words, this just tests
176 # whether there are no exceptions in the sampler
177 # when handling an all-beam search case.
197 - del model_runner
178
179
180 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
@@ -448,13 +428,13 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
428 ("Invalid test case, expected_penalization does not match computed"
429 "batch size")
430
451 - _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
431 + _, fake_logits, sampler = _prepare_test(batch_size)
432 sampling_metadata = SamplingMetadata.prepare(
433 seq_group_metadata_list,
434 seq_lens=seq_lens if seq_lens else None,
435 query_lens=seq_lens if seq_lens else None,
436 device=device,
457 - pin_memory=model_runner.pin_memory)
437 + pin_memory=is_pin_memory_available())
438 # the logits tensor is modified in-place by the sampler
439 _ = sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
440
@@ -480,8 +460,6 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
460 fake_logits[logits_idx, :] ==
461 -float('inf')) == 0, "No tokens should have been penalized"
462
483 - del model_runner
484 -
463 for test_case in test_cases:
464 run_test_case(**test_case)
465
@@ -492,8 +470,7 @@ def test_sampler_mixed(seed: int, device: str):
470 set_random_seed(seed)
471 torch.set_default_device(device)
472 batch_size = random.randint(1, 256)
495 - input_tensor, fake_logits, sampler, model_runner = _prepare_test(
496 - batch_size)
473 + input_tensor, fake_logits, sampler = _prepare_test(batch_size)
474
475 seq_group_metadata_list = []
476 expected_tokens: List[Optional[List[int]]] = []
@@ -534,13 +511,13 @@ def test_sampler_mixed(seed: int, device: str):
511 ))
512 seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
513
537 - def test_sampling(model_runner: ModelRunner):
514 + def test_sampling():
515 sampling_metadata = SamplingMetadata.prepare(
516 seq_group_metadata_list,
517 seq_lens,
518 query_lens=seq_lens,
519 device=device,
543 - pin_memory=model_runner.pin_memory)
520 + pin_memory=is_pin_memory_available())
521 sampler_output = sampler(logits=fake_logits,
522 sampling_metadata=sampling_metadata)
523
@@ -570,7 +547,7 @@ def test_sampler_mixed(seed: int, device: str):
547 assert nth_output.output_token in expected_tokens[i]
548
549 # Test batch
573 - test_sampling(model_runner)
550 + test_sampling()
551
552 # Shuffle the batch and resample
553 target_index = list(range(batch_size))
@@ -583,9 +560,7 @@ def test_sampler_mixed(seed: int, device: str):
560
561 # This time, results of seeded random samples will be compared with
562 # the corresponding sample in the pre-shuffled batch
586 - test_sampling(model_runner)
587 -
588 - del model_runner
563 + test_sampling()
564
565
566 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
@@ -605,12 +580,6 @@ def test_sampler_top_k_top_p(seed: int, device: str):
580 device=input_tensor.device,
581 dtype=input_tensor.dtype)
582 sampler = MockLogitsSampler(fake_logits)
608 - model_runner = ModelRunner(model_config=None,
609 - parallel_config=None,
610 - scheduler_config=None,
611 - device_config=None,
612 - load_config=None,
613 - lora_config=None)
583
584 generation_model = GenerationMixin()
585 generation_config = GenerationConfig(top_k=top_k,
@@ -641,7 +610,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
610 seq_lens,
611 query_lens=seq_lens,
612 device=device,
644 - pin_memory=model_runner.pin_memory)
613 + pin_memory=is_pin_memory_available())
614
615 sample_probs = None
616
@@ -657,5 +626,3 @@ def test_sampler_top_k_top_p(seed: int, device: str):
626 hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
627 assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
628 assert torch.equal(hf_probs.eq(0), sample_probs.eq(0))
660 -
661 - del model_runner
tests/test_logits_processor.py
+7 -16
@@ -9,7 +9,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 from vllm.model_executor.sampling_metadata import SamplingMetadata
10 from vllm.model_executor.utils import set_random_seed
11 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12 -from vllm.worker.model_runner import ModelRunner
12 +from vllm.utils import is_pin_memory_available
13
14
15 class MockLogitsProcessor(LogitsProcessor):
@@ -30,21 +30,15 @@ class MockLogitsProcessor(LogitsProcessor):
30
31
32 def _prepare_test(
33 - batch_size: int
34 -) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor, ModelRunner]:
33 + batch_size: int
34 +) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor]:
35 vocab_size = 32000
36 input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
37 fake_logits = torch.full((batch_size, vocab_size),
38 1e-2,
39 dtype=input_tensor.dtype)
40 logits_processor = MockLogitsProcessor(32000, 0.5, fake_logits)
41 - model_runner = ModelRunner(model_config=None,
42 - parallel_config=None,
43 - scheduler_config=None,
44 - device_config=None,
45 - load_config=None,
46 - lora_config=None)
47 - return input_tensor, fake_logits, logits_processor, model_runner
41 + return input_tensor, fake_logits, logits_processor
42
43
44 RANDOM_SEEDS = list(range(128))
@@ -59,8 +53,7 @@ def test_logits_processors(seed: int, device: str):
53 set_random_seed(seed)
54 torch.set_default_device(device)
55 batch_size = random.randint(1, 256)
62 - input_tensor, fake_logits, logits_processor, model_runner = _prepare_test(
63 - batch_size)
56 + input_tensor, fake_logits, logits_processor = _prepare_test(batch_size)
57
58 # This sample logits processor gives infinite score to the i-th token,
59 # where i is the length of the input sequence.
@@ -87,8 +80,8 @@ def test_logits_processors(seed: int, device: str):
80 seq_group_metadata_list,
81 seq_lens,
82 query_lens=seq_lens,
90 - device=model_runner.device,
91 - pin_memory=model_runner.pin_memory)
83 + device=device,
84 + pin_memory=is_pin_memory_available())
85 logits_processor_output = logits_processor(
86 embedding=None,
87 hidden_states=input_tensor,
@@ -99,5 +92,3 @@ def test_logits_processors(seed: int, device: str):
92 fake_logits *= logits_processor.scale
93 assert torch.allclose(logits_processor_output[:, 1], fake_logits[:, 1],
94 1e-4)
102 -
103 - del model_runner