A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix broken worker test (#1900)
Woosuk Kwon committed
Dec 2, 2023 at 22:17 UTC
cd3aa153a4e4974802385f209ad343149af02c07
1 file changed
+14
-9
tests/worker/test_model_runner.py
renamed
+14
-9
@@ -2,18 +2,19 @@ import random
2
import torch
3
4
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
5
-from vllm.worker.worker import Worker
5
+from vllm.worker.model_runner import ModelRunner
6
7
8
-def test_worker_prepare_inputs_for_prompt():
9
- worker = Worker(None, None, None)
10
- worker.block_size = 16
8
+def test_prepare_prompt():
9
+ model_runner = ModelRunner(None, None, None)
10
+ model_runner.set_block_size(16)
11
+
12
batch_size = random.randint(1, 256)
13
prompt_lens = []
14
seq_group_metadata_list = []
15
for i in range(batch_size):
16
# make sure all tokens fit into one block
16
- prompt_len = i % (worker.block_size - 1) + 1
17
+ prompt_len = i % (model_runner.block_size - 1) + 1
18
prompt_lens.append(prompt_len)
19
seq_data = list(range(prompt_len))
20
seq_group_metadata_list.append(
@@ -24,6 +25,7 @@ def test_worker_prepare_inputs_for_prompt():
25
sampling_params=SamplingParams(temperature=0),
26
block_tables={0: [1]},
27
))
28
+
29
expected_selected_token_indices = []
30
selected_token_start_idx = 0
31
max_seq_len = max(prompt_lens)
@@ -31,12 +33,15 @@ def test_worker_prepare_inputs_for_prompt():
33
expected_selected_token_indices.append(selected_token_start_idx +
34
prompt_len - 1)
35
selected_token_start_idx += max_seq_len
34
- input_tokens, input_positions, input_metadata = worker._prepare_inputs(
36
+ input_tokens, input_positions, _ = model_runner._prepare_prompt(
37
seq_group_metadata_list)
36
- assert input_tokens.shape == input_positions.shape == (batch_size,
37
- max_seq_len)
38
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
39
+ prompt_lens)
40
+ assert input_tokens.shape == (batch_size, max_seq_len)
41
+ assert input_positions.shape == (batch_size, max_seq_len)
42
torch.testing.assert_close(input_tokens, input_positions)
39
- actual = input_metadata.selected_token_indices
43
+
44
+ actual = sampling_metadata.selected_token_indices
45
expected = torch.tensor(expected_selected_token_indices,
46
device=actual.device,
47
dtype=actual.dtype)