@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd3aa153a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix broken worker test (#1900)

Woosuk Kwon committed Dec 2, 2023 at 22:17 UTC cd3aa153a4e4974802385f209ad343149af02c07
1 file changed +14 -9
tests/worker/test_model_runner.py renamed
+14 -9
@@ -2,18 +2,19 @@ import random
2 import torch
3
4 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
5 -from vllm.worker.worker import Worker
5 +from vllm.worker.model_runner import ModelRunner
6
7
8 -def test_worker_prepare_inputs_for_prompt():
9 - worker = Worker(None, None, None)
10 - worker.block_size = 16
8 +def test_prepare_prompt():
9 + model_runner = ModelRunner(None, None, None)
10 + model_runner.set_block_size(16)
11 +
12 batch_size = random.randint(1, 256)
13 prompt_lens = []
14 seq_group_metadata_list = []
15 for i in range(batch_size):
16 # make sure all tokens fit into one block
16 - prompt_len = i % (worker.block_size - 1) + 1
17 + prompt_len = i % (model_runner.block_size - 1) + 1
18 prompt_lens.append(prompt_len)
19 seq_data = list(range(prompt_len))
20 seq_group_metadata_list.append(
@@ -24,6 +25,7 @@ def test_worker_prepare_inputs_for_prompt():
25 sampling_params=SamplingParams(temperature=0),
26 block_tables={0: [1]},
27 ))
28 +
29 expected_selected_token_indices = []
30 selected_token_start_idx = 0
31 max_seq_len = max(prompt_lens)
@@ -31,12 +33,15 @@ def test_worker_prepare_inputs_for_prompt():
33 expected_selected_token_indices.append(selected_token_start_idx +
34 prompt_len - 1)
35 selected_token_start_idx += max_seq_len
34 - input_tokens, input_positions, input_metadata = worker._prepare_inputs(
36 + input_tokens, input_positions, _ = model_runner._prepare_prompt(
37 seq_group_metadata_list)
36 - assert input_tokens.shape == input_positions.shape == (batch_size,
37 - max_seq_len)
38 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
39 + prompt_lens)
40 + assert input_tokens.shape == (batch_size, max_seq_len)
41 + assert input_positions.shape == (batch_size, max_seq_len)
42 torch.testing.assert_close(input_tokens, input_positions)
39 - actual = input_metadata.selected_token_indices
43 +
44 + actual = sampling_metadata.selected_token_indices
45 expected = torch.tensor(expected_selected_token_indices,
46 device=actual.device,
47 dtype=actual.dtype)