A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[CI/Build] fix flaky test (#3602)
youkaichao committed
Mar 24, 2024 at 17:43 UTC
837e1851428e8aae7fc91a837620ac6c889089b0
1 file changed
+10
-16
tests/worker/test_model_runner.py
+10
-16
@@ -1,20 +1,16 @@
1
-import random
1
+import pytest
2
import torch
3
4
from vllm.config import ModelConfig
5
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
6
-from vllm.worker.model_runner import ModelRunner, _BATCH_SIZE_ALIGNMENT
6
+from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
7
8
9
-def get_aligned_size(batch_size: int, alignment: int):
10
- return ((batch_size + alignment - 1) // alignment * alignment)
11
-
12
-
13
-def test_prepare_prompt():
9
+@pytest.mark.parametrize("batch_size", list(range(1, 257)))
10
+def test_prepare_prompt(batch_size):
11
model_runner = ModelRunner(None, None, None, None, None)
12
model_runner.set_block_size(16)
13
17
- batch_size = random.randint(1, 256)
14
prompt_lens = []
15
seq_group_metadata_list = []
16
block_tables = {0: [1]}
@@ -111,7 +107,8 @@ def test_prepare_prompt():
107
torch.testing.assert_close(actual, expected)
108
109
114
-def test_prepare_decode_cuda_graph():
110
+@pytest.mark.parametrize("batch_size", list(range(1, 257)))
111
+def test_prepare_decode_cuda_graph(batch_size):
112
model_config = ModelConfig(
113
"facebook/opt-125m",
114
"facebook/opt-125m",
@@ -127,7 +124,6 @@ def test_prepare_decode_cuda_graph():
124
model_runner = ModelRunner(model_config, None, None, None, None)
125
model_runner.set_block_size(16)
126
130
- batch_size = random.randint(1, 256)
127
prompt_lens = []
128
seq_group_metadata_list = []
129
for i in range(batch_size):
@@ -147,13 +143,13 @@ def test_prepare_decode_cuda_graph():
143
input_tokens, input_positions, input_metadata, _, _, _ = (
144
model_runner._prepare_decode(seq_group_metadata_list))
145
146
+ expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
147
# Verify input metadata is correct for prompts.
148
device = model_runner.device
149
assert input_metadata.is_prompt is False
150
assert input_metadata.prompt_lens is None
151
assert input_metadata.num_prompt_tokens == 0
155
- assert input_metadata.num_generation_tokens == (get_aligned_size(
156
- len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT))
152
+ assert input_metadata.num_generation_tokens == expected_bs
153
assert input_metadata.max_seq_len is None
154
assert input_metadata.subquery_start_loc is None
155
assert input_metadata.seq_start_loc is None
@@ -173,10 +169,8 @@ def test_prepare_decode_cuda_graph():
169
assert input_metadata.use_cuda_graph is True
170
assert input_metadata.kv_cache_dtype == "auto"
171
176
- assert input_tokens.shape == (get_aligned_size(
177
- len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT), )
178
- assert input_positions.shape == (get_aligned_size(
179
- len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT), )
172
+ assert input_tokens.shape == (expected_bs, )
173
+ assert input_positions.shape == (expected_bs, )
174
torch.testing.assert_close(input_tokens, input_positions)
175
176
# Verify Sampling