@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 837e185142
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[CI/Build] fix flaky test (#3602)

youkaichao committed Mar 24, 2024 at 17:43 UTC 837e1851428e8aae7fc91a837620ac6c889089b0
1 file changed +10 -16
tests/worker/test_model_runner.py
+10 -16
@@ -1,20 +1,16 @@
1 -import random
1 +import pytest
2 import torch
3
4 from vllm.config import ModelConfig
5 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
6 -from vllm.worker.model_runner import ModelRunner, _BATCH_SIZE_ALIGNMENT
6 +from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
7
8
9 -def get_aligned_size(batch_size: int, alignment: int):
10 - return ((batch_size + alignment - 1) // alignment * alignment)
11 -
12 -
13 -def test_prepare_prompt():
9 +@pytest.mark.parametrize("batch_size", list(range(1, 257)))
10 +def test_prepare_prompt(batch_size):
11 model_runner = ModelRunner(None, None, None, None, None)
12 model_runner.set_block_size(16)
13
17 - batch_size = random.randint(1, 256)
14 prompt_lens = []
15 seq_group_metadata_list = []
16 block_tables = {0: [1]}
@@ -111,7 +107,8 @@ def test_prepare_prompt():
107 torch.testing.assert_close(actual, expected)
108
109
114 -def test_prepare_decode_cuda_graph():
110 +@pytest.mark.parametrize("batch_size", list(range(1, 257)))
111 +def test_prepare_decode_cuda_graph(batch_size):
112 model_config = ModelConfig(
113 "facebook/opt-125m",
114 "facebook/opt-125m",
@@ -127,7 +124,6 @@ def test_prepare_decode_cuda_graph():
124 model_runner = ModelRunner(model_config, None, None, None, None)
125 model_runner.set_block_size(16)
126
130 - batch_size = random.randint(1, 256)
127 prompt_lens = []
128 seq_group_metadata_list = []
129 for i in range(batch_size):
@@ -147,13 +143,13 @@ def test_prepare_decode_cuda_graph():
143 input_tokens, input_positions, input_metadata, _, _, _ = (
144 model_runner._prepare_decode(seq_group_metadata_list))
145
146 + expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
147 # Verify input metadata is correct for prompts.
148 device = model_runner.device
149 assert input_metadata.is_prompt is False
150 assert input_metadata.prompt_lens is None
151 assert input_metadata.num_prompt_tokens == 0
155 - assert input_metadata.num_generation_tokens == (get_aligned_size(
156 - len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT))
152 + assert input_metadata.num_generation_tokens == expected_bs
153 assert input_metadata.max_seq_len is None
154 assert input_metadata.subquery_start_loc is None
155 assert input_metadata.seq_start_loc is None
@@ -173,10 +169,8 @@ def test_prepare_decode_cuda_graph():
169 assert input_metadata.use_cuda_graph is True
170 assert input_metadata.kv_cache_dtype == "auto"
171
176 - assert input_tokens.shape == (get_aligned_size(
177 - len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT), )
178 - assert input_positions.shape == (get_aligned_size(
179 - len(seq_group_metadata_list), _BATCH_SIZE_ALIGNMENT), )
172 + assert input_tokens.shape == (expected_bs, )
173 + assert input_positions.shape == (expected_bs, )
174 torch.testing.assert_close(input_tokens, input_positions)
175
176 # Verify Sampling