A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Bugfix] Fix benchmark script bug: inaccurate stats for vllm backend when max_model_len < input_len + output_len (#13691)
Signed-off-by: WangErXiao <863579016@qq.com>
Robin committed
Feb 22, 2025 at 14:10 UTC
8aca27fa11bfe0539b72002761add7d990af325e
4 files changed
+43
-5
benchmarks/benchmark_guided.py
+13
@@ -46,6 +46,12 @@ def run_vllm(requests: List[SampleRequest],
46
warmup: bool = False) -> float:
47
from vllm import LLM, SamplingParams
48
llm = LLM(**vars(engine_args))
49
+ assert all(
50
+ llm.llm_engine.model_config.max_model_len >= (
51
+ request.prompt_len + request.expected_output_len)
52
+ for request in requests), (
53
+ "Please ensure that max_model_len is greater than the sum of"
54
+ " prompt_len and expected_output_len for all requests.")
55
56
# Add the requests to the engine.
57
prompts: List[str] = []
@@ -115,6 +121,13 @@ async def run_vllm_async(
121
async with build_async_engine_client_from_engine_args(
122
engine_args, disable_frontend_multiprocessing) as llm:
123
124
+ assert all(
125
+ llm.model_config.max_model_len >= (request.prompt_len +
126
+ request.expected_output_len)
127
+ for request in requests), (
128
+ "Please ensure that max_model_len is greater than the sum of"
129
+ " prompt_len and expected_output_len for all requests.")
130
+
131
# Add the requests to the engine.
132
prompts: List[str] = []
133
sampling_params: List[SamplingParams] = []
benchmarks/benchmark_latency.py
+4
@@ -42,6 +42,10 @@ def main(args: argparse.Namespace):
42
# NOTE(woosuk): If the request cannot be processed in a single batch,
43
# the engine will automatically process the request in multiple batches.
44
llm = LLM(**dataclasses.asdict(engine_args))
45
+ assert llm.llm_engine.model_config.max_model_len >= (
46
+ args.input_len + args.output_len), (
47
+ "Please ensure that max_model_len is greater than"
48
+ " the sum of input_len and output_len.")
49
50
sampling_params = SamplingParams(
51
n=args.n,
benchmarks/benchmark_prioritization.py
+14
-4
@@ -13,6 +13,11 @@ from vllm.engine.arg_utils import EngineArgs
13
from vllm.utils import FlexibleArgumentParser
14
15
16
+#Select a equi-probable random priority
17
+def get_random_flag():
18
+ return 0 if random.random() < 0.5 else 1
19
+
20
+
21
def sample_requests(
22
dataset_path: str,
23
num_requests: int,
@@ -55,8 +60,7 @@ def sample_requests(
60
# Prune too long sequences.
61
continue
62
58
- #Select a equi-probable random priority
59
- priority = 0 if random.random() < 0.5 else 1
63
+ priority = get_random_flag()
64
65
filtered_dataset.append((prompt, prompt_len, output_len, priority))
66
@@ -71,6 +75,12 @@ def run_vllm(
75
from vllm import LLM, SamplingParams
76
llm = LLM(**dataclasses.asdict(engine_args))
77
78
+ assert all(
79
+ llm.llm_engine.model_config.max_model_len >= (request[1] + request[2])
80
+ for request in requests), (
81
+ "Please ensure that max_model_len is greater than the sum of"
82
+ " input_len and output_len for all requests.")
83
+
84
# Add the requests to the engine.
85
prompts = []
86
sampling_params = []
@@ -103,8 +113,8 @@ def main(args: argparse.Namespace):
113
if args.dataset is None:
114
# Synthesize a prompt with the given input length.
115
prompt = "hi" * (args.input_len - 1)
106
- requests = [(prompt, args.input_len, args.output_len)
107
- for _ in range(args.num_prompts)]
116
+ requests = [(prompt, args.input_len, args.output_len,
117
+ get_random_flag()) for _ in range(args.num_prompts)]
118
else:
119
requests = sample_requests(args.dataset, args.num_prompts, tokenizer,
120
args.output_len)
benchmarks/benchmark_throughput.py
+12
-1
@@ -171,7 +171,12 @@ def run_vllm(
171
) -> float:
172
from vllm import LLM, SamplingParams
173
llm = LLM(**dataclasses.asdict(engine_args))
174
-
174
+ assert all(
175
+ llm.llm_engine.model_config.max_model_len >= (
176
+ request.prompt_len + request.expected_output_len)
177
+ for request in requests), (
178
+ "Please ensure that max_model_len is greater than the sum of"
179
+ " prompt_len and expected_output_len for all requests.")
180
# Add the requests to the engine.
181
prompts: List[TextPrompt] = []
182
sampling_params: List[SamplingParams] = []
@@ -229,6 +234,12 @@ async def run_vllm_async(
234
235
async with build_async_engine_client_from_engine_args(
236
engine_args, disable_frontend_multiprocessing) as llm:
237
+ assert all(
238
+ llm.model_config.max_model_len >= (request.prompt_len +
239
+ request.expected_output_len)
240
+ for request in requests), (
241
+ "Please ensure that max_model_len is greater than the sum of"
242
+ " prompt_len and expected_output_len for all requests.")
243
244
# Add the requests to the engine.
245
prompts: List[TextPrompt] = []