A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc] Refactor benchmark_throughput.py (#9779)
Signed-off-by: Linkun Chen <github+anyscale@lkchen.net> Co-authored-by: Linkun Chen <lkchen@github.com> Co-authored-by: Linkun Chen <github+anyscale@lkchen.net>
lkchen committed
Nov 4, 2024 at 14:32 UTC
9a5664d4a4d212a6ebad79b15b11eb8d3ab2a0b2
1 file changed
+55
-26
benchmarks/benchmark_throughput.py
+55
-26
@@ -4,7 +4,7 @@ import dataclasses
4
import json
5
import random
6
import time
7
-from typing import List, Optional, Tuple
7
+from typing import List, Optional
8
9
import torch
10
import uvloop
@@ -15,16 +15,35 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer,
15
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
16
from vllm.entrypoints.openai.api_server import (
17
build_async_engine_client_from_engine_args)
18
+from vllm.inputs import TextPrompt
19
+from vllm.multimodal import MultiModalDataDict
20
from vllm.sampling_params import BeamSearchParams
21
from vllm.utils import FlexibleArgumentParser, merge_async_iterators
22
23
24
+@dataclasses.dataclass
25
+class SampleRequest:
26
+ """A class representing a single inference request for benchmarking.
27
+
28
+ Attributes:
29
+ prompt: The input text prompt for the model.
30
+ multi_modal_data: Optional dictionary containing multi-modal data (e.g.
31
+ images).
32
+ prompt_len: The length of the prompt in tokens.
33
+ expected_output_len: The expected length of the output in tokens.
34
+ """
35
+ prompt: str
36
+ prompt_len: int
37
+ expected_output_len: int
38
+ multi_modal_data: Optional[MultiModalDataDict] = None
39
+
40
+
41
def sample_requests(
42
dataset_path: str,
43
num_requests: int,
44
tokenizer: PreTrainedTokenizerBase,
45
fixed_output_len: Optional[int],
27
-) -> List[Tuple[str, int, int]]:
46
+) -> List[SampleRequest]:
47
if fixed_output_len is not None and fixed_output_len < 4:
48
raise ValueError("output_len too small")
49
@@ -41,7 +60,7 @@ def sample_requests(
60
random.shuffle(dataset)
61
62
# Filter out sequences that are too long or too short
44
- filtered_dataset: List[Tuple[str, int, int]] = []
63
+ filtered_dataset: List[SampleRequest] = []
64
for i in range(len(dataset)):
65
if len(filtered_dataset) == num_requests:
66
break
@@ -60,13 +79,16 @@ def sample_requests(
79
if prompt_len > 1024 or prompt_len + output_len > 2048:
80
# Prune too long sequences.
81
continue
63
- filtered_dataset.append((prompt, prompt_len, output_len))
82
+ filtered_dataset.append(
83
+ SampleRequest(prompt=prompt,
84
+ prompt_len=prompt_len,
85
+ expected_output_len=output_len))
86
87
return filtered_dataset
88
89
90
def run_vllm(
69
- requests: List[Tuple[str, int, int]],
91
+ requests: List[SampleRequest],
92
n: int,
93
engine_args: EngineArgs,
94
) -> float:
@@ -74,17 +96,17 @@ def run_vllm(
96
llm = LLM(**dataclasses.asdict(engine_args))
97
98
# Add the requests to the engine.
77
- prompts: List[str] = []
99
+ prompts: List[TextPrompt] = []
100
sampling_params: List[SamplingParams] = []
79
- for prompt, _, output_len in requests:
80
- prompts.append(prompt)
101
+ for request in requests:
102
+ prompts.append(TextPrompt(prompt=request.prompt))
103
sampling_params.append(
104
SamplingParams(
105
n=n,
106
temperature=1.0,
107
top_p=1.0,
108
ignore_eos=True,
87
- max_tokens=output_len,
109
+ max_tokens=request.expected_output_len,
110
))
111
112
use_beam_search = False
@@ -94,11 +116,11 @@ def run_vllm(
116
llm.generate(prompts, sampling_params, use_tqdm=True)
117
end = time.perf_counter()
118
else:
97
- prompts = [prompt for prompt, _, _ in requests]
119
+ prompts = [request.prompt for request in requests]
120
# output_len should be the same for all requests.
121
output_len = requests[0][2]
100
- for prompt, input_len, _output_len in requests:
101
- assert _output_len == output_len
122
+ for request in requests:
123
+ assert request.expected_output_len == output_len
124
start = time.perf_counter()
125
llm.beam_search(
126
prompts,
@@ -112,7 +134,7 @@ def run_vllm(
134
135
136
async def run_vllm_async(
115
- requests: List[Tuple[str, int, int]],
137
+ requests: List[SampleRequest],
138
n: int,
139
engine_args: AsyncEngineArgs,
140
disable_frontend_multiprocessing: bool = False,
@@ -123,17 +145,17 @@ async def run_vllm_async(
145
engine_args, disable_frontend_multiprocessing) as llm:
146
147
# Add the requests to the engine.
126
- prompts: List[str] = []
148
+ prompts: List[TextPrompt] = []
149
sampling_params: List[SamplingParams] = []
128
- for prompt, _, output_len in requests:
129
- prompts.append(prompt)
150
+ for request in requests:
151
+ prompts.append(TextPrompt(prompt=request.prompt))
152
sampling_params.append(
153
SamplingParams(
154
n=n,
155
temperature=1.0,
156
top_p=1.0,
157
ignore_eos=True,
136
- max_tokens=output_len,
158
+ max_tokens=request.expected_output_len,
159
))
160
161
generators = []
@@ -149,7 +171,7 @@ async def run_vllm_async(
171
172
173
def run_hf(
152
- requests: List[Tuple[str, int, int]],
174
+ requests: List[SampleRequest],
175
model: str,
176
tokenizer: PreTrainedTokenizerBase,
177
n: int,
@@ -207,14 +229,14 @@ def run_hf(
229
230
231
def run_mii(
210
- requests: List[Tuple[str, int, int]],
232
+ requests: List[SampleRequest],
233
model: str,
234
tensor_parallel_size: int,
235
output_len: int,
236
) -> float:
237
from mii import client, serve
238
llm = serve(model, tensor_parallel=tensor_parallel_size)
217
- prompts = [prompt for prompt, _, _ in requests]
239
+ prompts = [request.prompt for request in requests]
240
241
start = time.perf_counter()
242
llm.generate(prompts, max_new_tokens=output_len)
@@ -243,8 +265,12 @@ def main(args: argparse.Namespace):
265
else:
266
raise ValueError(
267
f"Failed to synthesize a prompt with {args.input_len} tokens.")
246
- requests = [(prompt, args.input_len, args.output_len)
247
- for _ in range(args.num_prompts)]
268
+ requests = [
269
+ SampleRequest(prompt=prompt,
270
+ prompt_len=args.input_len,
271
+ expected_output_len=args.output_len)
272
+ for _ in range(args.num_prompts)
273
+ ]
274
else:
275
requests = sample_requests(args.dataset, args.num_prompts, tokenizer,
276
args.output_len)
@@ -270,9 +296,10 @@ def main(args: argparse.Namespace):
296
args.output_len)
297
else:
298
raise ValueError(f"Unknown backend: {args.backend}")
273
- total_num_tokens = sum(prompt_len + output_len
274
- for _, prompt_len, output_len in requests)
275
- total_output_tokens = sum(output_len for _, _, output_len in requests)
299
+ total_num_tokens = sum(request.prompt_len + request.expected_output_len
300
+ for request in requests)
301
+ total_output_tokens = sum(request.expected_output_len
302
+ for request in requests)
303
print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
304
f"{total_num_tokens / elapsed_time:.2f} total tokens/s, "
305
f"{total_output_tokens / elapsed_time:.2f} output tokens/s")
@@ -299,7 +326,9 @@ if __name__ == "__main__":
326
parser.add_argument("--dataset",
327
type=str,
328
default=None,
302
- help="Path to the dataset.")
329
+ help="Path to the dataset. The dataset is expected to "
330
+ "be a json in form of List[Dict[..., conversations: "
331
+ "List[Dict[..., value: <prompt_or_response>]]]]")
332
parser.add_argument("--input-len",
333
type=int,
334
default=None,