@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9a5664d4a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc] Refactor benchmark_throughput.py (#9779)

Signed-off-by: Linkun Chen <github+anyscale@lkchen.net> Co-authored-by: Linkun Chen <lkchen@github.com> Co-authored-by: Linkun Chen <github+anyscale@lkchen.net>

lkchen committed Nov 4, 2024 at 14:32 UTC 9a5664d4a4d212a6ebad79b15b11eb8d3ab2a0b2
1 file changed +55 -26
benchmarks/benchmark_throughput.py
+55 -26
@@ -4,7 +4,7 @@ import dataclasses
4 import json
5 import random
6 import time
7 -from typing import List, Optional, Tuple
7 +from typing import List, Optional
8
9 import torch
10 import uvloop
@@ -15,16 +15,35 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer,
15 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
16 from vllm.entrypoints.openai.api_server import (
17 build_async_engine_client_from_engine_args)
18 +from vllm.inputs import TextPrompt
19 +from vllm.multimodal import MultiModalDataDict
20 from vllm.sampling_params import BeamSearchParams
21 from vllm.utils import FlexibleArgumentParser, merge_async_iterators
22
23
24 +@dataclasses.dataclass
25 +class SampleRequest:
26 + """A class representing a single inference request for benchmarking.
27 +
28 + Attributes:
29 + prompt: The input text prompt for the model.
30 + multi_modal_data: Optional dictionary containing multi-modal data (e.g.
31 + images).
32 + prompt_len: The length of the prompt in tokens.
33 + expected_output_len: The expected length of the output in tokens.
34 + """
35 + prompt: str
36 + prompt_len: int
37 + expected_output_len: int
38 + multi_modal_data: Optional[MultiModalDataDict] = None
39 +
40 +
41 def sample_requests(
42 dataset_path: str,
43 num_requests: int,
44 tokenizer: PreTrainedTokenizerBase,
45 fixed_output_len: Optional[int],
27 -) -> List[Tuple[str, int, int]]:
46 +) -> List[SampleRequest]:
47 if fixed_output_len is not None and fixed_output_len < 4:
48 raise ValueError("output_len too small")
49
@@ -41,7 +60,7 @@ def sample_requests(
60 random.shuffle(dataset)
61
62 # Filter out sequences that are too long or too short
44 - filtered_dataset: List[Tuple[str, int, int]] = []
63 + filtered_dataset: List[SampleRequest] = []
64 for i in range(len(dataset)):
65 if len(filtered_dataset) == num_requests:
66 break
@@ -60,13 +79,16 @@ def sample_requests(
79 if prompt_len > 1024 or prompt_len + output_len > 2048:
80 # Prune too long sequences.
81 continue
63 - filtered_dataset.append((prompt, prompt_len, output_len))
82 + filtered_dataset.append(
83 + SampleRequest(prompt=prompt,
84 + prompt_len=prompt_len,
85 + expected_output_len=output_len))
86
87 return filtered_dataset
88
89
90 def run_vllm(
69 - requests: List[Tuple[str, int, int]],
91 + requests: List[SampleRequest],
92 n: int,
93 engine_args: EngineArgs,
94 ) -> float:
@@ -74,17 +96,17 @@ def run_vllm(
96 llm = LLM(**dataclasses.asdict(engine_args))
97
98 # Add the requests to the engine.
77 - prompts: List[str] = []
99 + prompts: List[TextPrompt] = []
100 sampling_params: List[SamplingParams] = []
79 - for prompt, _, output_len in requests:
80 - prompts.append(prompt)
101 + for request in requests:
102 + prompts.append(TextPrompt(prompt=request.prompt))
103 sampling_params.append(
104 SamplingParams(
105 n=n,
106 temperature=1.0,
107 top_p=1.0,
108 ignore_eos=True,
87 - max_tokens=output_len,
109 + max_tokens=request.expected_output_len,
110 ))
111
112 use_beam_search = False
@@ -94,11 +116,11 @@ def run_vllm(
116 llm.generate(prompts, sampling_params, use_tqdm=True)
117 end = time.perf_counter()
118 else:
97 - prompts = [prompt for prompt, _, _ in requests]
119 + prompts = [request.prompt for request in requests]
120 # output_len should be the same for all requests.
121 output_len = requests[0][2]
100 - for prompt, input_len, _output_len in requests:
101 - assert _output_len == output_len
122 + for request in requests:
123 + assert request.expected_output_len == output_len
124 start = time.perf_counter()
125 llm.beam_search(
126 prompts,
@@ -112,7 +134,7 @@ def run_vllm(
134
135
136 async def run_vllm_async(
115 - requests: List[Tuple[str, int, int]],
137 + requests: List[SampleRequest],
138 n: int,
139 engine_args: AsyncEngineArgs,
140 disable_frontend_multiprocessing: bool = False,
@@ -123,17 +145,17 @@ async def run_vllm_async(
145 engine_args, disable_frontend_multiprocessing) as llm:
146
147 # Add the requests to the engine.
126 - prompts: List[str] = []
148 + prompts: List[TextPrompt] = []
149 sampling_params: List[SamplingParams] = []
128 - for prompt, _, output_len in requests:
129 - prompts.append(prompt)
150 + for request in requests:
151 + prompts.append(TextPrompt(prompt=request.prompt))
152 sampling_params.append(
153 SamplingParams(
154 n=n,
155 temperature=1.0,
156 top_p=1.0,
157 ignore_eos=True,
136 - max_tokens=output_len,
158 + max_tokens=request.expected_output_len,
159 ))
160
161 generators = []
@@ -149,7 +171,7 @@ async def run_vllm_async(
171
172
173 def run_hf(
152 - requests: List[Tuple[str, int, int]],
174 + requests: List[SampleRequest],
175 model: str,
176 tokenizer: PreTrainedTokenizerBase,
177 n: int,
@@ -207,14 +229,14 @@ def run_hf(
229
230
231 def run_mii(
210 - requests: List[Tuple[str, int, int]],
232 + requests: List[SampleRequest],
233 model: str,
234 tensor_parallel_size: int,
235 output_len: int,
236 ) -> float:
237 from mii import client, serve
238 llm = serve(model, tensor_parallel=tensor_parallel_size)
217 - prompts = [prompt for prompt, _, _ in requests]
239 + prompts = [request.prompt for request in requests]
240
241 start = time.perf_counter()
242 llm.generate(prompts, max_new_tokens=output_len)
@@ -243,8 +265,12 @@ def main(args: argparse.Namespace):
265 else:
266 raise ValueError(
267 f"Failed to synthesize a prompt with {args.input_len} tokens.")
246 - requests = [(prompt, args.input_len, args.output_len)
247 - for _ in range(args.num_prompts)]
268 + requests = [
269 + SampleRequest(prompt=prompt,
270 + prompt_len=args.input_len,
271 + expected_output_len=args.output_len)
272 + for _ in range(args.num_prompts)
273 + ]
274 else:
275 requests = sample_requests(args.dataset, args.num_prompts, tokenizer,
276 args.output_len)
@@ -270,9 +296,10 @@ def main(args: argparse.Namespace):
296 args.output_len)
297 else:
298 raise ValueError(f"Unknown backend: {args.backend}")
273 - total_num_tokens = sum(prompt_len + output_len
274 - for _, prompt_len, output_len in requests)
275 - total_output_tokens = sum(output_len for _, _, output_len in requests)
299 + total_num_tokens = sum(request.prompt_len + request.expected_output_len
300 + for request in requests)
301 + total_output_tokens = sum(request.expected_output_len
302 + for request in requests)
303 print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
304 f"{total_num_tokens / elapsed_time:.2f} total tokens/s, "
305 f"{total_output_tokens / elapsed_time:.2f} output tokens/s")
@@ -299,7 +326,9 @@ if __name__ == "__main__":
326 parser.add_argument("--dataset",
327 type=str,
328 default=None,
302 - help="Path to the dataset.")
329 + help="Path to the dataset. The dataset is expected to "
330 + "be a json in form of List[Dict[..., conversations: "
331 + "List[Dict[..., value: <prompt_or_response>]]]]")
332 parser.add_argument("--input-len",
333 type=int,
334 default=None,