@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 660a7fcfa4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add DeepSpeed MII backend to benchmark script (#1649)

Woosuk Kwon committed Nov 14, 2023 at 12:35 UTC 660a7fcfa40d62305ecba6bc6352c4026d56d680
1 file changed +71 -12
benchmarks/benchmark_throughput.py
+71 -12
@@ -6,18 +6,21 @@ import time
6 from typing import List, Optional, Tuple
7
8 import torch
9 -from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
9 +from transformers import (AutoModelForCausalLM, AutoTokenizer,
10 + PreTrainedTokenizerBase)
11 from tqdm import tqdm
12
12 -from vllm import LLM, SamplingParams
13 -from vllm.transformers_utils.tokenizer import get_tokenizer
14 -
13
14 def sample_requests(
15 dataset_path: str,
16 num_requests: int,
17 tokenizer: PreTrainedTokenizerBase,
18 + fixed_output_len: Optional[int],
19 ) -> List[Tuple[str, int, int]]:
20 + if fixed_output_len is not None:
21 + if fixed_output_len < 4:
22 + raise ValueError("output_len too small")
23 +
24 # Load the dataset.
25 with open(dataset_path) as f:
26 dataset = json.load(f)
@@ -35,6 +38,8 @@ def sample_requests(
38 tokenized_dataset = []
39 for i in range(len(dataset)):
40 output_len = len(completion_token_ids[i])
41 + if fixed_output_len is not None:
42 + output_len = fixed_output_len
43 tokenized_dataset.append((prompts[i], prompt_token_ids[i], output_len))
44
45 # Filter out too long sequences.
@@ -66,6 +71,7 @@ def run_vllm(
71 trust_remote_code: bool,
72 dtype: str,
73 ) -> float:
74 + from vllm import LLM, SamplingParams
75 llm = LLM(
76 model=model,
77 tokenizer=tokenizer,
@@ -160,14 +166,37 @@ def run_hf(
166 return end - start
167
168
169 +def run_mii(
170 + requests: List[Tuple[str, int, int]],
171 + model: str,
172 + tensor_parallel_size: int,
173 + output_len: int,
174 +) -> float:
175 + from mii import pipeline
176 + llm = pipeline(model, tensor_parallel=tensor_parallel_size)
177 + prompts = [prompt for prompt, _, _ in requests]
178 +
179 + start = time.perf_counter()
180 + llm(prompts, max_new_tokens=output_len)
181 + end = time.perf_counter()
182 + return end - start
183 +
184 +
185 def main(args: argparse.Namespace):
186 print(args)
187 random.seed(args.seed)
188
189 # Sample the requests.
168 - tokenizer = get_tokenizer(args.tokenizer,
169 - trust_remote_code=args.trust_remote_code)
170 - requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
190 + tokenizer = AutoTokenizer.from_pretrained(
191 + args.tokenizer, trust_remote_code=args.trust_remote_code)
192 + if args.dataset is None:
193 + # Synthesize a prompt with the given input length.
194 + prompt = "hi" * (args.input_len - 1)
195 + requests = [(prompt, args.input_len, args.output_len)
196 + for _ in range(args.num_prompts)]
197 + else:
198 + requests = sample_requests(args.dataset, args.num_prompts, tokenizer,
199 + args.output_len)
200
201 if args.backend == "vllm":
202 elapsed_time = run_vllm(requests, args.model, args.tokenizer,
@@ -179,6 +208,9 @@ def main(args: argparse.Namespace):
208 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
209 args.use_beam_search, args.hf_max_batch_size,
210 args.trust_remote_code)
211 + elif args.backend == "mii":
212 + elapsed_time = run_mii(requests, args.model, args.tensor_parallel_size,
213 + args.output_len)
214 else:
215 raise ValueError(f"Unknown backend: {args.backend}")
216 total_num_tokens = sum(prompt_len + output_len
@@ -191,12 +223,21 @@ if __name__ == "__main__":
223 parser = argparse.ArgumentParser(description="Benchmark the throughput.")
224 parser.add_argument("--backend",
225 type=str,
194 - choices=["vllm", "hf"],
226 + choices=["vllm", "hf", "mii"],
227 default="vllm")
228 parser.add_argument("--dataset",
229 type=str,
198 - required=True,
230 + default=None,
231 help="Path to the dataset.")
232 + parser.add_argument("--input-len",
233 + type=int,
234 + default=None,
235 + help="Input prompt length for each request")
236 + parser.add_argument("--output-len",
237 + type=int,
238 + default=None,
239 + help="Output length for each request. Overrides the "
240 + "output length from the dataset.")
241 parser.add_argument("--model", type=str, default="facebook/opt-125m")
242 parser.add_argument("--tokenizer", type=str, default=None)
243 parser.add_argument('--quantization',
@@ -231,6 +272,13 @@ if __name__ == "__main__":
272 'for FP32 and FP16 models, and BF16 precision '
273 'for BF16 models.')
274 args = parser.parse_args()
275 + if args.tokenizer is None:
276 + args.tokenizer = args.model
277 + if args.dataset is None:
278 + assert args.input_len is not None
279 + assert args.output_len is not None
280 + else:
281 + assert args.input_len is None
282
283 if args.backend == "vllm":
284 if args.hf_max_batch_size is not None:
@@ -240,7 +288,18 @@ if __name__ == "__main__":
288 raise ValueError("HF max batch size is required for HF backend.")
289 if args.quantization is not None:
290 raise ValueError("Quantization is only for vLLM backend.")
243 - if args.tokenizer is None:
244 - args.tokenizer = args.model
245 -
291 + elif args.backend == "mii":
292 + if args.dtype != "auto":
293 + raise ValueError("dtype must be auto for MII backend.")
294 + if args.n != 1:
295 + raise ValueError("n must be 1 for MII backend.")
296 + if args.use_beam_search:
297 + raise ValueError("Beam search is not supported for MII backend.")
298 + if args.quantization is not None:
299 + raise ValueError("Quantization is only for vLLM backend.")
300 + if args.hf_max_batch_size is not None:
301 + raise ValueError("HF max batch size is only for HF backend.")
302 + if args.tokenizer != args.model:
303 + raise ValueError("Tokenizer must be the same as the model for MII "
304 + "backend.")
305 main(args)