@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f942acfe1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix latency benchmark script (#118)

Woosuk Kwon committed May 22, 2023 at 17:03 UTC 3f942acfe15de367931a63aa96c1931eb74799c0
2 files changed +43 -31
benchmark/benchmark_latency.py
+33 -29
@@ -1,71 +1,75 @@
1 import argparse
2 import time
3 -from typing import List
3
5 -from tqdm import tqdm
4 import numpy as np
5 import torch
6 +from tqdm import tqdm
7
9 -from cacheflow.core.server import (
10 - add_server_arguments, process_server_arguments,
11 - init_local_server_and_frontend_with_arguments)
12 -from cacheflow.sampling_params import SamplingParams
8 +from cacheflow import LLM, SamplingParams
9
10
11 def main(args: argparse.Namespace):
16 - server, frontend = init_local_server_and_frontend_with_arguments(args)
12 + print(args)
13 +
14 + # Process all the requests in a single batch if possible.
15 + # NOTE(woosuk): If the request cannot be processed in a single batch,
16 + # the server will automatically process the request in multiple batches.
17 + llm = LLM(
18 + model=args.model,
19 + tensor_parallel_size=args.tensor_parallel_size,
20 + max_num_seqs=args.batch_size,
21 + max_num_batched_tokens=args.batch_size * args.input_len,
22 + )
23
24 sampling_params = SamplingParams(
25 n=args.n,
26 temperature=0.0 if args.use_beam_search else 1.0,
27 top_p=1.0,
28 use_beam_search=args.use_beam_search,
23 - stop_token_ids=set(),
29 + ignore_eos=True,
30 max_tokens=args.output_len,
31 )
32 print(sampling_params)
27 - input_token_ids = [0] * args.input_len
33 + dummy_prompts = [""] * args.batch_size
34 + dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
35
29 - def profile_step(profile=False):
36 + def run_to_completion(profile: bool = False):
37 if profile:
38 torch.cuda.cudart().cudaProfilerStart()
32 - for _ in range(args.batch_size):
33 - dummy_prompt = ""
34 - frontend._add_query(dummy_prompt, input_token_ids, sampling_params)
35 - server.add_sequence_groups(frontend.get_inputs())
39 start_time = time.time()
37 - while True:
38 - server.step()
39 - if not server.has_unfinished_requests():
40 - break
40 +
41 + llm.generate(dummy_prompts, sampling_params, dummy_prompt_token_ids,
42 + use_tqdm=False)
43 +
44 end_time = time.time()
45 latency = end_time - start_time
46 if profile:
47 torch.cuda.cudart().cudaProfilerStop()
48 return latency
49
47 - print("Warm up step")
48 - profile_step()
50 + print("Warming up...")
51 + run_to_completion(profile=False)
52
53 # Benchmark.
54 latencies = []
52 - for _ in tqdm(range(3), desc="Profile step"):
53 - latencies.append(profile_step())
55 + for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
56 + latencies.append(run_to_completion(profile=False))
57 print(f'Avg latency: {np.mean(latencies)} seconds')
58
59
60 if __name__ == '__main__':
61 parser = argparse.ArgumentParser(
59 - description='Benchmark the latency of decoding a single sentence.')
60 - parser = add_server_arguments(parser)
62 + description='Benchmark the latency of processing a single batch of '
63 + 'requests till completion.')
64 + parser.add_argument('--model', type=str, default='facebook/opt-125m')
65 + parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
66 parser.add_argument('--input-len', type=int, default=32)
67 parser.add_argument('--output-len', type=int, default=128)
68 parser.add_argument('--batch-size', type=int, default=8)
64 - parser.add_argument('--n', type=int, default=1)
69 + parser.add_argument('--n', type=int, default=1,
70 + help='Number of generated sequences per prompt.')
71 parser.add_argument('--use-beam-search', action='store_true')
72 + parser.add_argument('--num-iters', type=int, default=3,
73 + help='Number of iterations to run.')
74 args = parser.parse_args()
67 - args = process_server_arguments(args)
68 - args.max_num_batched_tokens = max(
69 - args.max_num_batched_tokens, args.batch_size * args.input_len)
70 - print(args)
75 main(args)
cacheflow/entrypoints/llm.py
+10 -2
@@ -35,18 +35,26 @@ class LLM:
35 self,
36 prompts: List[str],
37 sampling_params: Optional[SamplingParams] = None,
38 + prompt_token_ids: Optional[List[List[int]]] = None,
39 use_tqdm: bool = True,
40 ) -> List[RequestOutput]:
41 if sampling_params is None:
42 + # Use default sampling params.
43 sampling_params = SamplingParams()
44 # Initialize tqdm.
45 if use_tqdm:
46 pbar = tqdm(total=len(prompts), desc="Processed prompts")
47
48 # Add requests to the server.
47 - for prompt in prompts:
49 + for i in range(len(prompts)):
50 + prompt = prompts[i]
51 + if prompt_token_ids is None:
52 + token_ids = None
53 + else:
54 + token_ids = prompt_token_ids[i]
55 request_id = str(next(self.request_counter))
49 - self.llm_server.add_request(request_id, prompt, sampling_params)
56 + self.llm_server.add_request(request_id, prompt, sampling_params,
57 + token_ids)
58
59 # Run the server.
60 outputs: List[RequestOutput] = []