A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix latency benchmark script (#118)
Woosuk Kwon committed
May 22, 2023 at 17:03 UTC
3f942acfe15de367931a63aa96c1931eb74799c0
2 files changed
+43
-31
benchmark/benchmark_latency.py
+33
-29
@@ -1,71 +1,75 @@
1
import argparse
2
import time
3
-from typing import List
3
5
-from tqdm import tqdm
4
import numpy as np
5
import torch
6
+from tqdm import tqdm
7
9
-from cacheflow.core.server import (
10
- add_server_arguments, process_server_arguments,
11
- init_local_server_and_frontend_with_arguments)
12
-from cacheflow.sampling_params import SamplingParams
8
+from cacheflow import LLM, SamplingParams
9
10
11
def main(args: argparse.Namespace):
16
- server, frontend = init_local_server_and_frontend_with_arguments(args)
12
+ print(args)
13
+
14
+ # Process all the requests in a single batch if possible.
15
+ # NOTE(woosuk): If the request cannot be processed in a single batch,
16
+ # the server will automatically process the request in multiple batches.
17
+ llm = LLM(
18
+ model=args.model,
19
+ tensor_parallel_size=args.tensor_parallel_size,
20
+ max_num_seqs=args.batch_size,
21
+ max_num_batched_tokens=args.batch_size * args.input_len,
22
+ )
23
24
sampling_params = SamplingParams(
25
n=args.n,
26
temperature=0.0 if args.use_beam_search else 1.0,
27
top_p=1.0,
28
use_beam_search=args.use_beam_search,
23
- stop_token_ids=set(),
29
+ ignore_eos=True,
30
max_tokens=args.output_len,
31
)
32
print(sampling_params)
27
- input_token_ids = [0] * args.input_len
33
+ dummy_prompts = [""] * args.batch_size
34
+ dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
35
29
- def profile_step(profile=False):
36
+ def run_to_completion(profile: bool = False):
37
if profile:
38
torch.cuda.cudart().cudaProfilerStart()
32
- for _ in range(args.batch_size):
33
- dummy_prompt = ""
34
- frontend._add_query(dummy_prompt, input_token_ids, sampling_params)
35
- server.add_sequence_groups(frontend.get_inputs())
39
start_time = time.time()
37
- while True:
38
- server.step()
39
- if not server.has_unfinished_requests():
40
- break
40
+
41
+ llm.generate(dummy_prompts, sampling_params, dummy_prompt_token_ids,
42
+ use_tqdm=False)
43
+
44
end_time = time.time()
45
latency = end_time - start_time
46
if profile:
47
torch.cuda.cudart().cudaProfilerStop()
48
return latency
49
47
- print("Warm up step")
48
- profile_step()
50
+ print("Warming up...")
51
+ run_to_completion(profile=False)
52
53
# Benchmark.
54
latencies = []
52
- for _ in tqdm(range(3), desc="Profile step"):
53
- latencies.append(profile_step())
55
+ for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
56
+ latencies.append(run_to_completion(profile=False))
57
print(f'Avg latency: {np.mean(latencies)} seconds')
58
59
60
if __name__ == '__main__':
61
parser = argparse.ArgumentParser(
59
- description='Benchmark the latency of decoding a single sentence.')
60
- parser = add_server_arguments(parser)
62
+ description='Benchmark the latency of processing a single batch of '
63
+ 'requests till completion.')
64
+ parser.add_argument('--model', type=str, default='facebook/opt-125m')
65
+ parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
66
parser.add_argument('--input-len', type=int, default=32)
67
parser.add_argument('--output-len', type=int, default=128)
68
parser.add_argument('--batch-size', type=int, default=8)
64
- parser.add_argument('--n', type=int, default=1)
69
+ parser.add_argument('--n', type=int, default=1,
70
+ help='Number of generated sequences per prompt.')
71
parser.add_argument('--use-beam-search', action='store_true')
72
+ parser.add_argument('--num-iters', type=int, default=3,
73
+ help='Number of iterations to run.')
74
args = parser.parse_args()
67
- args = process_server_arguments(args)
68
- args.max_num_batched_tokens = max(
69
- args.max_num_batched_tokens, args.batch_size * args.input_len)
70
- print(args)
75
main(args)
cacheflow/entrypoints/llm.py
+10
-2
@@ -35,18 +35,26 @@ class LLM:
35
self,
36
prompts: List[str],
37
sampling_params: Optional[SamplingParams] = None,
38
+ prompt_token_ids: Optional[List[List[int]]] = None,
39
use_tqdm: bool = True,
40
) -> List[RequestOutput]:
41
if sampling_params is None:
42
+ # Use default sampling params.
43
sampling_params = SamplingParams()
44
# Initialize tqdm.
45
if use_tqdm:
46
pbar = tqdm(total=len(prompts), desc="Processed prompts")
47
48
# Add requests to the server.
47
- for prompt in prompts:
49
+ for i in range(len(prompts)):
50
+ prompt = prompts[i]
51
+ if prompt_token_ids is None:
52
+ token_ids = None
53
+ else:
54
+ token_ids = prompt_token_ids[i]
55
request_id = str(next(self.request_counter))
49
- self.llm_server.add_request(request_id, prompt, sampling_params)
56
+ self.llm_server.add_request(request_id, prompt, sampling_params,
57
+ token_ids)
58
59
# Run the server.
60
outputs: List[RequestOutput] = []