@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 311490a720
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add script for benchmarking serving throughput (#145)

Woosuk Kwon committed Jun 14, 2023 at 19:55 UTC 311490a720951f322977d811eacea685a623b5a1
10 files changed +421 -415
benchmarks/benchmark_async_llm_server.py
+5 -3
@@ -10,6 +10,7 @@ def main(args: argparse.Namespace):
10 prompts = [f"Tell me a story with more than {''.join([str(i+1)] * 5)} words"
11 for i in range(args.n_threads)]
12
13 + api_url = f"http://{args.host}:{args.port}/generate"
14 headers = {"User-Agent": "CacheFlow Benchmark Client"}
15 ploads = [{
16 "prompt": p,
@@ -19,8 +20,8 @@ def main(args: argparse.Namespace):
20 } for p in prompts]
21
22 def send_request(results, i):
22 - response = requests.post(args.api_url, headers=headers,
23 - json=ploads[i], stream=True)
23 + response = requests.post(api_url, headers=headers, json=ploads[i],
24 + stream=True)
25 results[i] = response
26
27 # use args.n_threads to prompt the backend
@@ -50,7 +51,8 @@ def main(args: argparse.Namespace):
51
52 if __name__ == "__main__":
53 parser = argparse.ArgumentParser()
53 - parser.add_argument("--api-url", type=str, default="http://localhost:8001/generate")
54 + parser.add_argument("--host", type=str, default="localhost")
55 + parser.add_argument("--port", type=int, default=8001)
56 parser.add_argument("--max-tokens", type=int, default=128)
57 parser.add_argument("--n-threads", type=int, default=128)
58 args = parser.parse_args()
benchmarks/benchmark_latency.py
+1
@@ -1,3 +1,4 @@
1 +"""Benchmark the latency of processing a single batch of requests."""
2 import argparse
3 import time
4
benchmarks/benchmark_serving.py new
+237
@@ -0,0 +1,237 @@
1 +"""Benchmark online serving throughput.
2 +
3 +On the server side, run one of the following commands:
4 + (CacheFlow backend)
5 + python -m cacheflow.entrypoints.simple_fastapi_frontend \
6 + --disable-log-requests --model <your_model>
7 +
8 + (TGI backend)
9 + ./launch_hf_server.sh <your_model>
10 +
11 +On the client side, run:
12 + python benchmarks/benchmark_serving.py \
13 + --backend <backend> \
14 + --tokenizer <your_model> --dataset <target_dataset> \
15 + --request-rate <request_rate>
16 +"""
17 +import argparse
18 +import asyncio
19 +import json
20 +import random
21 +import time
22 +from typing import AsyncGenerator, List, Tuple
23 +
24 +import aiohttp
25 +import numpy as np
26 +from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase
27 +
28 +# (prompt len, output len, latency)
29 +REQUEST_LATENCY: List[Tuple[int, int, float]] = []
30 +
31 +
32 +def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
33 + config = AutoConfig.from_pretrained(model_name)
34 + if config.model_type == "llama":
35 + # A workaround for potential protobuf errors.
36 + model_name = "hf-internal-testing/llama-tokenizer"
37 + return AutoTokenizer.from_pretrained(model_name)
38 +
39 +
40 +def sample_requests(
41 + dataset_path: str,
42 + num_requests: int,
43 + tokenizer: PreTrainedTokenizerBase,
44 +) -> List[Tuple[str, int, int]]:
45 + # Load the dataset.
46 + with open(dataset_path) as f:
47 + dataset = json.load(f)
48 + # Filter out the conversations with less than 2 turns.
49 + dataset = [
50 + data for data in dataset
51 + if len(data["conversations"]) >= 2
52 + ]
53 + # Only keep the first two turns of each conversation.
54 + dataset = [
55 + (data["conversations"][0]["value"], data["conversations"][1]["value"])
56 + for data in dataset
57 + ]
58 +
59 + # Tokenize the prompts and completions.
60 + prompts = [prompt for prompt, _ in dataset]
61 + prompt_token_ids = tokenizer(prompts).input_ids
62 + completions = [completion for _, completion in dataset]
63 + completion_token_ids = tokenizer(completions).input_ids
64 + tokenized_dataset = []
65 + for i in range(len(dataset)):
66 + output_len = len(completion_token_ids[i])
67 + tokenized_dataset.append((prompts[i], prompt_token_ids[i], output_len))
68 +
69 + # Filter out too long sequences.
70 + filtered_dataset: List[Tuple[str, int, int]] = []
71 + for prompt, prompt_token_ids, output_len in tokenized_dataset:
72 + prompt_len = len(prompt_token_ids)
73 + if prompt_len < 4 or output_len < 4:
74 + # Prune too short sequences.
75 + # This is because TGI causes errors when the input or output length
76 + # is too short.
77 + continue
78 + if prompt_len > 1024 or prompt_len + output_len > 2048:
79 + # Prune too long sequences.
80 + continue
81 + filtered_dataset.append((prompt, prompt_len, output_len))
82 +
83 + # Sample the requests.
84 + sampled_requests = random.sample(filtered_dataset, num_requests)
85 + return sampled_requests
86 +
87 +
88 +async def get_request(
89 + input_requests: List[Tuple[str, int, int]],
90 + request_rate: float,
91 +) -> AsyncGenerator[Tuple[str, int, int], None]:
92 + input_requests = iter(input_requests)
93 + for request in input_requests:
94 + yield request
95 +
96 + if request_rate == float("inf"):
97 + # If the request rate is infinity, then we don't need to wait.
98 + continue
99 + # Sample the request interval from the exponential distribution.
100 + interval = np.random.exponential(1.0 / request_rate)
101 + # The next request will be sent after the interval.
102 + await asyncio.sleep(interval)
103 +
104 +
105 +async def send_request(
106 + backend: str,
107 + api_url: str,
108 + prompt: str,
109 + prompt_len: int,
110 + output_len: int,
111 + best_of: int,
112 + use_beam_search: bool,
113 +) -> None:
114 + request_start_time = time.time()
115 +
116 + headers = {"User-Agent": "Benchmark Client"}
117 + if backend == "cacheflow":
118 + pload = {
119 + "prompt": prompt,
120 + "n": 1,
121 + "best_of": best_of,
122 + "use_beam_search": use_beam_search,
123 + "temperature": 0.0 if use_beam_search else 1.0,
124 + "top_p": 1.0,
125 + "max_tokens": output_len,
126 + "ignore_eos": True,
127 + "stream": False,
128 + }
129 + elif backend == "tgi":
130 + assert not use_beam_search
131 + params = {
132 + "best_of": best_of,
133 + "max_new_tokens": output_len,
134 + "do_sample": True,
135 + }
136 + pload = {
137 + "inputs": prompt,
138 + "parameters": params,
139 + }
140 + else:
141 + raise ValueError(f"Unknown backend: {backend}")
142 +
143 + timeout = aiohttp.ClientTimeout(total=3 * 3600)
144 + async with aiohttp.ClientSession(timeout=timeout) as session:
145 + while True:
146 + async with session.post(api_url, headers=headers, json=pload) as response:
147 + chunks = []
148 + async for chunk, _ in response.content.iter_chunks():
149 + chunks.append(chunk)
150 + output = b"".join(chunks).decode("utf-8")
151 + output = json.loads(output)
152 +
153 + # Re-send the request if it failed.
154 + if "error" not in output:
155 + break
156 +
157 + request_end_time = time.time()
158 + request_latency = request_end_time - request_start_time
159 + REQUEST_LATENCY.append((prompt_len, output_len, request_latency))
160 +
161 +
162 +async def benchmark(
163 + backend: str,
164 + api_url: str,
165 + input_requests: List[Tuple[str, int, int]],
166 + best_of: int,
167 + use_beam_search: bool,
168 + request_rate: float,
169 +) -> None:
170 + tasks: List[asyncio.Task] = []
171 + async for request in get_request(input_requests, request_rate):
172 + prompt, prompt_len, output_len = request
173 + task = asyncio.create_task(send_request(backend, api_url, prompt,
174 + prompt_len, output_len,
175 + best_of, use_beam_search))
176 + tasks.append(task)
177 + await asyncio.gather(*tasks)
178 +
179 +
180 +def main(args: argparse.Namespace):
181 + print(args)
182 + random.seed(args.seed)
183 + np.random.seed(args.seed)
184 +
185 + api_url = f"http://{args.host}:{args.port}/generate"
186 + tokenizer = get_tokenizer(args.tokenizer)
187 + input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
188 +
189 + benchmark_start_time = time.time()
190 + asyncio.run(benchmark(args.backend, api_url, input_requests, args.best_of,
191 + args.use_beam_search, args.request_rate))
192 + benchmark_end_time = time.time()
193 + benchmark_time = benchmark_end_time - benchmark_start_time
194 + print(f"Total time: {benchmark_time:.2f} s")
195 + print(f"Throughput: {args.num_prompts / benchmark_time:.2f} requests/s")
196 +
197 + # Compute the latency statistics.
198 + avg_latency = np.mean([latency for _, _, latency in REQUEST_LATENCY])
199 + print(f"Average latency: {avg_latency:.2f} s")
200 + avg_per_token_latency = np.mean([
201 + latency / (prompt_len + output_len)
202 + for prompt_len, output_len, latency in REQUEST_LATENCY
203 + ])
204 + print(f"Average latency per token: {avg_per_token_latency:.2f} s")
205 + avg_per_output_token_latency = np.mean([
206 + latency / output_len
207 + for _, output_len, latency in REQUEST_LATENCY
208 + ])
209 + print("Average latency per output token: "
210 + f"{avg_per_output_token_latency:.2f} s")
211 +
212 +
213 +if __name__ == "__main__":
214 + parser = argparse.ArgumentParser(
215 + description="Benchmark the online serving throughput.")
216 + parser.add_argument("--backend", type=str, default="cacheflow",
217 + choices=["cacheflow", "tgi"])
218 + parser.add_argument("--host", type=str, default="localhost")
219 + parser.add_argument("--port", type=int, default=8001)
220 + parser.add_argument("--dataset", type=str, required=True,
221 + help="Path to the dataset.")
222 + parser.add_argument("--tokenizer", type=str, required=True,
223 + help="Name or path of the tokenizer.")
224 + parser.add_argument("--best-of", type=int, default=1,
225 + help="Generates `best_of` sequences per prompt and "
226 + "returns the best one.")
227 + parser.add_argument("--use-beam-search", action="store_true")
228 + parser.add_argument("--num-prompts", type=int, default=1000,
229 + help="Number of prompts to process.")
230 + parser.add_argument("--request-rate", type=float, default=float("inf"),
231 + help="Number of requests per second. If this is inf, "
232 + "then all the requests are sent at time 0. "
233 + "Otherwise, we use Poisson process to synthesize "
234 + "the request arrival times.")
235 + parser.add_argument("--seed", type=int, default=0)
236 + args = parser.parse_args()
237 + main(args)
benchmarks/benchmark_text_completion.py deleted
-255
@@ -1,255 +0,0 @@
1 -import argparse
2 -import logging
3 -import os
4 -import pickle
5 -import time
6 -from typing import List
7 -
8 -from tqdm import tqdm
9 -from transformers import AutoConfig
10 -
11 -from benchmark.trace import generate_text_completion_requests
12 -from cacheflow.master.server import (
13 - add_server_arguments, process_server_arguments,
14 - init_local_server_and_frontend_with_arguments)
15 -from cacheflow.sampling_params import SamplingParams
16 -
17 -
18 -logger = logging.getLogger(__name__)
19 -
20 -
21 -def main(args: argparse.Namespace):
22 - server, frontend = init_local_server_and_frontend_with_arguments(args)
23 -
24 - # Generate requests.
25 - requests = generate_text_completion_requests(
26 - args.dataset,
27 - args.request_rate,
28 - args.duration,
29 - args.seed,
30 - args.n1,
31 - args.n2,
32 - args.n3,
33 - args.n4,
34 - args.n6,
35 - args.n2_beam,
36 - args.n4_beam,
37 - args.n6_beam,
38 - args.n8_beam,
39 - )
40 -
41 - # Warm up.
42 - logger.info('Warming up.')
43 - num_warmup_requests = 8
44 - warmup_input_len = 8
45 - warmup_output_len = 32
46 - warmup_sampling_params = SamplingParams(
47 - n=1,
48 - temperature=1.0,
49 - top_p=0.99,
50 - max_num_steps=warmup_output_len,
51 - use_beam_search=False,
52 - stop_token_ids=set(),
53 - num_logprobs=0,
54 - context_window_size=None,
55 - )
56 - for _ in range(num_warmup_requests):
57 - frontend._add_query([0] * warmup_input_len, warmup_sampling_params)
58 - server.add_sequence_groups(frontend.get_inputs())
59 - while True:
60 - server.step()
61 - if not server.has_unfinished_requests():
62 - break
63 -
64 - # Start benchmarking.
65 - logger.info('Start benchmarking.')
66 - # Initialize tqdm.
67 - pbar = tqdm(total=len(requests), desc='Finished requests')
68 -
69 - finished = []
70 - server.scheduler.reset_stats()
71 - start_time = time.time()
72 - while True:
73 - now = time.time()
74 - if args.timeout is not None and now - start_time > args.timeout:
75 - logger.info('Timeout. Stop benchmarking.')
76 - break
77 -
78 - while requests:
79 - if requests[0][0] <= now - start_time:
80 - request_time, input_tokens, sampling_params = requests.pop(0)
81 - frontend._add_query(
82 - input_tokens, sampling_params, arrival_time=start_time + request_time)
83 - else:
84 - break
85 - server.add_sequence_groups(frontend.get_inputs())
86 - updated_seq_groups = server.step()
87 -
88 - now = time.time()
89 - for seq_group in updated_seq_groups:
90 - if not seq_group.is_finished():
91 - continue
92 - arrival_time = seq_group.arrival_time
93 - finish_time = now
94 - for seq in seq_group.get_seqs():
95 - seq_len = seq.get_len()
96 - output_len = seq_len - seq.prompt_len
97 - finished.append({
98 - 'group_id': seq_group.group_id,
99 - 'seq_id': seq.seq_id,
100 - 'arrival_time': arrival_time,
101 - 'finish_time': finish_time,
102 - 'prompt_len': seq.prompt_len,
103 - 'output_len': output_len,
104 - })
105 - pbar.update(1)
106 -
107 - if not (requests or server.has_unfinished_requests()):
108 - break
109 - pbar.close()
110 - logger.info('Finish benchmarking. Saving stats.')
111 - server.scheduler.save_stats(args.output_dir)
112 - with open(os.path.join(args.output_dir, 'sequences.pkl'), 'wb') as f:
113 - pickle.dump(finished, f)
114 - logger.info('Done.')
115 -
116 -
117 -def get_model_name(model: str) -> str:
118 - OPT_MODELS = [
119 - 'opt-125m',
120 - 'opt-350m',
121 - 'opt-1.3b',
122 - 'opt-2.7b',
123 - 'opt-6.7b',
124 - 'opt-13b',
125 - 'opt-30b',
126 - 'opt-66b',
127 - 'opt-175b',
128 - ]
129 - for opt_model in OPT_MODELS:
130 - if opt_model in model:
131 - return opt_model
132 -
133 - config = AutoConfig.from_pretrained(model)
134 - assert config.model_type == 'llama'
135 - hidden_size = config.hidden_size
136 - if hidden_size == 4096:
137 - return 'llama-7b'
138 - elif hidden_size == 5120:
139 - return 'llama-13b'
140 - elif hidden_size == 6656:
141 - return 'llama-30b'
142 - elif hidden_size == 8192:
143 - return 'llama-65b'
144 - else:
145 - raise ValueError(f'Unknown model: {model}')
146 -
147 -
148 -def get_dataset_name(dataset: str) -> str:
149 - if 'sharegpt' in dataset.lower():
150 - return 'sharegpt'
151 - elif 'alpaca' in dataset.lower():
152 - return 'alpaca'
153 - else:
154 - raise ValueError(f'Unknown dataset: {dataset}')
155 -
156 -
157 -def get_sampling_dir_name(
158 - n1: float,
159 - n2: float,
160 - n3: float,
161 - n4: float,
162 - n6: float,
163 - n2_beam: float,
164 - n4_beam: float,
165 - n6_beam: float,
166 - n8_beam: float,
167 -) -> str:
168 - method = ''
169 - if n1 > 0.0:
170 - method = 'n1' if n1 == 1.0 else method + f'n1-{n1}-'
171 - if n2 > 0.0:
172 - method = 'n2' if n2 == 1.0 else method + f'n2-{n2}-'
173 - if n3 > 0.0:
174 - method = 'n3' if n3 == 1.0 else method + f'n3-{n3}-'
175 - if n4 > 0.0:
176 - method = 'n4' if n4 == 1.0 else method + f'n4-{n4}-'
177 - if n6 > 0.0:
178 - method = 'n6' if n6 == 1.0 else method + f'n6-{n6}-'
179 - if n2_beam > 0.0:
180 - method = 'n2-beam' if n2_beam == 1.0 else method + f'n2-beam-{n2_beam}-'
181 - if n4_beam > 0.0:
182 - method = 'n4-beam' if n4_beam == 1.0 else method + f'n4-beam-{n4_beam}-'
183 - if n6_beam > 0.0:
184 - method = 'n6-beam' if n6_beam == 1.0 else method + f'n6-beam-{n6_beam}-'
185 - if n8_beam > 0.0:
186 - method = 'n8-beam' if n8_beam == 1.0 else method + f'n8-beam-{n8_beam}-'
187 - return method[:-1] if method.endswith('-') else method
188 -
189 -
190 -if __name__ == '__main__':
191 - parser = argparse.ArgumentParser(
192 - description='Benchmark the performance on a series of requests.')
193 - parser = add_server_arguments(parser)
194 - parser.add_argument('--output-dir', type=str, help='path to output directory', default=None)
195 -
196 - parser.add_argument('--dataset', type=str, help='path to dataset', required=True)
197 - parser.add_argument('--request-rate', type=float, help='reqs/sec', required=True)
198 - parser.add_argument('--duration', type=int, help='duration in seconds', required=True)
199 - parser.add_argument('--do-memory-analysis', action='store_true',
200 - help='do memory analysis (This will lower the throughput. Use this only for analysis.)')
201 - parser.add_argument('--timeout', type=int, help='time out in seconds', default=None)
202 -
203 - parser.add_argument('--n1', type=float, help='ratio of requests with n=1', default=0.0)
204 - parser.add_argument('--n2', type=float, help='ratio of requests with n=2', default=0.0)
205 - parser.add_argument('--n3', type=float, help='ratio of requests with n=3', default=0.0)
206 - parser.add_argument('--n4', type=float, help='ratio of requests with n=4', default=0.0)
207 - parser.add_argument('--n6', type=float, help='ratio of requests with n=6', default=0.0)
208 - parser.add_argument('--n2-beam', type=float, help='ratio of requests with n=2 & beam search', default=0.0)
209 - parser.add_argument('--n4-beam', type=float, help='ratio of requests with n=4 & beam search', default=0.0)
210 - parser.add_argument('--n6-beam', type=float, help='ratio of requests with n=6 & beam search', default=0.0)
211 - parser.add_argument('--n8-beam', type=float, help='ratio of requests with n=8 & beam search', default=0.0)
212 - args = parser.parse_args()
213 - args = process_server_arguments(args)
214 - if args.n1 + args.n2 + args.n3 + args.n4 + args.n6 + args.n2_beam + args.n4_beam + args.n6_beam + args.n8_beam != 1.0:
215 - raise ValueError('The ratios of requests must sum to 1.')
216 -
217 - model_name = get_model_name(args.model)
218 - dataset_name = get_dataset_name(args.dataset)
219 - if 'opt' in model_name:
220 - if 'opt' not in args.dataset.lower():
221 - raise ValueError(f'OPT models can only be used with OPT datasets.')
222 - elif 'llama' in model_name:
223 - if 'llama' not in args.dataset.lower():
224 - raise ValueError(f'Llama models can only be used with Llama datasets.')
225 -
226 - dataset_name = 'sharegpt' if 'sharegpt' in args.dataset else 'alpaca'
227 - sample_dir = get_sampling_dir_name(
228 - args.n1, args.n2, args.n3, args.n4, args.n6, args.n2_beam, args.n4_beam, args.n6_beam, args.n8_beam)
229 - if args.output_dir is None:
230 - args.output_dir = os.path.join(
231 - '../exp',
232 - dataset_name,
233 - f'{model_name}-tp{args.tensor_parallel_size}',
234 - sample_dir,
235 - 'cacheflow',
236 - f'block{args.block_size}',
237 - f'req-rate-{args.request_rate}',
238 - f'seed{args.seed}',
239 - f'duration-{args.duration}',
240 - )
241 - os.makedirs(args.output_dir, exist_ok=True)
242 -
243 - # Set up logging.
244 - logging.basicConfig(
245 - format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
246 - datefmt="%m/%d/%Y %H:%M:%S",
247 - level=logging.INFO,
248 - handlers=[
249 - logging.StreamHandler(),
250 - logging.FileHandler(os.path.join(args.output_dir, 'log.txt')),
251 - ],
252 - )
253 - logger.info(args)
254 -
255 - main(args)
benchmarks/benchmark_throughput.py
+138 -28
@@ -1,3 +1,4 @@
1 +"""Benchmark offline inference throughput."""
2 import argparse
3 import json
4 import random
@@ -5,14 +6,29 @@ import time
6 from typing import List, Tuple
7
8 from cacheflow import LLM, SamplingParams
8 -from transformers import PreTrainedTokenizerBase
9 +import torch
10 +from transformers import (AutoConfig, AutoTokenizer, AutoModelForCausalLM,
11 + PreTrainedTokenizerBase)
12 +from tqdm import tqdm
13 +
14 +
15 +def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
16 + config = AutoConfig.from_pretrained(model_name)
17 + if config.model_type == "llama":
18 + # A workaround for potential protobuf errors.
19 + model_name = "hf-internal-testing/llama-tokenizer"
20 + tokenizer = AutoTokenizer.from_pretrained(model_name)
21 + # To enable padding in the HF backend.
22 + tokenizer.pad_token = tokenizer.eos_token
23 + return tokenizer
24 + return AutoTokenizer.from_pretrained(model_name)
25
26
27 def sample_requests(
28 dataset_path: str,
29 num_requests: int,
30 tokenizer: PreTrainedTokenizerBase,
15 -) -> List[Tuple[List[int], int]]:
31 +) -> List[Tuple[str, int, int]]:
32 # Load the dataset.
33 with open(dataset_path) as f:
34 dataset = json.load(f)
@@ -35,45 +51,52 @@ def sample_requests(
51 tokenized_dataset = []
52 for i in range(len(dataset)):
53 output_len = len(completion_token_ids[i])
38 - tokenized_dataset.append((prompt_token_ids[i], output_len))
39 - # Filter out if the prompt length + output length is greater than 2048.
40 - tokenized_dataset = [
41 - (prompt_token_ids, output_len)
42 - for prompt_token_ids, output_len in tokenized_dataset
43 - if len(prompt_token_ids) + output_len <= 2048
44 - ]
54 + tokenized_dataset.append((prompts[i], prompt_token_ids[i], output_len))
55 +
56 + # Filter out too long sequences.
57 + filtered_dataset: List[Tuple[str, int, int]] = []
58 + for prompt, prompt_token_ids, output_len in tokenized_dataset:
59 + prompt_len = len(prompt_token_ids)
60 + if prompt_len < 4 or output_len < 4:
61 + # Prune too short sequences.
62 + continue
63 + if prompt_len > 1024 or prompt_len + output_len > 2048:
64 + # Prune too long sequences.
65 + continue
66 + filtered_dataset.append((prompt, prompt_len, output_len))
67
68 # Sample the requests.
47 - sampled_requests = random.sample(tokenized_dataset, num_requests)
69 + sampled_requests = random.sample(filtered_dataset, num_requests)
70 return sampled_requests
71
72
51 -def main(args: argparse.Namespace):
52 - print(args)
53 - random.seed(args.seed)
54 -
73 +def run_cacheflow(
74 + requests: List[Tuple[str, int, int]],
75 + model: str,
76 + tensor_parallel_size: int,
77 + seed: int,
78 + n: int,
79 + use_beam_search: bool,
80 +) -> float:
81 llm = LLM(
56 - model=args.model,
57 - tensor_parallel_size=args.tensor_parallel_size,
58 - seed=args.seed,
82 + model=model,
83 + tensor_parallel_size=tensor_parallel_size,
84 + seed=seed,
85 )
60 - tokenizer = llm.get_tokenizer()
61 - requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
86
87 # Add the requests to the server.
64 - for prompt_token_ids, output_len in requests:
88 + for prompt, _, output_len in requests:
89 sampling_params = SamplingParams(
66 - n=args.n,
67 - temperature=0.0 if args.use_beam_search else 1.0,
90 + n=n,
91 + temperature=0.0 if use_beam_search else 1.0,
92 top_p=1.0,
69 - use_beam_search=args.use_beam_search,
93 + use_beam_search=use_beam_search,
94 ignore_eos=True,
95 max_tokens=output_len,
96 )
97 # FIXME(woosuk): Do not use internal method.
98 llm._add_request(
75 - prompt=None,
76 - prompt_token_ids=prompt_token_ids,
99 + prompt=prompt,
100 sampling_params=sampling_params,
101 )
102
@@ -81,17 +104,95 @@ def main(args: argparse.Namespace):
104 # FIXME(woosuk): Do use internal method.
105 llm._run_server(use_tqdm=True)
106 end = time.time()
107 + return end - start
108 +
109 +
110 +def run_hf(
111 + requests: List[Tuple[str, int, int]],
112 + model: str,
113 + tokenizer: PreTrainedTokenizerBase,
114 + n: int,
115 + use_beam_search: bool,
116 + max_batch_size: int,
117 +) -> float:
118 + assert not use_beam_search
119 + tokenizer = get_tokenizer(model)
120 + llm = AutoModelForCausalLM.from_pretrained(
121 + model, torch_dtype=torch.float16)
122 + llm = llm.cuda()
123 +
124 + pbar = tqdm(total=len(requests))
125 + start = time.time()
126 + batch: List[str] = []
127 + max_prompt_len = 0
128 + max_output_len = 0
129 + for i in range(len(requests)):
130 + prompt, prompt_len, output_len = requests[i]
131 + # Add the prompt to the batch.
132 + batch.append(prompt)
133 + max_prompt_len = max(max_prompt_len, prompt_len)
134 + max_output_len = max(max_output_len, output_len)
135 + if len(batch) < max_batch_size and i != len(requests) - 1:
136 + # Check if we can add more requests to the batch.
137 + _, next_prompt_len, next_output_len = requests[i + 1]
138 + if (max(max_prompt_len, next_prompt_len) + max(
139 + max_output_len, next_output_len)) <= 2048:
140 + # We can add more requests to the batch.
141 + continue
142 +
143 + # Generate the sequences.
144 + input_ids = tokenizer(batch, return_tensors="pt", padding=True).input_ids
145 + llm_outputs = llm.generate(
146 + input_ids=input_ids.cuda(),
147 + do_sample=not use_beam_search,
148 + num_return_sequences=n,
149 + temperature=1.0,
150 + top_p=1.0,
151 + use_cache=True,
152 + max_new_tokens=max_output_len,
153 + )
154 + # Include the decoding time.
155 + tokenizer.batch_decode(llm_outputs, skip_special_tokens=True)
156 + pbar.update(len(batch))
157 +
158 + # Clear the batch.
159 + batch = []
160 + max_prompt_len = 0
161 + max_output_len = 0
162 + end = time.time()
163 + return end - start
164 +
165 +
166 +def main(args: argparse.Namespace):
167 + print(args)
168 + random.seed(args.seed)
169 +
170 + # Sample the requests.
171 + tokenizer = get_tokenizer(args.model)
172 + requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
173 +
174 + if args.backend == "cacheflow":
175 + elapsed_time = run_cacheflow(
176 + requests, args.model, args.tensor_parallel_size, args.seed, args.n,
177 + args.use_beam_search)
178 + elif args.backend == "hf":
179 + assert args.tensor_parallel_size == 1
180 + elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
181 + args.use_beam_search, args.hf_max_batch_size)
182 + else:
183 + raise ValueError(f"Unknown backend: {args.backend}")
184 total_num_tokens = sum(
85 - len(prompt_token_ids) + output_len
86 - for prompt_token_ids, output_len in requests
185 + prompt_len + output_len
186 + for _, prompt_len, output_len in requests
187 )
88 - elapsed_time = end - start
188 print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
189 f"{total_num_tokens / elapsed_time:.2f} tokens/s")
190
191
192 if __name__ == "__main__":
193 parser = argparse.ArgumentParser(description="Benchmark the throughput.")
194 + parser.add_argument("--backend", type=str, choices=["cacheflow", "hf"],
195 + default="cacheflow")
196 parser.add_argument("--dataset", type=str, required=True,
197 help="Path to the dataset.")
198 parser.add_argument("--model", type=str, default="facebook/opt-125m")
@@ -102,5 +203,14 @@ if __name__ == "__main__":
203 parser.add_argument("--num-prompts", type=int, default=1000,
204 help="Number of prompts to process.")
205 parser.add_argument("--seed", type=int, default=0)
206 + parser.add_argument("--hf-max-batch-size", type=int, default=None,
207 + help="Maximum batch size for HF backend.")
208 args = parser.parse_args()
209 + if args.backend == "cacheflow":
210 + if args.hf_max_batch_size is not None:
211 + raise ValueError("HF max batch size is only for HF backend.")
212 + elif args.backend == "hf":
213 + if args.hf_max_batch_size is None:
214 + raise ValueError("HF max batch size is required for HF backend.")
215 +
216 main(args)
benchmarks/launch_tgi_server.sh new
+16
@@ -0,0 +1,16 @@
1 +#!/bin/bash
2 +
3 +PORT=8001
4 +MODEL=$1
5 +TOKENS=$2
6 +
7 +docker run --gpus all --shm-size 1g -p $PORT:80 \
8 + -v $PWD/data:/data \
9 + ghcr.io/huggingface/text-generation-inference:0.8 \
10 + --model-id $MODEL \
11 + --sharded false \
12 + --max-input-length 1024 \
13 + --max-total-tokens 2048 \
14 + --max-best-of 5 \
15 + --max-concurrent-requests 5000 \
16 + --max-batch-total-tokens $TOKENS
benchmarks/trace.py deleted
-116
@@ -1,116 +0,0 @@
1 -import pickle
2 -import random
3 -from typing import List, Tuple
4 -
5 -import numpy as np
6 -
7 -from cacheflow.sampling_params import SamplingParams
8 -
9 -
10 -def generate_text_completion_requests(
11 - dataset: str,
12 - request_rate: float,
13 - duration: int,
14 - seed: int,
15 - n1: float = 0.0,
16 - n2: float = 0.0,
17 - n3: float = 0.0,
18 - n4: float = 0.0,
19 - n6: float = 0.0,
20 - n2_beam: float = 0.0,
21 - n4_beam: float = 0.0,
22 - n6_beam: float = 0.0,
23 - n8_beam: float = 0.0,
24 - max_seq_len: int = 2048,
25 - time_quantum: int = 10,
26 -) -> List[Tuple[float, List[int], SamplingParams]]:
27 - random.seed(seed)
28 - np.random.seed(seed)
29 -
30 - # Generate timestamps for requests using Poisson distribution.
31 - lam = request_rate * (time_quantum / 1000)
32 - quantums_per_sec = 1000 / time_quantum
33 - arrival_times = np.random.poisson(
34 - lam=lam, size=int(duration * quantums_per_sec))
35 - timestamps = []
36 - for i, n in enumerate(arrival_times):
37 - timestamps += [i * (time_quantum / 1000)] * n
38 -
39 - # Load and shuffle the dataset.
40 - num_requests = len(timestamps)
41 - with open(dataset, 'rb') as f:
42 - data = pickle.load(f)
43 -
44 - filtered = []
45 - for pair in data:
46 - input_tokens, output_tokens = pair
47 - input_len = len(input_tokens)
48 - output_len = len(output_tokens)
49 - # Filter out too long sequences.
50 - if input_len + output_len < max_seq_len:
51 - # Output tokens are not needed for the benchmark.
52 - filtered.append((input_tokens, output_len))
53 -
54 - data = []
55 - while len(data) < num_requests:
56 - data += filtered
57 - data = data[:num_requests]
58 - # Shuffle the data.
59 - assert len(data) == len(timestamps)
60 - random.shuffle(data)
61 -
62 - random_sampling_params_dict = {
63 - 'temperature': 1.0,
64 - 'top_p': 1.0,
65 - 'use_beam_search': False,
66 - 'stop_token_ids': set(),
67 - 'num_logprobs': 0,
68 - 'context_window_size': None,
69 - }
70 - beam_search_params_dict = {
71 - 'temperature': 0.0,
72 - 'top_p': 1.0,
73 - 'use_beam_search': True,
74 - 'stop_token_ids': set(),
75 - 'num_logprobs': 0,
76 - 'context_window_size': None,
77 - }
78 -
79 - # Generate requests based on the sampling parameter ratio.
80 - requests = []
81 - assert n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam + n8_beam == 1.0
82 - cum_sum = 0
83 - for timestamp, pair in zip(timestamps, data):
84 - input_tokens, output_len = pair
85 - if cum_sum < n1 * num_requests:
86 - sampling_params = SamplingParams(
87 - n=1, max_num_steps=output_len, **random_sampling_params_dict)
88 - elif cum_sum < (n1 + n2) * num_requests:
89 - sampling_params = SamplingParams(
90 - n=2, max_num_steps=output_len, **random_sampling_params_dict)
91 - elif cum_sum < (n1 + n2 + n3) * num_requests:
92 - sampling_params = SamplingParams(
93 - n=3, max_num_steps=output_len, **random_sampling_params_dict)
94 - elif cum_sum < (n1 + n2 + n3 + n4) * num_requests:
95 - sampling_params = SamplingParams(
96 - n=4, max_num_steps=output_len, **random_sampling_params_dict)
97 - elif cum_sum < (n1 + n2 + n3 + n4 + n6) * num_requests:
98 - sampling_params = SamplingParams(
99 - n=6, max_num_steps=output_len, **random_sampling_params_dict)
100 - elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam) * num_requests:
101 - sampling_params = SamplingParams(
102 - n=2, max_num_steps=output_len, **beam_search_params_dict)
103 - elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam) * num_requests:
104 - sampling_params = SamplingParams(
105 - n=4, max_num_steps=output_len, **beam_search_params_dict)
106 - elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam) * num_requests:
107 - sampling_params = SamplingParams(
108 - n=6, max_num_steps=output_len, **beam_search_params_dict)
109 - elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam + n8_beam) * num_requests:
110 - sampling_params = SamplingParams(
111 - n=8, max_num_steps=output_len, **beam_search_params_dict)
112 - else:
113 - raise ValueError('Invalid request ratio.')
114 - cum_sum += 1
115 - requests.append((timestamp, input_tokens, sampling_params))
116 - return requests
cacheflow/server/arg_utils.py
+3
@@ -120,6 +120,7 @@ class ServerArgs:
120 class AsyncServerArgs(ServerArgs):
121 """Arguments for asynchronous CacheFlow servers."""
122 server_use_ray: bool = False
123 + disable_log_requests: bool = False
124
125 @staticmethod
126 def add_cli_args(
@@ -129,4 +130,6 @@ class AsyncServerArgs(ServerArgs):
130 parser.add_argument('--server-use-ray', action='store_true',
131 help='use Ray to start the LLM server in a '
132 'separate process as the web server process.')
133 + parser.add_argument('--disable-log-requests', action='store_true',
134 + help='disable logging requests')
135 return parser
cacheflow/server/async_llm_server.py
+13 -7
@@ -32,12 +32,14 @@ class AsyncLLMServer:
32 server_use_ray: Whether to make LLMServer a Ray actor. If so, the
33 async frontend will be executed in a separate process as the
34 model workers.
35 + log_requests: Whether to log the requests.
36 *args, *kwargs: Arguments for LLMServer.
37 """
38 def __init__(self, worker_use_ray: bool, server_use_ray: bool,
38 - *args, **kwargs) -> None:
39 + log_requests: bool = True, *args, **kwargs) -> None:
40 self.worker_use_ray = worker_use_ray
41 self.server_use_ray = server_use_ray
42 + self.log_requests = log_requests
43 if not self.server_use_ray:
44 server_class = LLMServer
45 elif self.worker_use_ray:
@@ -106,10 +108,11 @@ class AsyncLLMServer:
108 request_event = asyncio.Event()
109 self.request_events[request_id] = request_event
110
109 - logger.info(f"Received request {request_id}: "
110 - f"prompt: {prompt!r}, "
111 - f"sampling params: {sampling_params}, "
112 - f"prompt token ids: {prompt_token_ids}.")
111 + if self.log_requests:
112 + logger.info(f"Received request {request_id}: "
113 + f"prompt: {prompt!r}, "
114 + f"sampling params: {sampling_params}, "
115 + f"prompt token ids: {prompt_token_ids}.")
116
117 # Add the request into the cacheflow server's waiting queue.
118 if self.server_use_ray:
@@ -152,7 +155,8 @@ class AsyncLLMServer:
155
156 # Once finished, release the resources of the sequence group.
157 if request_output.finished():
155 - logger.info(f"Finished request {request_id}.")
158 + if self.log_requests:
159 + logger.info(f"Finished request {request_id}.")
160
161 del self.request_outputs[request_id]
162 del self.request_events[request_id]
@@ -176,7 +180,8 @@ class AsyncLLMServer:
180 # The request has already finished or been aborted.
181 return
182
179 - logger.info(f"Aborted request {request_id}.")
183 + if self.log_requests:
184 + logger.info(f"Aborted request {request_id}.")
185
186 if self.server_use_ray:
187 await self.server.abort_request.remote(request_id)
@@ -206,6 +211,7 @@ class AsyncLLMServer:
211 # Create the LLM server.
212 server = cls(server_args.worker_use_ray,
213 server_args.server_use_ray,
214 + not server_args.disable_log_requests,
215 *server_configs,
216 distributed_init_method, devices,
217 log_stats=not server_args.disable_log_stats)
examples/simple_fastapi_client.py
+8 -6
@@ -1,12 +1,14 @@
1 import argparse
2 import json
3 -import requests
3 from typing import Iterable, List
4
6 -def clear_line(n:int = 1) -> None:
5 +import requests
6 +
7 +
8 +def clear_line(n: int = 1) -> None:
9 LINE_UP = '\033[1A'
10 LINE_CLEAR = '\x1b[2K'
9 - for i in range(n):
11 + for _ in range(n):
12 print(LINE_UP, end=LINE_CLEAR, flush=True)
13
14
@@ -53,7 +55,7 @@ if __name__ == "__main__":
55 n = args.n
56 stream = args.stream
57
56 - print(f"Prompt: {prompt}\n", flush=True)
58 + print(f"Prompt: {prompt!r}\n", flush=True)
59 response = post_http_request(prompt, api_url, n, stream)
60
61 if stream:
@@ -63,8 +65,8 @@ if __name__ == "__main__":
65 num_printed_lines = 0
66 for i, line in enumerate(h):
67 num_printed_lines += 1
66 - print(f"Beam candidate {i}: {line}", flush=True)
68 + print(f"Beam candidate {i}: {line!r}", flush=True)
69 else:
70 output = get_response(response)
71 for i, line in enumerate(output):
70 - print(f"Beam candidate {i}: {line}", flush=True)
72 + print(f"Beam candidate {i}: {line!r}", flush=True)