A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add script for benchmarking serving throughput (#145)
Woosuk Kwon committed
Jun 14, 2023 at 19:55 UTC
311490a720951f322977d811eacea685a623b5a1
10 files changed
+421
-415
benchmarks/benchmark_async_llm_server.py
+5
-3
@@ -10,6 +10,7 @@ def main(args: argparse.Namespace):
10
prompts = [f"Tell me a story with more than {''.join([str(i+1)] * 5)} words"
11
for i in range(args.n_threads)]
12
13
+ api_url = f"http://{args.host}:{args.port}/generate"
14
headers = {"User-Agent": "CacheFlow Benchmark Client"}
15
ploads = [{
16
"prompt": p,
@@ -19,8 +20,8 @@ def main(args: argparse.Namespace):
20
} for p in prompts]
21
22
def send_request(results, i):
22
- response = requests.post(args.api_url, headers=headers,
23
- json=ploads[i], stream=True)
23
+ response = requests.post(api_url, headers=headers, json=ploads[i],
24
+ stream=True)
25
results[i] = response
26
27
# use args.n_threads to prompt the backend
@@ -50,7 +51,8 @@ def main(args: argparse.Namespace):
51
52
if __name__ == "__main__":
53
parser = argparse.ArgumentParser()
53
- parser.add_argument("--api-url", type=str, default="http://localhost:8001/generate")
54
+ parser.add_argument("--host", type=str, default="localhost")
55
+ parser.add_argument("--port", type=int, default=8001)
56
parser.add_argument("--max-tokens", type=int, default=128)
57
parser.add_argument("--n-threads", type=int, default=128)
58
args = parser.parse_args()
benchmarks/benchmark_latency.py
+1
@@ -1,3 +1,4 @@
1
+"""Benchmark the latency of processing a single batch of requests."""
2
import argparse
3
import time
4
benchmarks/benchmark_serving.py
new
+237
@@ -0,0 +1,237 @@
1
+"""Benchmark online serving throughput.
2
+
3
+On the server side, run one of the following commands:
4
+ (CacheFlow backend)
5
+ python -m cacheflow.entrypoints.simple_fastapi_frontend \
6
+ --disable-log-requests --model <your_model>
7
+
8
+ (TGI backend)
9
+ ./launch_hf_server.sh <your_model>
10
+
11
+On the client side, run:
12
+ python benchmarks/benchmark_serving.py \
13
+ --backend <backend> \
14
+ --tokenizer <your_model> --dataset <target_dataset> \
15
+ --request-rate <request_rate>
16
+"""
17
+import argparse
18
+import asyncio
19
+import json
20
+import random
21
+import time
22
+from typing import AsyncGenerator, List, Tuple
23
+
24
+import aiohttp
25
+import numpy as np
26
+from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase
27
+
28
+# (prompt len, output len, latency)
29
+REQUEST_LATENCY: List[Tuple[int, int, float]] = []
30
+
31
+
32
+def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
33
+ config = AutoConfig.from_pretrained(model_name)
34
+ if config.model_type == "llama":
35
+ # A workaround for potential protobuf errors.
36
+ model_name = "hf-internal-testing/llama-tokenizer"
37
+ return AutoTokenizer.from_pretrained(model_name)
38
+
39
+
40
+def sample_requests(
41
+ dataset_path: str,
42
+ num_requests: int,
43
+ tokenizer: PreTrainedTokenizerBase,
44
+) -> List[Tuple[str, int, int]]:
45
+ # Load the dataset.
46
+ with open(dataset_path) as f:
47
+ dataset = json.load(f)
48
+ # Filter out the conversations with less than 2 turns.
49
+ dataset = [
50
+ data for data in dataset
51
+ if len(data["conversations"]) >= 2
52
+ ]
53
+ # Only keep the first two turns of each conversation.
54
+ dataset = [
55
+ (data["conversations"][0]["value"], data["conversations"][1]["value"])
56
+ for data in dataset
57
+ ]
58
+
59
+ # Tokenize the prompts and completions.
60
+ prompts = [prompt for prompt, _ in dataset]
61
+ prompt_token_ids = tokenizer(prompts).input_ids
62
+ completions = [completion for _, completion in dataset]
63
+ completion_token_ids = tokenizer(completions).input_ids
64
+ tokenized_dataset = []
65
+ for i in range(len(dataset)):
66
+ output_len = len(completion_token_ids[i])
67
+ tokenized_dataset.append((prompts[i], prompt_token_ids[i], output_len))
68
+
69
+ # Filter out too long sequences.
70
+ filtered_dataset: List[Tuple[str, int, int]] = []
71
+ for prompt, prompt_token_ids, output_len in tokenized_dataset:
72
+ prompt_len = len(prompt_token_ids)
73
+ if prompt_len < 4 or output_len < 4:
74
+ # Prune too short sequences.
75
+ # This is because TGI causes errors when the input or output length
76
+ # is too short.
77
+ continue
78
+ if prompt_len > 1024 or prompt_len + output_len > 2048:
79
+ # Prune too long sequences.
80
+ continue
81
+ filtered_dataset.append((prompt, prompt_len, output_len))
82
+
83
+ # Sample the requests.
84
+ sampled_requests = random.sample(filtered_dataset, num_requests)
85
+ return sampled_requests
86
+
87
+
88
+async def get_request(
89
+ input_requests: List[Tuple[str, int, int]],
90
+ request_rate: float,
91
+) -> AsyncGenerator[Tuple[str, int, int], None]:
92
+ input_requests = iter(input_requests)
93
+ for request in input_requests:
94
+ yield request
95
+
96
+ if request_rate == float("inf"):
97
+ # If the request rate is infinity, then we don't need to wait.
98
+ continue
99
+ # Sample the request interval from the exponential distribution.
100
+ interval = np.random.exponential(1.0 / request_rate)
101
+ # The next request will be sent after the interval.
102
+ await asyncio.sleep(interval)
103
+
104
+
105
+async def send_request(
106
+ backend: str,
107
+ api_url: str,
108
+ prompt: str,
109
+ prompt_len: int,
110
+ output_len: int,
111
+ best_of: int,
112
+ use_beam_search: bool,
113
+) -> None:
114
+ request_start_time = time.time()
115
+
116
+ headers = {"User-Agent": "Benchmark Client"}
117
+ if backend == "cacheflow":
118
+ pload = {
119
+ "prompt": prompt,
120
+ "n": 1,
121
+ "best_of": best_of,
122
+ "use_beam_search": use_beam_search,
123
+ "temperature": 0.0 if use_beam_search else 1.0,
124
+ "top_p": 1.0,
125
+ "max_tokens": output_len,
126
+ "ignore_eos": True,
127
+ "stream": False,
128
+ }
129
+ elif backend == "tgi":
130
+ assert not use_beam_search
131
+ params = {
132
+ "best_of": best_of,
133
+ "max_new_tokens": output_len,
134
+ "do_sample": True,
135
+ }
136
+ pload = {
137
+ "inputs": prompt,
138
+ "parameters": params,
139
+ }
140
+ else:
141
+ raise ValueError(f"Unknown backend: {backend}")
142
+
143
+ timeout = aiohttp.ClientTimeout(total=3 * 3600)
144
+ async with aiohttp.ClientSession(timeout=timeout) as session:
145
+ while True:
146
+ async with session.post(api_url, headers=headers, json=pload) as response:
147
+ chunks = []
148
+ async for chunk, _ in response.content.iter_chunks():
149
+ chunks.append(chunk)
150
+ output = b"".join(chunks).decode("utf-8")
151
+ output = json.loads(output)
152
+
153
+ # Re-send the request if it failed.
154
+ if "error" not in output:
155
+ break
156
+
157
+ request_end_time = time.time()
158
+ request_latency = request_end_time - request_start_time
159
+ REQUEST_LATENCY.append((prompt_len, output_len, request_latency))
160
+
161
+
162
+async def benchmark(
163
+ backend: str,
164
+ api_url: str,
165
+ input_requests: List[Tuple[str, int, int]],
166
+ best_of: int,
167
+ use_beam_search: bool,
168
+ request_rate: float,
169
+) -> None:
170
+ tasks: List[asyncio.Task] = []
171
+ async for request in get_request(input_requests, request_rate):
172
+ prompt, prompt_len, output_len = request
173
+ task = asyncio.create_task(send_request(backend, api_url, prompt,
174
+ prompt_len, output_len,
175
+ best_of, use_beam_search))
176
+ tasks.append(task)
177
+ await asyncio.gather(*tasks)
178
+
179
+
180
+def main(args: argparse.Namespace):
181
+ print(args)
182
+ random.seed(args.seed)
183
+ np.random.seed(args.seed)
184
+
185
+ api_url = f"http://{args.host}:{args.port}/generate"
186
+ tokenizer = get_tokenizer(args.tokenizer)
187
+ input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
188
+
189
+ benchmark_start_time = time.time()
190
+ asyncio.run(benchmark(args.backend, api_url, input_requests, args.best_of,
191
+ args.use_beam_search, args.request_rate))
192
+ benchmark_end_time = time.time()
193
+ benchmark_time = benchmark_end_time - benchmark_start_time
194
+ print(f"Total time: {benchmark_time:.2f} s")
195
+ print(f"Throughput: {args.num_prompts / benchmark_time:.2f} requests/s")
196
+
197
+ # Compute the latency statistics.
198
+ avg_latency = np.mean([latency for _, _, latency in REQUEST_LATENCY])
199
+ print(f"Average latency: {avg_latency:.2f} s")
200
+ avg_per_token_latency = np.mean([
201
+ latency / (prompt_len + output_len)
202
+ for prompt_len, output_len, latency in REQUEST_LATENCY
203
+ ])
204
+ print(f"Average latency per token: {avg_per_token_latency:.2f} s")
205
+ avg_per_output_token_latency = np.mean([
206
+ latency / output_len
207
+ for _, output_len, latency in REQUEST_LATENCY
208
+ ])
209
+ print("Average latency per output token: "
210
+ f"{avg_per_output_token_latency:.2f} s")
211
+
212
+
213
+if __name__ == "__main__":
214
+ parser = argparse.ArgumentParser(
215
+ description="Benchmark the online serving throughput.")
216
+ parser.add_argument("--backend", type=str, default="cacheflow",
217
+ choices=["cacheflow", "tgi"])
218
+ parser.add_argument("--host", type=str, default="localhost")
219
+ parser.add_argument("--port", type=int, default=8001)
220
+ parser.add_argument("--dataset", type=str, required=True,
221
+ help="Path to the dataset.")
222
+ parser.add_argument("--tokenizer", type=str, required=True,
223
+ help="Name or path of the tokenizer.")
224
+ parser.add_argument("--best-of", type=int, default=1,
225
+ help="Generates `best_of` sequences per prompt and "
226
+ "returns the best one.")
227
+ parser.add_argument("--use-beam-search", action="store_true")
228
+ parser.add_argument("--num-prompts", type=int, default=1000,
229
+ help="Number of prompts to process.")
230
+ parser.add_argument("--request-rate", type=float, default=float("inf"),
231
+ help="Number of requests per second. If this is inf, "
232
+ "then all the requests are sent at time 0. "
233
+ "Otherwise, we use Poisson process to synthesize "
234
+ "the request arrival times.")
235
+ parser.add_argument("--seed", type=int, default=0)
236
+ args = parser.parse_args()
237
+ main(args)
benchmarks/benchmark_text_completion.py
deleted
-255
@@ -1,255 +0,0 @@
1
-import argparse
2
-import logging
3
-import os
4
-import pickle
5
-import time
6
-from typing import List
7
-
8
-from tqdm import tqdm
9
-from transformers import AutoConfig
10
-
11
-from benchmark.trace import generate_text_completion_requests
12
-from cacheflow.master.server import (
13
- add_server_arguments, process_server_arguments,
14
- init_local_server_and_frontend_with_arguments)
15
-from cacheflow.sampling_params import SamplingParams
16
-
17
-
18
-logger = logging.getLogger(__name__)
19
-
20
-
21
-def main(args: argparse.Namespace):
22
- server, frontend = init_local_server_and_frontend_with_arguments(args)
23
-
24
- # Generate requests.
25
- requests = generate_text_completion_requests(
26
- args.dataset,
27
- args.request_rate,
28
- args.duration,
29
- args.seed,
30
- args.n1,
31
- args.n2,
32
- args.n3,
33
- args.n4,
34
- args.n6,
35
- args.n2_beam,
36
- args.n4_beam,
37
- args.n6_beam,
38
- args.n8_beam,
39
- )
40
-
41
- # Warm up.
42
- logger.info('Warming up.')
43
- num_warmup_requests = 8
44
- warmup_input_len = 8
45
- warmup_output_len = 32
46
- warmup_sampling_params = SamplingParams(
47
- n=1,
48
- temperature=1.0,
49
- top_p=0.99,
50
- max_num_steps=warmup_output_len,
51
- use_beam_search=False,
52
- stop_token_ids=set(),
53
- num_logprobs=0,
54
- context_window_size=None,
55
- )
56
- for _ in range(num_warmup_requests):
57
- frontend._add_query([0] * warmup_input_len, warmup_sampling_params)
58
- server.add_sequence_groups(frontend.get_inputs())
59
- while True:
60
- server.step()
61
- if not server.has_unfinished_requests():
62
- break
63
-
64
- # Start benchmarking.
65
- logger.info('Start benchmarking.')
66
- # Initialize tqdm.
67
- pbar = tqdm(total=len(requests), desc='Finished requests')
68
-
69
- finished = []
70
- server.scheduler.reset_stats()
71
- start_time = time.time()
72
- while True:
73
- now = time.time()
74
- if args.timeout is not None and now - start_time > args.timeout:
75
- logger.info('Timeout. Stop benchmarking.')
76
- break
77
-
78
- while requests:
79
- if requests[0][0] <= now - start_time:
80
- request_time, input_tokens, sampling_params = requests.pop(0)
81
- frontend._add_query(
82
- input_tokens, sampling_params, arrival_time=start_time + request_time)
83
- else:
84
- break
85
- server.add_sequence_groups(frontend.get_inputs())
86
- updated_seq_groups = server.step()
87
-
88
- now = time.time()
89
- for seq_group in updated_seq_groups:
90
- if not seq_group.is_finished():
91
- continue
92
- arrival_time = seq_group.arrival_time
93
- finish_time = now
94
- for seq in seq_group.get_seqs():
95
- seq_len = seq.get_len()
96
- output_len = seq_len - seq.prompt_len
97
- finished.append({
98
- 'group_id': seq_group.group_id,
99
- 'seq_id': seq.seq_id,
100
- 'arrival_time': arrival_time,
101
- 'finish_time': finish_time,
102
- 'prompt_len': seq.prompt_len,
103
- 'output_len': output_len,
104
- })
105
- pbar.update(1)
106
-
107
- if not (requests or server.has_unfinished_requests()):
108
- break
109
- pbar.close()
110
- logger.info('Finish benchmarking. Saving stats.')
111
- server.scheduler.save_stats(args.output_dir)
112
- with open(os.path.join(args.output_dir, 'sequences.pkl'), 'wb') as f:
113
- pickle.dump(finished, f)
114
- logger.info('Done.')
115
-
116
-
117
-def get_model_name(model: str) -> str:
118
- OPT_MODELS = [
119
- 'opt-125m',
120
- 'opt-350m',
121
- 'opt-1.3b',
122
- 'opt-2.7b',
123
- 'opt-6.7b',
124
- 'opt-13b',
125
- 'opt-30b',
126
- 'opt-66b',
127
- 'opt-175b',
128
- ]
129
- for opt_model in OPT_MODELS:
130
- if opt_model in model:
131
- return opt_model
132
-
133
- config = AutoConfig.from_pretrained(model)
134
- assert config.model_type == 'llama'
135
- hidden_size = config.hidden_size
136
- if hidden_size == 4096:
137
- return 'llama-7b'
138
- elif hidden_size == 5120:
139
- return 'llama-13b'
140
- elif hidden_size == 6656:
141
- return 'llama-30b'
142
- elif hidden_size == 8192:
143
- return 'llama-65b'
144
- else:
145
- raise ValueError(f'Unknown model: {model}')
146
-
147
-
148
-def get_dataset_name(dataset: str) -> str:
149
- if 'sharegpt' in dataset.lower():
150
- return 'sharegpt'
151
- elif 'alpaca' in dataset.lower():
152
- return 'alpaca'
153
- else:
154
- raise ValueError(f'Unknown dataset: {dataset}')
155
-
156
-
157
-def get_sampling_dir_name(
158
- n1: float,
159
- n2: float,
160
- n3: float,
161
- n4: float,
162
- n6: float,
163
- n2_beam: float,
164
- n4_beam: float,
165
- n6_beam: float,
166
- n8_beam: float,
167
-) -> str:
168
- method = ''
169
- if n1 > 0.0:
170
- method = 'n1' if n1 == 1.0 else method + f'n1-{n1}-'
171
- if n2 > 0.0:
172
- method = 'n2' if n2 == 1.0 else method + f'n2-{n2}-'
173
- if n3 > 0.0:
174
- method = 'n3' if n3 == 1.0 else method + f'n3-{n3}-'
175
- if n4 > 0.0:
176
- method = 'n4' if n4 == 1.0 else method + f'n4-{n4}-'
177
- if n6 > 0.0:
178
- method = 'n6' if n6 == 1.0 else method + f'n6-{n6}-'
179
- if n2_beam > 0.0:
180
- method = 'n2-beam' if n2_beam == 1.0 else method + f'n2-beam-{n2_beam}-'
181
- if n4_beam > 0.0:
182
- method = 'n4-beam' if n4_beam == 1.0 else method + f'n4-beam-{n4_beam}-'
183
- if n6_beam > 0.0:
184
- method = 'n6-beam' if n6_beam == 1.0 else method + f'n6-beam-{n6_beam}-'
185
- if n8_beam > 0.0:
186
- method = 'n8-beam' if n8_beam == 1.0 else method + f'n8-beam-{n8_beam}-'
187
- return method[:-1] if method.endswith('-') else method
188
-
189
-
190
-if __name__ == '__main__':
191
- parser = argparse.ArgumentParser(
192
- description='Benchmark the performance on a series of requests.')
193
- parser = add_server_arguments(parser)
194
- parser.add_argument('--output-dir', type=str, help='path to output directory', default=None)
195
-
196
- parser.add_argument('--dataset', type=str, help='path to dataset', required=True)
197
- parser.add_argument('--request-rate', type=float, help='reqs/sec', required=True)
198
- parser.add_argument('--duration', type=int, help='duration in seconds', required=True)
199
- parser.add_argument('--do-memory-analysis', action='store_true',
200
- help='do memory analysis (This will lower the throughput. Use this only for analysis.)')
201
- parser.add_argument('--timeout', type=int, help='time out in seconds', default=None)
202
-
203
- parser.add_argument('--n1', type=float, help='ratio of requests with n=1', default=0.0)
204
- parser.add_argument('--n2', type=float, help='ratio of requests with n=2', default=0.0)
205
- parser.add_argument('--n3', type=float, help='ratio of requests with n=3', default=0.0)
206
- parser.add_argument('--n4', type=float, help='ratio of requests with n=4', default=0.0)
207
- parser.add_argument('--n6', type=float, help='ratio of requests with n=6', default=0.0)
208
- parser.add_argument('--n2-beam', type=float, help='ratio of requests with n=2 & beam search', default=0.0)
209
- parser.add_argument('--n4-beam', type=float, help='ratio of requests with n=4 & beam search', default=0.0)
210
- parser.add_argument('--n6-beam', type=float, help='ratio of requests with n=6 & beam search', default=0.0)
211
- parser.add_argument('--n8-beam', type=float, help='ratio of requests with n=8 & beam search', default=0.0)
212
- args = parser.parse_args()
213
- args = process_server_arguments(args)
214
- if args.n1 + args.n2 + args.n3 + args.n4 + args.n6 + args.n2_beam + args.n4_beam + args.n6_beam + args.n8_beam != 1.0:
215
- raise ValueError('The ratios of requests must sum to 1.')
216
-
217
- model_name = get_model_name(args.model)
218
- dataset_name = get_dataset_name(args.dataset)
219
- if 'opt' in model_name:
220
- if 'opt' not in args.dataset.lower():
221
- raise ValueError(f'OPT models can only be used with OPT datasets.')
222
- elif 'llama' in model_name:
223
- if 'llama' not in args.dataset.lower():
224
- raise ValueError(f'Llama models can only be used with Llama datasets.')
225
-
226
- dataset_name = 'sharegpt' if 'sharegpt' in args.dataset else 'alpaca'
227
- sample_dir = get_sampling_dir_name(
228
- args.n1, args.n2, args.n3, args.n4, args.n6, args.n2_beam, args.n4_beam, args.n6_beam, args.n8_beam)
229
- if args.output_dir is None:
230
- args.output_dir = os.path.join(
231
- '../exp',
232
- dataset_name,
233
- f'{model_name}-tp{args.tensor_parallel_size}',
234
- sample_dir,
235
- 'cacheflow',
236
- f'block{args.block_size}',
237
- f'req-rate-{args.request_rate}',
238
- f'seed{args.seed}',
239
- f'duration-{args.duration}',
240
- )
241
- os.makedirs(args.output_dir, exist_ok=True)
242
-
243
- # Set up logging.
244
- logging.basicConfig(
245
- format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
246
- datefmt="%m/%d/%Y %H:%M:%S",
247
- level=logging.INFO,
248
- handlers=[
249
- logging.StreamHandler(),
250
- logging.FileHandler(os.path.join(args.output_dir, 'log.txt')),
251
- ],
252
- )
253
- logger.info(args)
254
-
255
- main(args)
benchmarks/benchmark_throughput.py
+138
-28
@@ -1,3 +1,4 @@
1
+"""Benchmark offline inference throughput."""
2
import argparse
3
import json
4
import random
@@ -5,14 +6,29 @@ import time
6
from typing import List, Tuple
7
8
from cacheflow import LLM, SamplingParams
8
-from transformers import PreTrainedTokenizerBase
9
+import torch
10
+from transformers import (AutoConfig, AutoTokenizer, AutoModelForCausalLM,
11
+ PreTrainedTokenizerBase)
12
+from tqdm import tqdm
13
+
14
+
15
+def get_tokenizer(model_name: str) -> PreTrainedTokenizerBase:
16
+ config = AutoConfig.from_pretrained(model_name)
17
+ if config.model_type == "llama":
18
+ # A workaround for potential protobuf errors.
19
+ model_name = "hf-internal-testing/llama-tokenizer"
20
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
21
+ # To enable padding in the HF backend.
22
+ tokenizer.pad_token = tokenizer.eos_token
23
+ return tokenizer
24
+ return AutoTokenizer.from_pretrained(model_name)
25
26
27
def sample_requests(
28
dataset_path: str,
29
num_requests: int,
30
tokenizer: PreTrainedTokenizerBase,
15
-) -> List[Tuple[List[int], int]]:
31
+) -> List[Tuple[str, int, int]]:
32
# Load the dataset.
33
with open(dataset_path) as f:
34
dataset = json.load(f)
@@ -35,45 +51,52 @@ def sample_requests(
51
tokenized_dataset = []
52
for i in range(len(dataset)):
53
output_len = len(completion_token_ids[i])
38
- tokenized_dataset.append((prompt_token_ids[i], output_len))
39
- # Filter out if the prompt length + output length is greater than 2048.
40
- tokenized_dataset = [
41
- (prompt_token_ids, output_len)
42
- for prompt_token_ids, output_len in tokenized_dataset
43
- if len(prompt_token_ids) + output_len <= 2048
44
- ]
54
+ tokenized_dataset.append((prompts[i], prompt_token_ids[i], output_len))
55
+
56
+ # Filter out too long sequences.
57
+ filtered_dataset: List[Tuple[str, int, int]] = []
58
+ for prompt, prompt_token_ids, output_len in tokenized_dataset:
59
+ prompt_len = len(prompt_token_ids)
60
+ if prompt_len < 4 or output_len < 4:
61
+ # Prune too short sequences.
62
+ continue
63
+ if prompt_len > 1024 or prompt_len + output_len > 2048:
64
+ # Prune too long sequences.
65
+ continue
66
+ filtered_dataset.append((prompt, prompt_len, output_len))
67
68
# Sample the requests.
47
- sampled_requests = random.sample(tokenized_dataset, num_requests)
69
+ sampled_requests = random.sample(filtered_dataset, num_requests)
70
return sampled_requests
71
72
51
-def main(args: argparse.Namespace):
52
- print(args)
53
- random.seed(args.seed)
54
-
73
+def run_cacheflow(
74
+ requests: List[Tuple[str, int, int]],
75
+ model: str,
76
+ tensor_parallel_size: int,
77
+ seed: int,
78
+ n: int,
79
+ use_beam_search: bool,
80
+) -> float:
81
llm = LLM(
56
- model=args.model,
57
- tensor_parallel_size=args.tensor_parallel_size,
58
- seed=args.seed,
82
+ model=model,
83
+ tensor_parallel_size=tensor_parallel_size,
84
+ seed=seed,
85
)
60
- tokenizer = llm.get_tokenizer()
61
- requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
86
87
# Add the requests to the server.
64
- for prompt_token_ids, output_len in requests:
88
+ for prompt, _, output_len in requests:
89
sampling_params = SamplingParams(
66
- n=args.n,
67
- temperature=0.0 if args.use_beam_search else 1.0,
90
+ n=n,
91
+ temperature=0.0 if use_beam_search else 1.0,
92
top_p=1.0,
69
- use_beam_search=args.use_beam_search,
93
+ use_beam_search=use_beam_search,
94
ignore_eos=True,
95
max_tokens=output_len,
96
)
97
# FIXME(woosuk): Do not use internal method.
98
llm._add_request(
75
- prompt=None,
76
- prompt_token_ids=prompt_token_ids,
99
+ prompt=prompt,
100
sampling_params=sampling_params,
101
)
102
@@ -81,17 +104,95 @@ def main(args: argparse.Namespace):
104
# FIXME(woosuk): Do use internal method.
105
llm._run_server(use_tqdm=True)
106
end = time.time()
107
+ return end - start
108
+
109
+
110
+def run_hf(
111
+ requests: List[Tuple[str, int, int]],
112
+ model: str,
113
+ tokenizer: PreTrainedTokenizerBase,
114
+ n: int,
115
+ use_beam_search: bool,
116
+ max_batch_size: int,
117
+) -> float:
118
+ assert not use_beam_search
119
+ tokenizer = get_tokenizer(model)
120
+ llm = AutoModelForCausalLM.from_pretrained(
121
+ model, torch_dtype=torch.float16)
122
+ llm = llm.cuda()
123
+
124
+ pbar = tqdm(total=len(requests))
125
+ start = time.time()
126
+ batch: List[str] = []
127
+ max_prompt_len = 0
128
+ max_output_len = 0
129
+ for i in range(len(requests)):
130
+ prompt, prompt_len, output_len = requests[i]
131
+ # Add the prompt to the batch.
132
+ batch.append(prompt)
133
+ max_prompt_len = max(max_prompt_len, prompt_len)
134
+ max_output_len = max(max_output_len, output_len)
135
+ if len(batch) < max_batch_size and i != len(requests) - 1:
136
+ # Check if we can add more requests to the batch.
137
+ _, next_prompt_len, next_output_len = requests[i + 1]
138
+ if (max(max_prompt_len, next_prompt_len) + max(
139
+ max_output_len, next_output_len)) <= 2048:
140
+ # We can add more requests to the batch.
141
+ continue
142
+
143
+ # Generate the sequences.
144
+ input_ids = tokenizer(batch, return_tensors="pt", padding=True).input_ids
145
+ llm_outputs = llm.generate(
146
+ input_ids=input_ids.cuda(),
147
+ do_sample=not use_beam_search,
148
+ num_return_sequences=n,
149
+ temperature=1.0,
150
+ top_p=1.0,
151
+ use_cache=True,
152
+ max_new_tokens=max_output_len,
153
+ )
154
+ # Include the decoding time.
155
+ tokenizer.batch_decode(llm_outputs, skip_special_tokens=True)
156
+ pbar.update(len(batch))
157
+
158
+ # Clear the batch.
159
+ batch = []
160
+ max_prompt_len = 0
161
+ max_output_len = 0
162
+ end = time.time()
163
+ return end - start
164
+
165
+
166
+def main(args: argparse.Namespace):
167
+ print(args)
168
+ random.seed(args.seed)
169
+
170
+ # Sample the requests.
171
+ tokenizer = get_tokenizer(args.model)
172
+ requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
173
+
174
+ if args.backend == "cacheflow":
175
+ elapsed_time = run_cacheflow(
176
+ requests, args.model, args.tensor_parallel_size, args.seed, args.n,
177
+ args.use_beam_search)
178
+ elif args.backend == "hf":
179
+ assert args.tensor_parallel_size == 1
180
+ elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
181
+ args.use_beam_search, args.hf_max_batch_size)
182
+ else:
183
+ raise ValueError(f"Unknown backend: {args.backend}")
184
total_num_tokens = sum(
85
- len(prompt_token_ids) + output_len
86
- for prompt_token_ids, output_len in requests
185
+ prompt_len + output_len
186
+ for _, prompt_len, output_len in requests
187
)
88
- elapsed_time = end - start
188
print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
189
f"{total_num_tokens / elapsed_time:.2f} tokens/s")
190
191
192
if __name__ == "__main__":
193
parser = argparse.ArgumentParser(description="Benchmark the throughput.")
194
+ parser.add_argument("--backend", type=str, choices=["cacheflow", "hf"],
195
+ default="cacheflow")
196
parser.add_argument("--dataset", type=str, required=True,
197
help="Path to the dataset.")
198
parser.add_argument("--model", type=str, default="facebook/opt-125m")
@@ -102,5 +203,14 @@ if __name__ == "__main__":
203
parser.add_argument("--num-prompts", type=int, default=1000,
204
help="Number of prompts to process.")
205
parser.add_argument("--seed", type=int, default=0)
206
+ parser.add_argument("--hf-max-batch-size", type=int, default=None,
207
+ help="Maximum batch size for HF backend.")
208
args = parser.parse_args()
209
+ if args.backend == "cacheflow":
210
+ if args.hf_max_batch_size is not None:
211
+ raise ValueError("HF max batch size is only for HF backend.")
212
+ elif args.backend == "hf":
213
+ if args.hf_max_batch_size is None:
214
+ raise ValueError("HF max batch size is required for HF backend.")
215
+
216
main(args)
benchmarks/launch_tgi_server.sh
new
+16
@@ -0,0 +1,16 @@
1
+#!/bin/bash
2
+
3
+PORT=8001
4
+MODEL=$1
5
+TOKENS=$2
6
+
7
+docker run --gpus all --shm-size 1g -p $PORT:80 \
8
+ -v $PWD/data:/data \
9
+ ghcr.io/huggingface/text-generation-inference:0.8 \
10
+ --model-id $MODEL \
11
+ --sharded false \
12
+ --max-input-length 1024 \
13
+ --max-total-tokens 2048 \
14
+ --max-best-of 5 \
15
+ --max-concurrent-requests 5000 \
16
+ --max-batch-total-tokens $TOKENS
benchmarks/trace.py
deleted
-116
@@ -1,116 +0,0 @@
1
-import pickle
2
-import random
3
-from typing import List, Tuple
4
-
5
-import numpy as np
6
-
7
-from cacheflow.sampling_params import SamplingParams
8
-
9
-
10
-def generate_text_completion_requests(
11
- dataset: str,
12
- request_rate: float,
13
- duration: int,
14
- seed: int,
15
- n1: float = 0.0,
16
- n2: float = 0.0,
17
- n3: float = 0.0,
18
- n4: float = 0.0,
19
- n6: float = 0.0,
20
- n2_beam: float = 0.0,
21
- n4_beam: float = 0.0,
22
- n6_beam: float = 0.0,
23
- n8_beam: float = 0.0,
24
- max_seq_len: int = 2048,
25
- time_quantum: int = 10,
26
-) -> List[Tuple[float, List[int], SamplingParams]]:
27
- random.seed(seed)
28
- np.random.seed(seed)
29
-
30
- # Generate timestamps for requests using Poisson distribution.
31
- lam = request_rate * (time_quantum / 1000)
32
- quantums_per_sec = 1000 / time_quantum
33
- arrival_times = np.random.poisson(
34
- lam=lam, size=int(duration * quantums_per_sec))
35
- timestamps = []
36
- for i, n in enumerate(arrival_times):
37
- timestamps += [i * (time_quantum / 1000)] * n
38
-
39
- # Load and shuffle the dataset.
40
- num_requests = len(timestamps)
41
- with open(dataset, 'rb') as f:
42
- data = pickle.load(f)
43
-
44
- filtered = []
45
- for pair in data:
46
- input_tokens, output_tokens = pair
47
- input_len = len(input_tokens)
48
- output_len = len(output_tokens)
49
- # Filter out too long sequences.
50
- if input_len + output_len < max_seq_len:
51
- # Output tokens are not needed for the benchmark.
52
- filtered.append((input_tokens, output_len))
53
-
54
- data = []
55
- while len(data) < num_requests:
56
- data += filtered
57
- data = data[:num_requests]
58
- # Shuffle the data.
59
- assert len(data) == len(timestamps)
60
- random.shuffle(data)
61
-
62
- random_sampling_params_dict = {
63
- 'temperature': 1.0,
64
- 'top_p': 1.0,
65
- 'use_beam_search': False,
66
- 'stop_token_ids': set(),
67
- 'num_logprobs': 0,
68
- 'context_window_size': None,
69
- }
70
- beam_search_params_dict = {
71
- 'temperature': 0.0,
72
- 'top_p': 1.0,
73
- 'use_beam_search': True,
74
- 'stop_token_ids': set(),
75
- 'num_logprobs': 0,
76
- 'context_window_size': None,
77
- }
78
-
79
- # Generate requests based on the sampling parameter ratio.
80
- requests = []
81
- assert n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam + n8_beam == 1.0
82
- cum_sum = 0
83
- for timestamp, pair in zip(timestamps, data):
84
- input_tokens, output_len = pair
85
- if cum_sum < n1 * num_requests:
86
- sampling_params = SamplingParams(
87
- n=1, max_num_steps=output_len, **random_sampling_params_dict)
88
- elif cum_sum < (n1 + n2) * num_requests:
89
- sampling_params = SamplingParams(
90
- n=2, max_num_steps=output_len, **random_sampling_params_dict)
91
- elif cum_sum < (n1 + n2 + n3) * num_requests:
92
- sampling_params = SamplingParams(
93
- n=3, max_num_steps=output_len, **random_sampling_params_dict)
94
- elif cum_sum < (n1 + n2 + n3 + n4) * num_requests:
95
- sampling_params = SamplingParams(
96
- n=4, max_num_steps=output_len, **random_sampling_params_dict)
97
- elif cum_sum < (n1 + n2 + n3 + n4 + n6) * num_requests:
98
- sampling_params = SamplingParams(
99
- n=6, max_num_steps=output_len, **random_sampling_params_dict)
100
- elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam) * num_requests:
101
- sampling_params = SamplingParams(
102
- n=2, max_num_steps=output_len, **beam_search_params_dict)
103
- elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam) * num_requests:
104
- sampling_params = SamplingParams(
105
- n=4, max_num_steps=output_len, **beam_search_params_dict)
106
- elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam) * num_requests:
107
- sampling_params = SamplingParams(
108
- n=6, max_num_steps=output_len, **beam_search_params_dict)
109
- elif cum_sum < (n1 + n2 + n3 + n4 + n6 + n2_beam + n4_beam + n6_beam + n8_beam) * num_requests:
110
- sampling_params = SamplingParams(
111
- n=8, max_num_steps=output_len, **beam_search_params_dict)
112
- else:
113
- raise ValueError('Invalid request ratio.')
114
- cum_sum += 1
115
- requests.append((timestamp, input_tokens, sampling_params))
116
- return requests
cacheflow/server/arg_utils.py
+3
@@ -120,6 +120,7 @@ class ServerArgs:
120
class AsyncServerArgs(ServerArgs):
121
"""Arguments for asynchronous CacheFlow servers."""
122
server_use_ray: bool = False
123
+ disable_log_requests: bool = False
124
125
@staticmethod
126
def add_cli_args(
@@ -129,4 +130,6 @@ class AsyncServerArgs(ServerArgs):
130
parser.add_argument('--server-use-ray', action='store_true',
131
help='use Ray to start the LLM server in a '
132
'separate process as the web server process.')
133
+ parser.add_argument('--disable-log-requests', action='store_true',
134
+ help='disable logging requests')
135
return parser
cacheflow/server/async_llm_server.py
+13
-7
@@ -32,12 +32,14 @@ class AsyncLLMServer:
32
server_use_ray: Whether to make LLMServer a Ray actor. If so, the
33
async frontend will be executed in a separate process as the
34
model workers.
35
+ log_requests: Whether to log the requests.
36
*args, *kwargs: Arguments for LLMServer.
37
"""
38
def __init__(self, worker_use_ray: bool, server_use_ray: bool,
38
- *args, **kwargs) -> None:
39
+ log_requests: bool = True, *args, **kwargs) -> None:
40
self.worker_use_ray = worker_use_ray
41
self.server_use_ray = server_use_ray
42
+ self.log_requests = log_requests
43
if not self.server_use_ray:
44
server_class = LLMServer
45
elif self.worker_use_ray:
@@ -106,10 +108,11 @@ class AsyncLLMServer:
108
request_event = asyncio.Event()
109
self.request_events[request_id] = request_event
110
109
- logger.info(f"Received request {request_id}: "
110
- f"prompt: {prompt!r}, "
111
- f"sampling params: {sampling_params}, "
112
- f"prompt token ids: {prompt_token_ids}.")
111
+ if self.log_requests:
112
+ logger.info(f"Received request {request_id}: "
113
+ f"prompt: {prompt!r}, "
114
+ f"sampling params: {sampling_params}, "
115
+ f"prompt token ids: {prompt_token_ids}.")
116
117
# Add the request into the cacheflow server's waiting queue.
118
if self.server_use_ray:
@@ -152,7 +155,8 @@ class AsyncLLMServer:
155
156
# Once finished, release the resources of the sequence group.
157
if request_output.finished():
155
- logger.info(f"Finished request {request_id}.")
158
+ if self.log_requests:
159
+ logger.info(f"Finished request {request_id}.")
160
161
del self.request_outputs[request_id]
162
del self.request_events[request_id]
@@ -176,7 +180,8 @@ class AsyncLLMServer:
180
# The request has already finished or been aborted.
181
return
182
179
- logger.info(f"Aborted request {request_id}.")
183
+ if self.log_requests:
184
+ logger.info(f"Aborted request {request_id}.")
185
186
if self.server_use_ray:
187
await self.server.abort_request.remote(request_id)
@@ -206,6 +211,7 @@ class AsyncLLMServer:
211
# Create the LLM server.
212
server = cls(server_args.worker_use_ray,
213
server_args.server_use_ray,
214
+ not server_args.disable_log_requests,
215
*server_configs,
216
distributed_init_method, devices,
217
log_stats=not server_args.disable_log_stats)
examples/simple_fastapi_client.py
+8
-6
@@ -1,12 +1,14 @@
1
import argparse
2
import json
3
-import requests
3
from typing import Iterable, List
4
6
-def clear_line(n:int = 1) -> None:
5
+import requests
6
+
7
+
8
+def clear_line(n: int = 1) -> None:
9
LINE_UP = '\033[1A'
10
LINE_CLEAR = '\x1b[2K'
9
- for i in range(n):
11
+ for _ in range(n):
12
print(LINE_UP, end=LINE_CLEAR, flush=True)
13
14
@@ -53,7 +55,7 @@ if __name__ == "__main__":
55
n = args.n
56
stream = args.stream
57
56
- print(f"Prompt: {prompt}\n", flush=True)
58
+ print(f"Prompt: {prompt!r}\n", flush=True)
59
response = post_http_request(prompt, api_url, n, stream)
60
61
if stream:
@@ -63,8 +65,8 @@ if __name__ == "__main__":
65
num_printed_lines = 0
66
for i, line in enumerate(h):
67
num_printed_lines += 1
66
- print(f"Beam candidate {i}: {line}", flush=True)
68
+ print(f"Beam candidate {i}: {line!r}", flush=True)
69
else:
70
output = get_response(response)
71
for i, line in enumerate(output):
70
- print(f"Beam candidate {i}: {line}", flush=True)
72
+ print(f"Beam candidate {i}: {line!r}", flush=True)