A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Benchmark] Add BurstGPT to benchmark_serving (#13063)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Woosuk Kwon committed
Feb 10, 2025 at 21:25 UTC
58047c6f0410fc7a86b64c88c092a246984b2342
2 files changed
+47
-1
benchmarks/README.md
+8
@@ -19,3 +19,11 @@ mkdir coco -p
19
wget http://images.cocodataset.org/zips/train2017.zip -O coco/train2017.zip
20
unzip coco/train2017.zip -d coco/
21
```
22
+
23
+# Downloading the BurstGPT dataset
24
+
25
+You can download the BurstGPT v1.1 dataset by running:
26
+
27
+```bash
28
+wget https://github.com/HPMLL/BurstGPT/releases/download/v1.1/BurstGPT_without_fails_2.csv
29
+```
benchmarks/benchmark_serving.py
+39
-1
@@ -38,6 +38,7 @@ from datetime import datetime
38
from typing import Any, AsyncGenerator, Collection, Dict, List, Optional, Tuple
39
40
import numpy as np
41
+import pandas as pd
42
from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
43
RequestFuncOutput)
44
from datasets import load_dataset
@@ -131,6 +132,35 @@ def sample_sharegpt_requests(
132
return filtered_dataset
133
134
135
+def sample_burstgpt_requests(
136
+ dataset_path: str,
137
+ num_requests: int,
138
+ random_seed: int,
139
+ tokenizer: PreTrainedTokenizerBase,
140
+) -> List[Tuple[str, int, int, None]]:
141
+ df = pd.read_csv(dataset_path)
142
+ gpt4_df = df[df["Model"] == "GPT-4"]
143
+ # Remove the failed requests (i.e., response length is 0)
144
+ gpt4_df = gpt4_df[gpt4_df["Response tokens"] > 0]
145
+ # Randomly sample num_requests from the dataset
146
+ if num_requests <= len(gpt4_df):
147
+ gpt4_df = gpt4_df.sample(n=num_requests, random_state=random_seed)
148
+ else:
149
+ gpt4_df = gpt4_df.sample(n=num_requests,
150
+ random_state=random_seed,
151
+ replace=True)
152
+ # Convert the dataframe to a list of tuples
153
+ dataset = gpt4_df.values.tolist()
154
+ input_requests = []
155
+ for i in range(num_requests):
156
+ input_len = int(dataset[i][2])
157
+ output_len = int(dataset[i][3])
158
+ prompt = tokenizer.decode([(i + j) % tokenizer.vocab_size
159
+ for j in range(input_len)])
160
+ input_requests.append((prompt, input_len, output_len, None))
161
+ return input_requests
162
+
163
+
164
def sample_sonnet_requests(
165
dataset_path: str,
166
num_requests: int,
@@ -830,6 +860,14 @@ def main(args: argparse.Namespace):
860
fixed_output_len=args.sharegpt_output_len,
861
)
862
863
+ elif args.dataset_name == "burstgpt":
864
+ input_requests = sample_burstgpt_requests(
865
+ dataset_path=args.dataset_path,
866
+ num_requests=args.num_prompts,
867
+ random_seed=args.seed,
868
+ tokenizer=tokenizer,
869
+ )
870
+
871
elif args.dataset_name == "sonnet":
872
# Do not format the prompt, pass to message directly
873
if args.backend == "openai-chat":
@@ -995,7 +1033,7 @@ if __name__ == "__main__":
1033
"--dataset-name",
1034
type=str,
1035
default="sharegpt",
998
- choices=["sharegpt", "sonnet", "random", "hf"],
1036
+ choices=["sharegpt", "burstgpt", "sonnet", "random", "hf"],
1037
help="Name of the dataset to benchmark on.",
1038
)
1039
parser.add_argument("--dataset-path",