A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Update ShareGPT Dataset Sampling in Serving Benchmark (#4279)
Roger Wang committed
Apr 24, 2024 at 09:49 UTC
7923dcad12edae7dcfd6e0cf7ce2984b3bcecf0f
1 file changed
+28
-22
benchmarks/benchmark_serving.py
+28
-22
@@ -27,7 +27,7 @@ import time
27
import warnings
28
from dataclasses import dataclass
29
from datetime import datetime
30
-from typing import AsyncGenerator, List, Tuple
30
+from typing import AsyncGenerator, List, Optional, Tuple
31
32
import numpy as np
33
from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
@@ -58,7 +58,11 @@ def sample_sharegpt_requests(
58
dataset_path: str,
59
num_requests: int,
60
tokenizer: PreTrainedTokenizerBase,
61
+ fixed_output_len: Optional[int] = None,
62
) -> List[Tuple[str, int, int]]:
63
+ if fixed_output_len is not None and fixed_output_len < 4:
64
+ raise ValueError("output_len too small")
65
+
66
# Load the dataset.
67
with open(dataset_path) as f:
68
dataset = json.load(f)
@@ -68,38 +72,32 @@ def sample_sharegpt_requests(
72
dataset = [(data["conversations"][0]["value"],
73
data["conversations"][1]["value"]) for data in dataset]
74
71
- # some of these will be filtered out, so sample more than we need
72
- sampled_indices = random.sample(range(len(dataset)),
73
- int(num_requests * 1.2))
74
- dataset = [dataset[i] for i in sampled_indices]
75
-
76
- # Tokenize the prompts and completions.
77
- prompts = [prompt for prompt, _ in dataset]
78
- prompt_token_ids = tokenizer(prompts).input_ids
79
- completions = [completion for _, completion in dataset]
80
- completion_token_ids = tokenizer(completions).input_ids
81
- tokenized_dataset = []
82
- for i in range(len(dataset)):
83
- output_len = len(completion_token_ids[i])
84
- tokenized_dataset.append((prompts[i], prompt_token_ids[i], output_len))
75
+ # Shuffle the dataset.
76
+ random.shuffle(dataset)
77
86
- # Filter out too long sequences.
78
+ # Filter out sequences that are too long or too short
79
filtered_dataset: List[Tuple[str, int, int]] = []
88
- for prompt, prompt_token_ids, output_len in tokenized_dataset:
80
+ for i in range(len(dataset)):
81
+ if len(filtered_dataset) == num_requests:
82
+ break
83
+
84
+ # Tokenize the prompts and completions.
85
+ prompt = dataset[i][0]
86
+ prompt_token_ids = tokenizer(prompt).input_ids
87
+ completion = dataset[i][1]
88
+ completion_token_ids = tokenizer(completion).input_ids
89
prompt_len = len(prompt_token_ids)
90
+ output_len = len(completion_token_ids
91
+ ) if fixed_output_len is None else fixed_output_len
92
if prompt_len < 4 or output_len < 4:
93
# Prune too short sequences.
92
- # This is because TGI causes errors when the input or output length
93
- # is too short.
94
continue
95
if prompt_len > 1024 or prompt_len + output_len > 2048:
96
# Prune too long sequences.
97
continue
98
filtered_dataset.append((prompt, prompt_len, output_len))
99
100
- # Sample the requests.
101
- sampled_requests = random.sample(filtered_dataset, num_requests)
102
- return sampled_requests
100
+ return filtered_dataset
101
102
103
def sample_sonnet_requests(
@@ -361,6 +359,7 @@ def main(args: argparse.Namespace):
359
dataset_path=args.dataset,
360
num_requests=args.num_prompts,
361
tokenizer=tokenizer,
362
+ fixed_output_len=args.sharegpt_output_len,
363
)
364
365
elif args.dataset_name == "sharegpt":
@@ -368,6 +367,7 @@ def main(args: argparse.Namespace):
367
dataset_path=args.dataset_path,
368
num_requests=args.num_prompts,
369
tokenizer=tokenizer,
370
+ fixed_output_len=args.sharegpt_output_len,
371
)
372
373
elif args.dataset_name == "sonnet":
@@ -524,6 +524,12 @@ if __name__ == "__main__":
524
default=1000,
525
help="Number of prompts to process.",
526
)
527
+ parser.add_argument(
528
+ "--sharegpt-output-len",
529
+ type=int,
530
+ default=None,
531
+ help="Output length for each request. Overrides the output length "
532
+ "from the ShareGPT dataset.")
533
parser.add_argument(
534
"--sonnet-input-len",
535
type=int,