@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7923dcad12
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Update ShareGPT Dataset Sampling in Serving Benchmark (#4279)

Roger Wang committed Apr 24, 2024 at 09:49 UTC 7923dcad12edae7dcfd6e0cf7ce2984b3bcecf0f
1 file changed +28 -22
benchmarks/benchmark_serving.py
+28 -22
@@ -27,7 +27,7 @@ import time
27 import warnings
28 from dataclasses import dataclass
29 from datetime import datetime
30 -from typing import AsyncGenerator, List, Tuple
30 +from typing import AsyncGenerator, List, Optional, Tuple
31
32 import numpy as np
33 from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
@@ -58,7 +58,11 @@ def sample_sharegpt_requests(
58 dataset_path: str,
59 num_requests: int,
60 tokenizer: PreTrainedTokenizerBase,
61 + fixed_output_len: Optional[int] = None,
62 ) -> List[Tuple[str, int, int]]:
63 + if fixed_output_len is not None and fixed_output_len < 4:
64 + raise ValueError("output_len too small")
65 +
66 # Load the dataset.
67 with open(dataset_path) as f:
68 dataset = json.load(f)
@@ -68,38 +72,32 @@ def sample_sharegpt_requests(
72 dataset = [(data["conversations"][0]["value"],
73 data["conversations"][1]["value"]) for data in dataset]
74
71 - # some of these will be filtered out, so sample more than we need
72 - sampled_indices = random.sample(range(len(dataset)),
73 - int(num_requests * 1.2))
74 - dataset = [dataset[i] for i in sampled_indices]
75 -
76 - # Tokenize the prompts and completions.
77 - prompts = [prompt for prompt, _ in dataset]
78 - prompt_token_ids = tokenizer(prompts).input_ids
79 - completions = [completion for _, completion in dataset]
80 - completion_token_ids = tokenizer(completions).input_ids
81 - tokenized_dataset = []
82 - for i in range(len(dataset)):
83 - output_len = len(completion_token_ids[i])
84 - tokenized_dataset.append((prompts[i], prompt_token_ids[i], output_len))
75 + # Shuffle the dataset.
76 + random.shuffle(dataset)
77
86 - # Filter out too long sequences.
78 + # Filter out sequences that are too long or too short
79 filtered_dataset: List[Tuple[str, int, int]] = []
88 - for prompt, prompt_token_ids, output_len in tokenized_dataset:
80 + for i in range(len(dataset)):
81 + if len(filtered_dataset) == num_requests:
82 + break
83 +
84 + # Tokenize the prompts and completions.
85 + prompt = dataset[i][0]
86 + prompt_token_ids = tokenizer(prompt).input_ids
87 + completion = dataset[i][1]
88 + completion_token_ids = tokenizer(completion).input_ids
89 prompt_len = len(prompt_token_ids)
90 + output_len = len(completion_token_ids
91 + ) if fixed_output_len is None else fixed_output_len
92 if prompt_len < 4 or output_len < 4:
93 # Prune too short sequences.
92 - # This is because TGI causes errors when the input or output length
93 - # is too short.
94 continue
95 if prompt_len > 1024 or prompt_len + output_len > 2048:
96 # Prune too long sequences.
97 continue
98 filtered_dataset.append((prompt, prompt_len, output_len))
99
100 - # Sample the requests.
101 - sampled_requests = random.sample(filtered_dataset, num_requests)
102 - return sampled_requests
100 + return filtered_dataset
101
102
103 def sample_sonnet_requests(
@@ -361,6 +359,7 @@ def main(args: argparse.Namespace):
359 dataset_path=args.dataset,
360 num_requests=args.num_prompts,
361 tokenizer=tokenizer,
362 + fixed_output_len=args.sharegpt_output_len,
363 )
364
365 elif args.dataset_name == "sharegpt":
@@ -368,6 +367,7 @@ def main(args: argparse.Namespace):
367 dataset_path=args.dataset_path,
368 num_requests=args.num_prompts,
369 tokenizer=tokenizer,
370 + fixed_output_len=args.sharegpt_output_len,
371 )
372
373 elif args.dataset_name == "sonnet":
@@ -524,6 +524,12 @@ if __name__ == "__main__":
524 default=1000,
525 help="Number of prompts to process.",
526 )
527 + parser.add_argument(
528 + "--sharegpt-output-len",
529 + type=int,
530 + default=None,
531 + help="Output length for each request. Overrides the output length "
532 + "from the ShareGPT dataset.")
533 parser.add_argument(
534 "--sonnet-input-len",
535 type=int,