A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
add benchmark for fix length input and output (#5857)
Co-authored-by: Roger Wang <ywang@roblox.com>
Haichuan committed
Jul 7, 2024 at 15:42 UTC
333306a252a0ef42ef07b11b64df6ff1ac2e2d6d
1 file changed
+60
-5
benchmarks/benchmark_serving.py
+60
-5
@@ -17,7 +17,7 @@ On the client side, run:
17
--dataset-path <path to dataset> \
18
--request-rate <request_rate> \ # By default <request_rate> is inf
19
--num-prompts <num_prompts> # By default <num_prompts> is 1000
20
-
20
+
21
when using tgi backend, add
22
--endpoint /generate_stream
23
to the end of the command above.
@@ -77,7 +77,6 @@ def sample_sharegpt_requests(
77
) -> List[Tuple[str, int, int]]:
78
if fixed_output_len is not None and fixed_output_len < 4:
79
raise ValueError("output_len too small")
80
-
80
# Load the dataset.
81
with open(dataset_path) as f:
82
dataset = json.load(f)
@@ -185,6 +184,31 @@ def sample_sonnet_requests(
184
return sampled_requests
185
186
187
+def sample_random_requests(
188
+ input_len: int, output_len: int, num_prompts: int, range_ratio: float,
189
+ tokenizer: PreTrainedTokenizerBase) -> List[Tuple[str, int, int]]:
190
+
191
+ input_lens = np.random.randint(
192
+ int(input_len * range_ratio),
193
+ input_len + 1,
194
+ size=num_prompts,
195
+ )
196
+ output_lens = np.random.randint(
197
+ int(output_len * range_ratio),
198
+ output_len + 1,
199
+ size=num_prompts,
200
+ )
201
+ offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
202
+ input_requests = []
203
+ for i in range(args.num_prompts):
204
+ prompt = tokenizer.decode([(offsets[i] + i + j) % tokenizer.vocab_size
205
+ for j in range(input_lens[i])])
206
+ input_requests.append(
207
+ (prompt, int(input_lens[i]), int(output_lens[i])))
208
+
209
+ return input_requests
210
+
211
+
212
async def get_request(
213
input_requests: List[Tuple[str, int, int]],
214
request_rate: float,
@@ -196,6 +220,7 @@ async def get_request(
220
if request_rate == float("inf"):
221
# If the request rate is infinity, then we don't need to wait.
222
continue
223
+
224
# Sample the request interval from the exponential distribution.
225
interval = np.random.exponential(1.0 / request_rate)
226
# The next request will be sent after the interval.
@@ -219,7 +244,7 @@ def calculate_metrics(
244
# We use the tokenizer to count the number of output tokens for all
245
# serving backends instead of looking at len(outputs[i].itl) since
246
# multiple output tokens may be bundled together
222
- # Note: this may inflate the output token count slightly
247
+ # Note : this may inflate the output token count slightly
248
output_len = len(
249
tokenizer(outputs[i].generated_text,
250
add_special_tokens=False).input_ids)
@@ -456,6 +481,15 @@ def main(args: argparse.Namespace):
481
for prompt, prompt_formatted, prompt_len,
482
output_len in input_requests]
483
484
+ elif args.dataset_name == "random":
485
+ input_requests = sample_random_requests(
486
+ input_len=args.input_len,
487
+ output_len=args.output_len,
488
+ num_prompts=args.num_prompts,
489
+ range_ratio=args.range_ratio,
490
+ tokenizer=tokenizer,
491
+ )
492
+
493
else:
494
raise ValueError(f"Unknown dataset: {args.dataset_name}")
495
@@ -549,7 +583,7 @@ if __name__ == "__main__":
583
"--dataset-name",
584
type=str,
585
default="sharegpt",
552
- choices=["sharegpt", "sonnet"],
586
+ choices=["sharegpt", "sonnet", "random"],
587
help="Name of the dataset to benchmark on.",
588
)
589
parser.add_argument("--dataset-path",
@@ -566,7 +600,7 @@ if __name__ == "__main__":
600
"--tokenizer",
601
type=str,
602
help=
569
- "Name or path of the tokenizer, if not using the default tokenizer.",
603
+ "Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
604
)
605
parser.add_argument(
606
"--best-of",
@@ -609,6 +643,27 @@ if __name__ == "__main__":
643
help=
644
"Number of prefix tokens per request, used only for sonnet dataset.",
645
)
646
+ parser.add_argument(
647
+ "--random-input-len",
648
+ type=int,
649
+ default=1024,
650
+ help=
651
+ "Number of input tokens per request, used only for random sampling.",
652
+ )
653
+ parser.add_argument(
654
+ "--random-output-len",
655
+ type=int,
656
+ default=128,
657
+ help=
658
+ "Number of output tokens per request, used only for random sampling.",
659
+ )
660
+ parser.add_argument(
661
+ "--random-range-ratio",
662
+ type=float,
663
+ default=1.0,
664
+ help="Range of sampled ratio of input/output length, "
665
+ "used only for random sampling.",
666
+ )
667
parser.add_argument(
668
"--request-rate",
669
type=float,