@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 333306a252
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

add benchmark for fix length input and output (#5857)

Co-authored-by: Roger Wang <ywang@roblox.com>

Haichuan committed Jul 7, 2024 at 15:42 UTC 333306a252a0ef42ef07b11b64df6ff1ac2e2d6d
1 file changed +60 -5
benchmarks/benchmark_serving.py
+60 -5
@@ -17,7 +17,7 @@ On the client side, run:
17 --dataset-path <path to dataset> \
18 --request-rate <request_rate> \ # By default <request_rate> is inf
19 --num-prompts <num_prompts> # By default <num_prompts> is 1000
20 -
20 +
21 when using tgi backend, add
22 --endpoint /generate_stream
23 to the end of the command above.
@@ -77,7 +77,6 @@ def sample_sharegpt_requests(
77 ) -> List[Tuple[str, int, int]]:
78 if fixed_output_len is not None and fixed_output_len < 4:
79 raise ValueError("output_len too small")
80 -
80 # Load the dataset.
81 with open(dataset_path) as f:
82 dataset = json.load(f)
@@ -185,6 +184,31 @@ def sample_sonnet_requests(
184 return sampled_requests
185
186
187 +def sample_random_requests(
188 + input_len: int, output_len: int, num_prompts: int, range_ratio: float,
189 + tokenizer: PreTrainedTokenizerBase) -> List[Tuple[str, int, int]]:
190 +
191 + input_lens = np.random.randint(
192 + int(input_len * range_ratio),
193 + input_len + 1,
194 + size=num_prompts,
195 + )
196 + output_lens = np.random.randint(
197 + int(output_len * range_ratio),
198 + output_len + 1,
199 + size=num_prompts,
200 + )
201 + offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts)
202 + input_requests = []
203 + for i in range(args.num_prompts):
204 + prompt = tokenizer.decode([(offsets[i] + i + j) % tokenizer.vocab_size
205 + for j in range(input_lens[i])])
206 + input_requests.append(
207 + (prompt, int(input_lens[i]), int(output_lens[i])))
208 +
209 + return input_requests
210 +
211 +
212 async def get_request(
213 input_requests: List[Tuple[str, int, int]],
214 request_rate: float,
@@ -196,6 +220,7 @@ async def get_request(
220 if request_rate == float("inf"):
221 # If the request rate is infinity, then we don't need to wait.
222 continue
223 +
224 # Sample the request interval from the exponential distribution.
225 interval = np.random.exponential(1.0 / request_rate)
226 # The next request will be sent after the interval.
@@ -219,7 +244,7 @@ def calculate_metrics(
244 # We use the tokenizer to count the number of output tokens for all
245 # serving backends instead of looking at len(outputs[i].itl) since
246 # multiple output tokens may be bundled together
222 - # Note: this may inflate the output token count slightly
247 + # Note : this may inflate the output token count slightly
248 output_len = len(
249 tokenizer(outputs[i].generated_text,
250 add_special_tokens=False).input_ids)
@@ -456,6 +481,15 @@ def main(args: argparse.Namespace):
481 for prompt, prompt_formatted, prompt_len,
482 output_len in input_requests]
483
484 + elif args.dataset_name == "random":
485 + input_requests = sample_random_requests(
486 + input_len=args.input_len,
487 + output_len=args.output_len,
488 + num_prompts=args.num_prompts,
489 + range_ratio=args.range_ratio,
490 + tokenizer=tokenizer,
491 + )
492 +
493 else:
494 raise ValueError(f"Unknown dataset: {args.dataset_name}")
495
@@ -549,7 +583,7 @@ if __name__ == "__main__":
583 "--dataset-name",
584 type=str,
585 default="sharegpt",
552 - choices=["sharegpt", "sonnet"],
586 + choices=["sharegpt", "sonnet", "random"],
587 help="Name of the dataset to benchmark on.",
588 )
589 parser.add_argument("--dataset-path",
@@ -566,7 +600,7 @@ if __name__ == "__main__":
600 "--tokenizer",
601 type=str,
602 help=
569 - "Name or path of the tokenizer, if not using the default tokenizer.",
603 + "Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
604 )
605 parser.add_argument(
606 "--best-of",
@@ -609,6 +643,27 @@ if __name__ == "__main__":
643 help=
644 "Number of prefix tokens per request, used only for sonnet dataset.",
645 )
646 + parser.add_argument(
647 + "--random-input-len",
648 + type=int,
649 + default=1024,
650 + help=
651 + "Number of input tokens per request, used only for random sampling.",
652 + )
653 + parser.add_argument(
654 + "--random-output-len",
655 + type=int,
656 + default=128,
657 + help=
658 + "Number of output tokens per request, used only for random sampling.",
659 + )
660 + parser.add_argument(
661 + "--random-range-ratio",
662 + type=float,
663 + default=1.0,
664 + help="Range of sampled ratio of input/output length, "
665 + "used only for random sampling.",
666 + )
667 parser.add_argument(
668 "--request-rate",
669 type=float,