@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e5cab71531
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend] Add --logprobs argument to `benchmark_serving.py` (#8191)

afeldman-nm committed Sep 6, 2024 at 12:01 UTC e5cab71531360345e5b30b98dfcfec8087d6cddf
3 files changed +19 -1
benchmarks/backend_request_func.py
+2
@@ -24,6 +24,7 @@ class RequestFuncInput:
24 model: str
25 best_of: int = 1
26 use_beam_search: bool = False
27 + logprobs: Optional[int] = None
28
29
30 @dataclass
@@ -236,6 +237,7 @@ async def async_request_openai_completions(
237 "temperature": 0.0,
238 "best_of": request_func_input.best_of,
239 "max_tokens": request_func_input.output_len,
240 + "logprobs": request_func_input.logprobs,
241 "stream": True,
242 }
243 headers = {
benchmarks/benchmark_serving.py
+16
@@ -318,6 +318,7 @@ async def benchmark(
318 model_id: str,
319 tokenizer: PreTrainedTokenizerBase,
320 input_requests: List[Tuple[str, int, int]],
321 + logprobs: Optional[int],
322 best_of: int,
323 use_beam_search: bool,
324 request_rate: float,
@@ -339,6 +340,7 @@ async def benchmark(
340 api_url=api_url,
341 prompt_len=test_prompt_len,
342 output_len=test_output_len,
343 + logprobs=logprobs,
344 best_of=best_of,
345 use_beam_search=use_beam_search,
346 )
@@ -358,6 +360,7 @@ async def benchmark(
360 api_url=base_url + "/start_profile",
361 prompt_len=test_prompt_len,
362 output_len=test_output_len,
363 + logprobs=logprobs,
364 best_of=best_of,
365 use_beam_search=use_beam_search,
366 )
@@ -379,6 +382,7 @@ async def benchmark(
382 api_url=api_url,
383 prompt_len=prompt_len,
384 output_len=output_len,
385 + logprobs=logprobs,
386 best_of=best_of,
387 use_beam_search=use_beam_search,
388 )
@@ -396,6 +400,7 @@ async def benchmark(
400 api_url=base_url + "/stop_profile",
401 prompt_len=test_prompt_len,
402 output_len=test_output_len,
403 + logprobs=logprobs,
404 best_of=best_of,
405 use_beam_search=use_beam_search,
406 )
@@ -580,6 +585,7 @@ def main(args: argparse.Namespace):
585 model_id=model_id,
586 tokenizer=tokenizer,
587 input_requests=input_requests,
588 + logprobs=args.logprobs,
589 best_of=args.best_of,
590 use_beam_search=args.use_beam_search,
591 request_rate=args.request_rate,
@@ -721,6 +727,16 @@ if __name__ == "__main__":
727 help=
728 "Number of output tokens per request, used only for sonnet dataset.",
729 )
730 + parser.add_argument(
731 + "--logprobs",
732 + type=int,
733 + default=None,
734 + help=("Number of logprobs-per-token to compute & return as part of "
735 + "the request. If unspecified, then either (1) if beam search "
736 + "is disabled, no logprobs are computed & a single dummy "
737 + "logprob is returned for each token; or (2) if beam search "
738 + "is enabled 1 logprob per token is computed"),
739 + )
740 parser.add_argument(
741 "--sonnet-prefix-len",
742 type=int,
tests/multi_step/test_correctness_llm.py
+1 -1
@@ -57,7 +57,7 @@ def test_multi_step_llm(
57 GPU -> CPU output transfer
58 num_prompts: number of example prompts under test
59 num_logprobs: corresponds to the `logprobs` argument to the OpenAI
60 - completions endpoint; `None` -> no logprobs
60 + completions endpoint; `None` -> 1 logprob returned.
61 """
62
63 prompts = example_prompts