A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Frontend] Add --logprobs argument to `benchmark_serving.py` (#8191)
afeldman-nm committed
Sep 6, 2024 at 12:01 UTC
e5cab71531360345e5b30b98dfcfec8087d6cddf
3 files changed
+19
-1
benchmarks/backend_request_func.py
+2
@@ -24,6 +24,7 @@ class RequestFuncInput:
24
model: str
25
best_of: int = 1
26
use_beam_search: bool = False
27
+ logprobs: Optional[int] = None
28
29
30
@dataclass
@@ -236,6 +237,7 @@ async def async_request_openai_completions(
237
"temperature": 0.0,
238
"best_of": request_func_input.best_of,
239
"max_tokens": request_func_input.output_len,
240
+ "logprobs": request_func_input.logprobs,
241
"stream": True,
242
}
243
headers = {
benchmarks/benchmark_serving.py
+16
@@ -318,6 +318,7 @@ async def benchmark(
318
model_id: str,
319
tokenizer: PreTrainedTokenizerBase,
320
input_requests: List[Tuple[str, int, int]],
321
+ logprobs: Optional[int],
322
best_of: int,
323
use_beam_search: bool,
324
request_rate: float,
@@ -339,6 +340,7 @@ async def benchmark(
340
api_url=api_url,
341
prompt_len=test_prompt_len,
342
output_len=test_output_len,
343
+ logprobs=logprobs,
344
best_of=best_of,
345
use_beam_search=use_beam_search,
346
)
@@ -358,6 +360,7 @@ async def benchmark(
360
api_url=base_url + "/start_profile",
361
prompt_len=test_prompt_len,
362
output_len=test_output_len,
363
+ logprobs=logprobs,
364
best_of=best_of,
365
use_beam_search=use_beam_search,
366
)
@@ -379,6 +382,7 @@ async def benchmark(
382
api_url=api_url,
383
prompt_len=prompt_len,
384
output_len=output_len,
385
+ logprobs=logprobs,
386
best_of=best_of,
387
use_beam_search=use_beam_search,
388
)
@@ -396,6 +400,7 @@ async def benchmark(
400
api_url=base_url + "/stop_profile",
401
prompt_len=test_prompt_len,
402
output_len=test_output_len,
403
+ logprobs=logprobs,
404
best_of=best_of,
405
use_beam_search=use_beam_search,
406
)
@@ -580,6 +585,7 @@ def main(args: argparse.Namespace):
585
model_id=model_id,
586
tokenizer=tokenizer,
587
input_requests=input_requests,
588
+ logprobs=args.logprobs,
589
best_of=args.best_of,
590
use_beam_search=args.use_beam_search,
591
request_rate=args.request_rate,
@@ -721,6 +727,16 @@ if __name__ == "__main__":
727
help=
728
"Number of output tokens per request, used only for sonnet dataset.",
729
)
730
+ parser.add_argument(
731
+ "--logprobs",
732
+ type=int,
733
+ default=None,
734
+ help=("Number of logprobs-per-token to compute & return as part of "
735
+ "the request. If unspecified, then either (1) if beam search "
736
+ "is disabled, no logprobs are computed & a single dummy "
737
+ "logprob is returned for each token; or (2) if beam search "
738
+ "is enabled 1 logprob per token is computed"),
739
+ )
740
parser.add_argument(
741
"--sonnet-prefix-len",
742
type=int,
tests/multi_step/test_correctness_llm.py
+1
-1
@@ -57,7 +57,7 @@ def test_multi_step_llm(
57
GPU -> CPU output transfer
58
num_prompts: number of example prompts under test
59
num_logprobs: corresponds to the `logprobs` argument to the OpenAI
60
- completions endpoint; `None` -> no logprobs
60
+ completions endpoint; `None` -> 1 logprob returned.
61
"""
62
63
prompts = example_prompts