A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Frontend] Add "input speed" to tqdm postfix alongside output speed (#5425)
Michael Goin committed
Jun 12, 2024 at 18:42 UTC
7d19de2e9c9a94658c36b55011b803a7991d0335
1 file changed
+12
-5
vllm/entrypoints/llm.py
+12
-5
@@ -545,11 +545,13 @@ class LLM:
545
total=num_requests,
546
desc="Processed prompts",
547
dynamic_ncols=True,
548
- postfix=f"Generation Speed: {0:.2f} toks/s",
548
+ postfix=(f"est. speed input: {0:.2f} toks/s, "
549
+ f"output: {0:.2f} toks/s"),
550
)
551
# Run the engine.
552
outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
552
- total_toks = 0
553
+ total_in_toks = 0
554
+ total_out_toks = 0
555
while self.llm_engine.has_unfinished_requests():
556
step_outputs = self.llm_engine.step()
557
for output in step_outputs:
@@ -558,10 +560,15 @@ class LLM:
560
if use_tqdm:
561
if isinstance(output, RequestOutput):
562
# Calculate tokens only for RequestOutput
561
- total_toks += sum(
563
+ total_in_toks += len(output.prompt_token_ids)
564
+ in_spd = total_in_toks / pbar.format_dict["elapsed"]
565
+ total_out_toks += sum(
566
len(stp.token_ids) for stp in output.outputs)
563
- spd = total_toks / pbar.format_dict["elapsed"]
564
- pbar.postfix = f"Generation Speed: {spd:.2f} toks/s"
567
+ out_spd = total_out_toks / pbar.format_dict[
568
+ "elapsed"]
569
+ pbar.postfix = (
570
+ f"est. speed input: {in_spd:.2f} toks/s, "
571
+ f"output: {out_spd:.2f} toks/s")
572
pbar.update(1)
573
if use_tqdm:
574
pbar.close()