@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7d19de2e9c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Frontend] Add "input speed" to tqdm postfix alongside output speed (#5425)

Michael Goin committed Jun 12, 2024 at 18:42 UTC 7d19de2e9c9a94658c36b55011b803a7991d0335
1 file changed +12 -5
vllm/entrypoints/llm.py
+12 -5
@@ -545,11 +545,13 @@ class LLM:
545 total=num_requests,
546 desc="Processed prompts",
547 dynamic_ncols=True,
548 - postfix=f"Generation Speed: {0:.2f} toks/s",
548 + postfix=(f"est. speed input: {0:.2f} toks/s, "
549 + f"output: {0:.2f} toks/s"),
550 )
551 # Run the engine.
552 outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
552 - total_toks = 0
553 + total_in_toks = 0
554 + total_out_toks = 0
555 while self.llm_engine.has_unfinished_requests():
556 step_outputs = self.llm_engine.step()
557 for output in step_outputs:
@@ -558,10 +560,15 @@ class LLM:
560 if use_tqdm:
561 if isinstance(output, RequestOutput):
562 # Calculate tokens only for RequestOutput
561 - total_toks += sum(
563 + total_in_toks += len(output.prompt_token_ids)
564 + in_spd = total_in_toks / pbar.format_dict["elapsed"]
565 + total_out_toks += sum(
566 len(stp.token_ids) for stp in output.outputs)
563 - spd = total_toks / pbar.format_dict["elapsed"]
564 - pbar.postfix = f"Generation Speed: {spd:.2f} toks/s"
567 + out_spd = total_out_toks / pbar.format_dict[
568 + "elapsed"]
569 + pbar.postfix = (
570 + f"est. speed input: {in_spd:.2f} toks/s, "
571 + f"output: {out_spd:.2f} toks/s")
572 pbar.update(1)
573 if use_tqdm:
574 pbar.close()