A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Minor] Update the tqdm bar for parallel sampling (#14571)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Mar 10, 2025 at 14:23 UTC
0967110e42a33d016c0cf5214bd32c8ce3faae19
1 file changed
+3
-2
vllm/entrypoints/llm.py
+3
-2
@@ -1384,8 +1384,9 @@ class LLM:
1384
if use_tqdm:
1385
if isinstance(output, RequestOutput):
1386
# Calculate tokens only for RequestOutput
1387
+ n = len(output.outputs)
1388
assert output.prompt_token_ids is not None
1388
- total_in_toks += len(output.prompt_token_ids)
1389
+ total_in_toks += len(output.prompt_token_ids) * n
1390
in_spd = total_in_toks / pbar.format_dict["elapsed"]
1391
total_out_toks += sum(
1392
len(stp.token_ids) for stp in output.outputs)
@@ -1394,7 +1395,7 @@ class LLM:
1395
pbar.postfix = (
1396
f"est. speed input: {in_spd:.2f} toks/s, "
1397
f"output: {out_spd:.2f} toks/s")
1397
- pbar.update(len(output.outputs))
1398
+ pbar.update(n)
1399
else:
1400
pbar.update(1)
1401