A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend] add tok/s speed metric to llm class when using tqdm (#4400)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Mahmoud Ashraf committed
May 9, 2024 at 08:02 UTC
16bc0a098f6d34050637c3336183fb6966300dd5
1 file changed
+12
-4
vllm/entrypoints/llm.py
+12
-4
@@ -238,17 +238,25 @@ class LLM:
238
# Initialize tqdm.
239
if use_tqdm:
240
num_requests = self.llm_engine.get_num_unfinished_requests()
241
- pbar = tqdm(total=num_requests,
242
- desc="Processed prompts",
243
- dynamic_ncols=True)
241
+ pbar = tqdm(
242
+ total=num_requests,
243
+ desc="Processed prompts",
244
+ dynamic_ncols=True,
245
+ postfix=f"Generation Speed: {0:.2f} toks/s",
246
+ )
247
# Run the engine.
248
outputs: List[RequestOutput] = []
249
+ total_toks = 0
250
while self.llm_engine.has_unfinished_requests():
251
step_outputs = self.llm_engine.step()
252
for output in step_outputs:
253
if output.finished:
254
outputs.append(output)
255
if use_tqdm:
256
+ total_toks += (sum(
257
+ len(stp.token_ids) for stp in output.outputs))
258
+ spd = total_toks / pbar.format_dict["elapsed"]
259
+ pbar.postfix = f"Generation Speed: {spd:.2f} toks/s"
260
pbar.update(1)
261
if use_tqdm:
262
pbar.close()
@@ -256,4 +264,4 @@ class LLM:
264
# This is necessary because some requests may be finished earlier than
265
# its previous requests.
266
outputs = sorted(outputs, key=lambda x: int(x.request_id))
259
- return outputs
\ No newline at end of file
267
+ return outputs