@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 16bc0a098f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend] add tok/s speed metric to llm class when using tqdm (#4400)

Co-authored-by: Michael Goin <michael@neuralmagic.com>

Mahmoud Ashraf committed May 9, 2024 at 08:02 UTC 16bc0a098f6d34050637c3336183fb6966300dd5
1 file changed +12 -4
vllm/entrypoints/llm.py
+12 -4
@@ -238,17 +238,25 @@ class LLM:
238 # Initialize tqdm.
239 if use_tqdm:
240 num_requests = self.llm_engine.get_num_unfinished_requests()
241 - pbar = tqdm(total=num_requests,
242 - desc="Processed prompts",
243 - dynamic_ncols=True)
241 + pbar = tqdm(
242 + total=num_requests,
243 + desc="Processed prompts",
244 + dynamic_ncols=True,
245 + postfix=f"Generation Speed: {0:.2f} toks/s",
246 + )
247 # Run the engine.
248 outputs: List[RequestOutput] = []
249 + total_toks = 0
250 while self.llm_engine.has_unfinished_requests():
251 step_outputs = self.llm_engine.step()
252 for output in step_outputs:
253 if output.finished:
254 outputs.append(output)
255 if use_tqdm:
256 + total_toks += (sum(
257 + len(stp.token_ids) for stp in output.outputs))
258 + spd = total_toks / pbar.format_dict["elapsed"]
259 + pbar.postfix = f"Generation Speed: {spd:.2f} toks/s"
260 pbar.update(1)
261 if use_tqdm:
262 pbar.close()
@@ -256,4 +264,4 @@ class LLM:
264 # This is necessary because some requests may be finished earlier than
265 # its previous requests.
266 outputs = sorted(outputs, key=lambda x: int(x.request_id))
259 - return outputs
\ No newline at end of file
267 + return outputs