A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Frontend] Show progress bar for adding requests (#17525)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 1, 2025 at 20:15 UTC
1903c0b8a3a023b09869427719052c209aecb78c
1 file changed
+12
-2
vllm/entrypoints/llm.py
+12
-2
@@ -462,10 +462,12 @@ class LLM:
462
self._validate_and_add_requests(
463
prompts=parsed_prompts,
464
params=sampling_params,
465
+ use_tqdm=use_tqdm,
466
lora_request=lora_request,
467
prompt_adapter_request=prompt_adapter_request,
468
guided_options=guided_options_request,
468
- priority=priority)
469
+ priority=priority,
470
+ )
471
472
outputs = self._run_engine(use_tqdm=use_tqdm)
473
return self.engine_class.validate_outputs(outputs, RequestOutput)
@@ -957,6 +959,7 @@ class LLM:
959
self._validate_and_add_requests(
960
prompts=parsed_prompts,
961
params=pooling_params,
962
+ use_tqdm=use_tqdm,
963
lora_request=lora_request,
964
tokenization_kwargs=tokenization_kwargs,
965
prompt_adapter_request=prompt_adapter_request,
@@ -1127,6 +1130,7 @@ class LLM:
1130
self._validate_and_add_requests(
1131
prompts=parsed_prompts,
1132
params=pooling_params,
1133
+ use_tqdm=use_tqdm,
1134
lora_request=lora_request,
1135
prompt_adapter_request=prompt_adapter_request,
1136
)
@@ -1332,6 +1336,8 @@ class LLM:
1336
prompts: Union[PromptType, Sequence[PromptType]],
1337
params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
1338
Sequence[PoolingParams]],
1339
+ *,
1340
+ use_tqdm: bool,
1341
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
1342
prompt_adapter_request: Optional[PromptAdapterRequest],
1343
tokenization_kwargs: Optional[dict[str, Any]] = None,
@@ -1367,7 +1373,11 @@ class LLM:
1373
sp.output_kind = RequestOutputKind.FINAL_ONLY
1374
1375
# Add requests to the engine.
1370
- for i, prompt in enumerate(prompts):
1376
+ it = prompts
1377
+ if use_tqdm:
1378
+ it = tqdm(it, desc="Adding requests")
1379
+
1380
+ for i, prompt in enumerate(it):
1381
self._add_request(
1382
prompt,
1383
params[i] if isinstance(params, Sequence) else params,