@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 58fcc8545a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend] Add progress reporting to run_batch.py (#8060)

Co-authored-by: Adam Lugowski <adam.lugowski@parasail.io>

Adam Lugowski committed Sep 9, 2024 at 11:16 UTC 58fcc8545a149c9c5b1f91f417a68f5ba1fdabf3
1 file changed +48 -6
vllm/entrypoints/openai/run_batch.py
+48 -6
@@ -1,9 +1,11 @@
1 import asyncio
2 from io import StringIO
3 -from typing import Awaitable, Callable, List
3 +from typing import Awaitable, Callable, List, Optional
4
5 import aiohttp
6 +import torch
7 from prometheus_client import start_http_server
8 +from tqdm import tqdm
9
10 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -78,6 +80,38 @@ def parse_args():
80 return parser.parse_args()
81
82
83 +# explicitly use pure text format, with a newline at the end
84 +# this makes it impossible to see the animation in the progress bar
85 +# but will avoid messing up with ray or multiprocessing, which wraps
86 +# each line of output with some prefix.
87 +_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
88 +
89 +
90 +class BatchProgressTracker:
91 +
92 + def __init__(self):
93 + self._total = 0
94 + self._pbar: Optional[tqdm] = None
95 +
96 + def submitted(self):
97 + self._total += 1
98 +
99 + def completed(self):
100 + if self._pbar:
101 + self._pbar.update()
102 +
103 + def pbar(self) -> tqdm:
104 + enable_tqdm = not torch.distributed.is_initialized(
105 + ) or torch.distributed.get_rank() == 0
106 + self._pbar = tqdm(total=self._total,
107 + unit="req",
108 + desc="Running batch",
109 + mininterval=5,
110 + disable=not enable_tqdm,
111 + bar_format=_BAR_FORMAT)
112 + return self._pbar
113 +
114 +
115 async def read_file(path_or_url: str) -> str:
116 if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
117 async with aiohttp.ClientSession() as session, \
@@ -102,7 +136,8 @@ async def write_file(path_or_url: str, data: str) -> None:
136
137
138 async def run_request(serving_engine_func: Callable,
105 - request: BatchRequestInput) -> BatchRequestOutput:
139 + request: BatchRequestInput,
140 + tracker: BatchProgressTracker) -> BatchRequestOutput:
141 response = await serving_engine_func(request.body)
142
143 if isinstance(response, (ChatCompletionResponse, EmbeddingResponse)):
@@ -125,6 +160,7 @@ async def run_request(serving_engine_func: Callable,
160 else:
161 raise ValueError("Request must not be sent in stream mode")
162
163 + tracker.completed()
164 return batch_output
165
166
@@ -164,6 +200,9 @@ async def main(args):
200 request_logger=request_logger,
201 )
202
203 + tracker = BatchProgressTracker()
204 + logger.info("Reading batch from %s...", args.input_file)
205 +
206 # Submit all requests in the file to the engine "concurrently".
207 response_futures: List[Awaitable[BatchRequestOutput]] = []
208 for request_json in (await read_file(args.input_file)).strip().split("\n"):
@@ -178,16 +217,19 @@ async def main(args):
217 if request.url == "/v1/chat/completions":
218 response_futures.append(
219 run_request(openai_serving_chat.create_chat_completion,
181 - request))
220 + request, tracker))
221 + tracker.submitted()
222 elif request.url == "/v1/embeddings":
223 response_futures.append(
184 - run_request(openai_serving_embedding.create_embedding,
185 - request))
224 + run_request(openai_serving_embedding.create_embedding, request,
225 + tracker))
226 + tracker.submitted()
227 else:
228 raise ValueError("Only /v1/chat/completions and /v1/embeddings are"
229 "supported in the batch endpoint.")
230
190 - responses = await asyncio.gather(*response_futures)
231 + with tracker.pbar():
232 + responses = await asyncio.gather(*response_futures)
233
234 output_buffer = StringIO()
235 for response in responses: