A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Frontend] Add progress reporting to run_batch.py (#8060)
Co-authored-by: Adam Lugowski <adam.lugowski@parasail.io>
Adam Lugowski committed
Sep 9, 2024 at 11:16 UTC
58fcc8545a149c9c5b1f91f417a68f5ba1fdabf3
1 file changed
+48
-6
vllm/entrypoints/openai/run_batch.py
+48
-6
@@ -1,9 +1,11 @@
1
import asyncio
2
from io import StringIO
3
-from typing import Awaitable, Callable, List
3
+from typing import Awaitable, Callable, List, Optional
4
5
import aiohttp
6
+import torch
7
from prometheus_client import start_http_server
8
+from tqdm import tqdm
9
10
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -78,6 +80,38 @@ def parse_args():
80
return parser.parse_args()
81
82
83
+# explicitly use pure text format, with a newline at the end
84
+# this makes it impossible to see the animation in the progress bar
85
+# but will avoid messing up with ray or multiprocessing, which wraps
86
+# each line of output with some prefix.
87
+_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
88
+
89
+
90
+class BatchProgressTracker:
91
+
92
+ def __init__(self):
93
+ self._total = 0
94
+ self._pbar: Optional[tqdm] = None
95
+
96
+ def submitted(self):
97
+ self._total += 1
98
+
99
+ def completed(self):
100
+ if self._pbar:
101
+ self._pbar.update()
102
+
103
+ def pbar(self) -> tqdm:
104
+ enable_tqdm = not torch.distributed.is_initialized(
105
+ ) or torch.distributed.get_rank() == 0
106
+ self._pbar = tqdm(total=self._total,
107
+ unit="req",
108
+ desc="Running batch",
109
+ mininterval=5,
110
+ disable=not enable_tqdm,
111
+ bar_format=_BAR_FORMAT)
112
+ return self._pbar
113
+
114
+
115
async def read_file(path_or_url: str) -> str:
116
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
117
async with aiohttp.ClientSession() as session, \
@@ -102,7 +136,8 @@ async def write_file(path_or_url: str, data: str) -> None:
136
137
138
async def run_request(serving_engine_func: Callable,
105
- request: BatchRequestInput) -> BatchRequestOutput:
139
+ request: BatchRequestInput,
140
+ tracker: BatchProgressTracker) -> BatchRequestOutput:
141
response = await serving_engine_func(request.body)
142
143
if isinstance(response, (ChatCompletionResponse, EmbeddingResponse)):
@@ -125,6 +160,7 @@ async def run_request(serving_engine_func: Callable,
160
else:
161
raise ValueError("Request must not be sent in stream mode")
162
163
+ tracker.completed()
164
return batch_output
165
166
@@ -164,6 +200,9 @@ async def main(args):
200
request_logger=request_logger,
201
)
202
203
+ tracker = BatchProgressTracker()
204
+ logger.info("Reading batch from %s...", args.input_file)
205
+
206
# Submit all requests in the file to the engine "concurrently".
207
response_futures: List[Awaitable[BatchRequestOutput]] = []
208
for request_json in (await read_file(args.input_file)).strip().split("\n"):
@@ -178,16 +217,19 @@ async def main(args):
217
if request.url == "/v1/chat/completions":
218
response_futures.append(
219
run_request(openai_serving_chat.create_chat_completion,
181
- request))
220
+ request, tracker))
221
+ tracker.submitted()
222
elif request.url == "/v1/embeddings":
223
response_futures.append(
184
- run_request(openai_serving_embedding.create_embedding,
185
- request))
224
+ run_request(openai_serving_embedding.create_embedding, request,
225
+ tracker))
226
+ tracker.submitted()
227
else:
228
raise ValueError("Only /v1/chat/completions and /v1/embeddings are"
229
"supported in the batch endpoint.")
230
190
- responses = await asyncio.gather(*response_futures)
231
+ with tracker.pbar():
232
+ responses = await asyncio.gather(*response_futures)
233
234
output_buffer = StringIO()
235
for response in responses: