A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix progress bar and allow HTTPS in `benchmark_serving.py` (#2552)
Harry Mellor committed
Jan 22, 2024 at 22:40 UTC
63e835cbccec62cc34ed27a6133d9a7f4af4a068
1 file changed
+9
-3
benchmarks/benchmark_serving.py
+9
-3
@@ -101,6 +101,7 @@ async def send_request(
101
output_len: int,
102
best_of: int,
103
use_beam_search: bool,
104
+ pbar: tqdm
105
) -> None:
106
request_start_time = time.perf_counter()
107
@@ -151,6 +152,8 @@ async def send_request(
152
request_end_time = time.perf_counter()
153
request_latency = request_end_time - request_start_time
154
REQUEST_LATENCY.append((prompt_len, output_len, request_latency))
155
+ pbar.update(1)
156
+
157
158
159
async def benchmark(
@@ -163,13 +166,15 @@ async def benchmark(
166
request_rate: float,
167
) -> None:
168
tasks: List[asyncio.Task] = []
169
+ pbar = tqdm(total=len(input_requests))
170
async for request in get_request(input_requests, request_rate):
171
prompt, prompt_len, output_len = request
172
task = asyncio.create_task(
173
send_request(backend, model, api_url, prompt, prompt_len,
170
- output_len, best_of, use_beam_search))
174
+ output_len, best_of, use_beam_search, pbar))
175
tasks.append(task)
172
- await tqdm.gather(*tasks)
176
+ await asyncio.gather(*tasks)
177
+ pbar.close()
178
179
180
def main(args: argparse.Namespace):
@@ -177,7 +182,7 @@ def main(args: argparse.Namespace):
182
random.seed(args.seed)
183
np.random.seed(args.seed)
184
180
- api_url = f"http://{args.host}:{args.port}{args.endpoint}"
185
+ api_url = f"{args.protocol}://{args.host}:{args.port}{args.endpoint}"
186
tokenizer = get_tokenizer(args.tokenizer,
187
trust_remote_code=args.trust_remote_code)
188
input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
@@ -212,6 +217,7 @@ if __name__ == "__main__":
217
type=str,
218
default="vllm",
219
choices=["vllm", "tgi"])
220
+ parser.add_argument("--protocol", type=str, default="http", choices=["http", "https"])
221
parser.add_argument("--host", type=str, default="localhost")
222
parser.add_argument("--port", type=int, default=8000)
223
parser.add_argument("--endpoint", type=str, default="/generate")