@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 63e835cbcc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix progress bar and allow HTTPS in `benchmark_serving.py` (#2552)

Harry Mellor committed Jan 22, 2024 at 22:40 UTC 63e835cbccec62cc34ed27a6133d9a7f4af4a068
1 file changed +9 -3
benchmarks/benchmark_serving.py
+9 -3
@@ -101,6 +101,7 @@ async def send_request(
101 output_len: int,
102 best_of: int,
103 use_beam_search: bool,
104 + pbar: tqdm
105 ) -> None:
106 request_start_time = time.perf_counter()
107
@@ -151,6 +152,8 @@ async def send_request(
152 request_end_time = time.perf_counter()
153 request_latency = request_end_time - request_start_time
154 REQUEST_LATENCY.append((prompt_len, output_len, request_latency))
155 + pbar.update(1)
156 +
157
158
159 async def benchmark(
@@ -163,13 +166,15 @@ async def benchmark(
166 request_rate: float,
167 ) -> None:
168 tasks: List[asyncio.Task] = []
169 + pbar = tqdm(total=len(input_requests))
170 async for request in get_request(input_requests, request_rate):
171 prompt, prompt_len, output_len = request
172 task = asyncio.create_task(
173 send_request(backend, model, api_url, prompt, prompt_len,
170 - output_len, best_of, use_beam_search))
174 + output_len, best_of, use_beam_search, pbar))
175 tasks.append(task)
172 - await tqdm.gather(*tasks)
176 + await asyncio.gather(*tasks)
177 + pbar.close()
178
179
180 def main(args: argparse.Namespace):
@@ -177,7 +182,7 @@ def main(args: argparse.Namespace):
182 random.seed(args.seed)
183 np.random.seed(args.seed)
184
180 - api_url = f"http://{args.host}:{args.port}{args.endpoint}"
185 + api_url = f"{args.protocol}://{args.host}:{args.port}{args.endpoint}"
186 tokenizer = get_tokenizer(args.tokenizer,
187 trust_remote_code=args.trust_remote_code)
188 input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
@@ -212,6 +217,7 @@ if __name__ == "__main__":
217 type=str,
218 default="vllm",
219 choices=["vllm", "tgi"])
220 + parser.add_argument("--protocol", type=str, default="http", choices=["http", "https"])
221 parser.add_argument("--host", type=str, default="localhost")
222 parser.add_argument("--port", type=int, default=8000)
223 parser.add_argument("--endpoint", type=str, default="/generate")