A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Minor Fix] Fix comments in benchmark_serving (#3252)
TianYu GUO committed
Mar 8, 2024 at 14:22 UTC
1ece1ae829dcbc4b1b19b3e2d3042457615e862f
1 file changed
+3
-3
benchmarks/benchmark_serving.py
+3
-3
@@ -12,7 +12,7 @@ On the server side, run one of the following commands:
12
On the client side, run:
13
python benchmarks/benchmark_serving.py \
14
--backend <backend> \
15
- --tokenizer <your_model> --dataset <target_dataset> \
15
+ --model <your_model> --dataset <target_dataset> \
16
--request-rate <request_rate>
17
"""
18
import argparse
@@ -171,10 +171,10 @@ async def benchmark(
171
else:
172
raise ValueError(f"Unknown backend: {backend}")
173
174
- pbar = None if disable_tqdm else tqdm(total=len(input_requests))
175
-
174
print(f"Traffic request rate: {request_rate}")
175
176
+ pbar = None if disable_tqdm else tqdm(total=len(input_requests))
177
+
178
benchmark_start_time = time.perf_counter()
179
tasks = []
180
async for request in get_request(input_requests, request_rate):