@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ece1ae829
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Minor Fix] Fix comments in benchmark_serving (#3252)

TianYu GUO committed Mar 8, 2024 at 14:22 UTC 1ece1ae829dcbc4b1b19b3e2d3042457615e862f
1 file changed +3 -3
benchmarks/benchmark_serving.py
+3 -3
@@ -12,7 +12,7 @@ On the server side, run one of the following commands:
12 On the client side, run:
13 python benchmarks/benchmark_serving.py \
14 --backend <backend> \
15 - --tokenizer <your_model> --dataset <target_dataset> \
15 + --model <your_model> --dataset <target_dataset> \
16 --request-rate <request_rate>
17 """
18 import argparse
@@ -171,10 +171,10 @@ async def benchmark(
171 else:
172 raise ValueError(f"Unknown backend: {backend}")
173
174 - pbar = None if disable_tqdm else tqdm(total=len(input_requests))
175 -
174 print(f"Traffic request rate: {request_rate}")
175
176 + pbar = None if disable_tqdm else tqdm(total=len(input_requests))
177 +
178 benchmark_start_time = time.perf_counter()
179 tasks = []
180 async for request in get_request(input_requests, request_rate):