A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Remove max_num_seqs in latency benchmark script (#1855)
Woosuk Kwon committed
Nov 30, 2023 at 00:00 UTC
51d3cb951d5666b518a937f90477fe131d61c687
1 file changed
-1
benchmarks/benchmark_latency.py
-1
@@ -19,7 +19,6 @@ def main(args: argparse.Namespace):
19
tokenizer=args.tokenizer,
20
quantization=args.quantization,
21
tensor_parallel_size=args.tensor_parallel_size,
22
- max_num_seqs=args.batch_size,
22
trust_remote_code=args.trust_remote_code,
23
dtype=args.dtype,
24
)