@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5a10eb0ef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Added `dtype` arg to benchmarks (#1228)

kg6-sleipnir committed Oct 1, 2023 at 00:04 UTC b5a10eb0ef68f45c7dbdef2917e02bebca780d1a
2 files changed +22 -1
benchmarks/benchmark_latency.py
+10
@@ -23,6 +23,7 @@ def main(args: argparse.Namespace):
23 max_num_seqs=args.batch_size,
24 max_num_batched_tokens=args.batch_size * args.input_len,
25 trust_remote_code=args.trust_remote_code,
26 + dtype=args.dtype,
27 )
28
29 sampling_params = SamplingParams(
@@ -87,5 +88,14 @@ if __name__ == '__main__':
88 parser.add_argument('--trust-remote-code',
89 action='store_true',
90 help='trust remote code from huggingface')
91 + parser.add_argument(
92 + '--dtype',
93 + type=str,
94 + default='auto',
95 + choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
96 + help='data type for model weights and activations. '
97 + 'The "auto" option will use FP16 precision '
98 + 'for FP32 and FP16 models, and BF16 precision '
99 + 'for BF16 models.')
100 args = parser.parse_args()
101 main(args)
benchmarks/benchmark_throughput.py
+12 -1
@@ -64,6 +64,7 @@ def run_vllm(
64 n: int,
65 use_beam_search: bool,
66 trust_remote_code: bool,
67 + dtype: str,
68 ) -> float:
69 llm = LLM(
70 model=model,
@@ -72,6 +73,7 @@ def run_vllm(
73 tensor_parallel_size=tensor_parallel_size,
74 seed=seed,
75 trust_remote_code=trust_remote_code,
76 + dtype=dtype,
77 )
78
79 # Add the requests to the engine.
@@ -171,7 +173,7 @@ def main(args: argparse.Namespace):
173 elapsed_time = run_vllm(requests, args.model, args.tokenizer,
174 args.quantization, args.tensor_parallel_size,
175 args.seed, args.n, args.use_beam_search,
174 - args.trust_remote_code)
176 + args.trust_remote_code, args.dtype)
177 elif args.backend == "hf":
178 assert args.tensor_parallel_size == 1
179 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -219,6 +221,15 @@ if __name__ == "__main__":
221 parser.add_argument('--trust-remote-code',
222 action='store_true',
223 help='trust remote code from huggingface')
224 + parser.add_argument(
225 + '--dtype',
226 + type=str,
227 + default='auto',
228 + choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
229 + help='data type for model weights and activations. '
230 + 'The "auto" option will use FP16 precision '
231 + 'for FP32 and FP16 models, and BF16 precision '
232 + 'for BF16 models.')
233 args = parser.parse_args()
234
235 if args.backend == "vllm":