A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Added `dtype` arg to benchmarks (#1228)
kg6-sleipnir committed
Oct 1, 2023 at 00:04 UTC
b5a10eb0ef68f45c7dbdef2917e02bebca780d1a
2 files changed
+22
-1
benchmarks/benchmark_latency.py
+10
@@ -23,6 +23,7 @@ def main(args: argparse.Namespace):
23
max_num_seqs=args.batch_size,
24
max_num_batched_tokens=args.batch_size * args.input_len,
25
trust_remote_code=args.trust_remote_code,
26
+ dtype=args.dtype,
27
)
28
29
sampling_params = SamplingParams(
@@ -87,5 +88,14 @@ if __name__ == '__main__':
88
parser.add_argument('--trust-remote-code',
89
action='store_true',
90
help='trust remote code from huggingface')
91
+ parser.add_argument(
92
+ '--dtype',
93
+ type=str,
94
+ default='auto',
95
+ choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
96
+ help='data type for model weights and activations. '
97
+ 'The "auto" option will use FP16 precision '
98
+ 'for FP32 and FP16 models, and BF16 precision '
99
+ 'for BF16 models.')
100
args = parser.parse_args()
101
main(args)
benchmarks/benchmark_throughput.py
+12
-1
@@ -64,6 +64,7 @@ def run_vllm(
64
n: int,
65
use_beam_search: bool,
66
trust_remote_code: bool,
67
+ dtype: str,
68
) -> float:
69
llm = LLM(
70
model=model,
@@ -72,6 +73,7 @@ def run_vllm(
73
tensor_parallel_size=tensor_parallel_size,
74
seed=seed,
75
trust_remote_code=trust_remote_code,
76
+ dtype=dtype,
77
)
78
79
# Add the requests to the engine.
@@ -171,7 +173,7 @@ def main(args: argparse.Namespace):
173
elapsed_time = run_vllm(requests, args.model, args.tokenizer,
174
args.quantization, args.tensor_parallel_size,
175
args.seed, args.n, args.use_beam_search,
174
- args.trust_remote_code)
176
+ args.trust_remote_code, args.dtype)
177
elif args.backend == "hf":
178
assert args.tensor_parallel_size == 1
179
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -219,6 +221,15 @@ if __name__ == "__main__":
221
parser.add_argument('--trust-remote-code',
222
action='store_true',
223
help='trust remote code from huggingface')
224
+ parser.add_argument(
225
+ '--dtype',
226
+ type=str,
227
+ default='auto',
228
+ choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
229
+ help='data type for model weights and activations. '
230
+ 'The "auto" option will use FP16 precision '
231
+ 'for FP32 and FP16 models, and BF16 precision '
232
+ 'for BF16 models.')
233
args = parser.parse_args()
234
235
if args.backend == "vllm":