@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d8714530d1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc]Add param max-model-len in benchmark_latency.py (#5629)

DearPlanet committed Jun 19, 2024 at 18:19 UTC d8714530d11603a159a46ea0dde299f95807cfde
1 file changed +7
benchmarks/benchmark_latency.py
+7
@@ -29,6 +29,7 @@ def main(args: argparse.Namespace):
29 tensor_parallel_size=args.tensor_parallel_size,
30 trust_remote_code=args.trust_remote_code,
31 dtype=args.dtype,
32 + max_model_len=args.max_model_len,
33 enforce_eager=args.enforce_eager,
34 kv_cache_dtype=args.kv_cache_dtype,
35 quantization_param_path=args.quantization_param_path,
@@ -150,6 +151,12 @@ if __name__ == '__main__':
151 parser.add_argument('--trust-remote-code',
152 action='store_true',
153 help='trust remote code from huggingface')
154 + parser.add_argument(
155 + '--max-model-len',
156 + type=int,
157 + default=None,
158 + help='Maximum length of a sequence (including prompt and output). '
159 + 'If None, will be derived from the model.')
160 parser.add_argument(
161 '--dtype',
162 type=str,