A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Misc]Add param max-model-len in benchmark_latency.py (#5629)
DearPlanet committed
Jun 19, 2024 at 18:19 UTC
d8714530d11603a159a46ea0dde299f95807cfde
1 file changed
+7
benchmarks/benchmark_latency.py
+7
@@ -29,6 +29,7 @@ def main(args: argparse.Namespace):
29
tensor_parallel_size=args.tensor_parallel_size,
30
trust_remote_code=args.trust_remote_code,
31
dtype=args.dtype,
32
+ max_model_len=args.max_model_len,
33
enforce_eager=args.enforce_eager,
34
kv_cache_dtype=args.kv_cache_dtype,
35
quantization_param_path=args.quantization_param_path,
@@ -150,6 +151,12 @@ if __name__ == '__main__':
151
parser.add_argument('--trust-remote-code',
152
action='store_true',
153
help='trust remote code from huggingface')
154
+ parser.add_argument(
155
+ '--max-model-len',
156
+ type=int,
157
+ default=None,
158
+ help='Maximum length of a sequence (including prompt and output). '
159
+ 'If None, will be derived from the model.')
160
parser.add_argument(
161
'--dtype',
162
type=str,