A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Support max-model-len argument for throughput benchmark (#1858)
aisensiy committed
Dec 1, 2023 at 00:10 UTC
8d8c2f6ffe305bc25725800827208e851649e2d3
1 file changed
+10
-1
benchmarks/benchmark_throughput.py
+10
-1
@@ -69,6 +69,7 @@ def run_vllm(
69
use_beam_search: bool,
70
trust_remote_code: bool,
71
dtype: str,
72
+ max_model_len: Optional[int] = None,
73
) -> float:
74
from vllm import LLM, SamplingParams
75
llm = LLM(
@@ -79,6 +80,7 @@ def run_vllm(
80
seed=seed,
81
trust_remote_code=trust_remote_code,
82
dtype=dtype,
83
+ max_model_len=max_model_len,
84
)
85
86
# Add the requests to the engine.
@@ -201,7 +203,8 @@ def main(args: argparse.Namespace):
203
elapsed_time = run_vllm(requests, args.model, args.tokenizer,
204
args.quantization, args.tensor_parallel_size,
205
args.seed, args.n, args.use_beam_search,
204
- args.trust_remote_code, args.dtype)
206
+ args.trust_remote_code, args.dtype,
207
+ args.max_model_len)
208
elif args.backend == "hf":
209
assert args.tensor_parallel_size == 1
210
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -261,6 +264,12 @@ if __name__ == "__main__":
264
parser.add_argument('--trust-remote-code',
265
action='store_true',
266
help='trust remote code from huggingface')
267
+ parser.add_argument(
268
+ '--max-model-len',
269
+ type=int,
270
+ default=None,
271
+ help='Maximum length of a sequence (including prompt and output). '
272
+ 'If None, will be derived from the model.')
273
parser.add_argument(
274
'--dtype',
275
type=str,