A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Misc] add the "download-dir" option to the latency/throughput benchmarks (#3621)
AmadeusChan committed
Mar 27, 2024 at 16:39 UTC
195693143634e7cf3841cc33cdbf0906f3b1af3d
2 files changed
+32
-19
benchmarks/benchmark_latency.py
+16
-12
@@ -16,18 +16,17 @@ def main(args: argparse.Namespace):
16
17
# NOTE(woosuk): If the request cannot be processed in a single batch,
18
# the engine will automatically process the request in multiple batches.
19
- llm = LLM(
20
- model=args.model,
21
- tokenizer=args.tokenizer,
22
- quantization=args.quantization,
23
- tensor_parallel_size=args.tensor_parallel_size,
24
- trust_remote_code=args.trust_remote_code,
25
- dtype=args.dtype,
26
- enforce_eager=args.enforce_eager,
27
- kv_cache_dtype=args.kv_cache_dtype,
28
- device=args.device,
29
- ray_workers_use_nsight=args.ray_workers_use_nsight,
30
- )
19
+ llm = LLM(model=args.model,
20
+ tokenizer=args.tokenizer,
21
+ quantization=args.quantization,
22
+ tensor_parallel_size=args.tensor_parallel_size,
23
+ trust_remote_code=args.trust_remote_code,
24
+ dtype=args.dtype,
25
+ enforce_eager=args.enforce_eager,
26
+ kv_cache_dtype=args.kv_cache_dtype,
27
+ device=args.device,
28
+ ray_workers_use_nsight=args.ray_workers_use_nsight,
29
+ download_dir=args.download_dir)
30
31
sampling_params = SamplingParams(
32
n=args.n,
@@ -151,5 +150,10 @@ if __name__ == '__main__':
150
action='store_true',
151
help="If specified, use nsight to profile ray workers",
152
)
153
+ parser.add_argument('--download-dir',
154
+ type=str,
155
+ default=None,
156
+ help='directory to download and load the weights, '
157
+ 'default to the default cache dir of huggingface')
158
args = parser.parse_args()
159
main(args)
benchmarks/benchmark_throughput.py
+16
-7
@@ -75,6 +75,7 @@ def run_vllm(
75
device: str,
76
enable_prefix_caching: bool,
77
gpu_memory_utilization: float = 0.9,
78
+ download_dir: Optional[str] = None,
79
) -> float:
80
from vllm import LLM, SamplingParams
81
llm = LLM(model=model,
@@ -89,7 +90,8 @@ def run_vllm(
90
enforce_eager=enforce_eager,
91
kv_cache_dtype=kv_cache_dtype,
92
device=device,
92
- enable_prefix_caching=enable_prefix_caching)
93
+ enable_prefix_caching=enable_prefix_caching,
94
+ download_dir=download_dir)
95
96
# Add the requests to the engine.
97
for prompt, _, output_len in requests:
@@ -208,12 +210,14 @@ def main(args: argparse.Namespace):
210
args.output_len)
211
212
if args.backend == "vllm":
211
- elapsed_time = run_vllm(
212
- requests, args.model, args.tokenizer, args.quantization,
213
- args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
214
- args.trust_remote_code, args.dtype, args.max_model_len,
215
- args.enforce_eager, args.kv_cache_dtype, args.device,
216
- args.enable_prefix_caching, args.gpu_memory_utilization)
213
+ elapsed_time = run_vllm(requests, args.model, args.tokenizer,
214
+ args.quantization, args.tensor_parallel_size,
215
+ args.seed, args.n, args.use_beam_search,
216
+ args.trust_remote_code, args.dtype,
217
+ args.max_model_len, args.enforce_eager,
218
+ args.kv_cache_dtype, args.device,
219
+ args.enable_prefix_caching,
220
+ args.gpu_memory_utilization, args.download_dir)
221
elif args.backend == "hf":
222
assert args.tensor_parallel_size == 1
223
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -314,6 +318,11 @@ if __name__ == "__main__":
318
"--enable-prefix-caching",
319
action='store_true',
320
help="enable automatic prefix caching for vLLM backend.")
321
+ parser.add_argument('--download-dir',
322
+ type=str,
323
+ default=None,
324
+ help='directory to download and load the weights, '
325
+ 'default to the default cache dir of huggingface')
326
args = parser.parse_args()
327
if args.tokenizer is None:
328
args.tokenizer = args.model