@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1956931436
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc] add the "download-dir" option to the latency/throughput benchmarks (#3621)

AmadeusChan committed Mar 27, 2024 at 16:39 UTC 195693143634e7cf3841cc33cdbf0906f3b1af3d
2 files changed +32 -19
benchmarks/benchmark_latency.py
+16 -12
@@ -16,18 +16,17 @@ def main(args: argparse.Namespace):
16
17 # NOTE(woosuk): If the request cannot be processed in a single batch,
18 # the engine will automatically process the request in multiple batches.
19 - llm = LLM(
20 - model=args.model,
21 - tokenizer=args.tokenizer,
22 - quantization=args.quantization,
23 - tensor_parallel_size=args.tensor_parallel_size,
24 - trust_remote_code=args.trust_remote_code,
25 - dtype=args.dtype,
26 - enforce_eager=args.enforce_eager,
27 - kv_cache_dtype=args.kv_cache_dtype,
28 - device=args.device,
29 - ray_workers_use_nsight=args.ray_workers_use_nsight,
30 - )
19 + llm = LLM(model=args.model,
20 + tokenizer=args.tokenizer,
21 + quantization=args.quantization,
22 + tensor_parallel_size=args.tensor_parallel_size,
23 + trust_remote_code=args.trust_remote_code,
24 + dtype=args.dtype,
25 + enforce_eager=args.enforce_eager,
26 + kv_cache_dtype=args.kv_cache_dtype,
27 + device=args.device,
28 + ray_workers_use_nsight=args.ray_workers_use_nsight,
29 + download_dir=args.download_dir)
30
31 sampling_params = SamplingParams(
32 n=args.n,
@@ -151,5 +150,10 @@ if __name__ == '__main__':
150 action='store_true',
151 help="If specified, use nsight to profile ray workers",
152 )
153 + parser.add_argument('--download-dir',
154 + type=str,
155 + default=None,
156 + help='directory to download and load the weights, '
157 + 'default to the default cache dir of huggingface')
158 args = parser.parse_args()
159 main(args)
benchmarks/benchmark_throughput.py
+16 -7
@@ -75,6 +75,7 @@ def run_vllm(
75 device: str,
76 enable_prefix_caching: bool,
77 gpu_memory_utilization: float = 0.9,
78 + download_dir: Optional[str] = None,
79 ) -> float:
80 from vllm import LLM, SamplingParams
81 llm = LLM(model=model,
@@ -89,7 +90,8 @@ def run_vllm(
90 enforce_eager=enforce_eager,
91 kv_cache_dtype=kv_cache_dtype,
92 device=device,
92 - enable_prefix_caching=enable_prefix_caching)
93 + enable_prefix_caching=enable_prefix_caching,
94 + download_dir=download_dir)
95
96 # Add the requests to the engine.
97 for prompt, _, output_len in requests:
@@ -208,12 +210,14 @@ def main(args: argparse.Namespace):
210 args.output_len)
211
212 if args.backend == "vllm":
211 - elapsed_time = run_vllm(
212 - requests, args.model, args.tokenizer, args.quantization,
213 - args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
214 - args.trust_remote_code, args.dtype, args.max_model_len,
215 - args.enforce_eager, args.kv_cache_dtype, args.device,
216 - args.enable_prefix_caching, args.gpu_memory_utilization)
213 + elapsed_time = run_vllm(requests, args.model, args.tokenizer,
214 + args.quantization, args.tensor_parallel_size,
215 + args.seed, args.n, args.use_beam_search,
216 + args.trust_remote_code, args.dtype,
217 + args.max_model_len, args.enforce_eager,
218 + args.kv_cache_dtype, args.device,
219 + args.enable_prefix_caching,
220 + args.gpu_memory_utilization, args.download_dir)
221 elif args.backend == "hf":
222 assert args.tensor_parallel_size == 1
223 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -314,6 +318,11 @@ if __name__ == "__main__":
318 "--enable-prefix-caching",
319 action='store_true',
320 help="enable automatic prefix caching for vLLM backend.")
321 + parser.add_argument('--download-dir',
322 + type=str,
323 + default=None,
324 + help='directory to download and load the weights, '
325 + 'default to the default cache dir of huggingface')
326 args = parser.parse_args()
327 if args.tokenizer is None:
328 args.tokenizer = args.model