@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cb6fdaa0a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc] Make benchmarks use EngineArgs (#9529)

Jeremy Arnold committed Oct 22, 2024 at 17:40 UTC cb6fdaa0a0b31985df4fa3ddf069c022c1faacb9
4 files changed +38 -512
benchmarks/benchmark_latency.py
+7 -148
@@ -1,5 +1,6 @@
1 """Benchmark the latency of processing a single batch of requests."""
2 import argparse
3 +import dataclasses
4 import json
5 import time
6 from pathlib import Path
@@ -10,43 +11,19 @@ import torch
11 from tqdm import tqdm
12
13 from vllm import LLM, SamplingParams
13 -from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14 +from vllm.engine.arg_utils import EngineArgs
15 from vllm.inputs import PromptType
15 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 from vllm.utils import FlexibleArgumentParser
17
18
19 def main(args: argparse.Namespace):
20 print(args)
21
22 + engine_args = EngineArgs.from_cli_args(args)
23 +
24 # NOTE(woosuk): If the request cannot be processed in a single batch,
25 # the engine will automatically process the request in multiple batches.
24 - llm = LLM(
25 - model=args.model,
26 - speculative_model=args.speculative_model,
27 - num_speculative_tokens=args.num_speculative_tokens,
28 - speculative_draft_tensor_parallel_size=\
29 - args.speculative_draft_tensor_parallel_size,
30 - tokenizer=args.tokenizer,
31 - quantization=args.quantization,
32 - tensor_parallel_size=args.tensor_parallel_size,
33 - trust_remote_code=args.trust_remote_code,
34 - dtype=args.dtype,
35 - max_model_len=args.max_model_len,
36 - enforce_eager=args.enforce_eager,
37 - kv_cache_dtype=args.kv_cache_dtype,
38 - quantization_param_path=args.quantization_param_path,
39 - device=args.device,
40 - ray_workers_use_nsight=args.ray_workers_use_nsight,
41 - enable_chunked_prefill=args.enable_chunked_prefill,
42 - download_dir=args.download_dir,
43 - block_size=args.block_size,
44 - gpu_memory_utilization=args.gpu_memory_utilization,
45 - load_format=args.load_format,
46 - distributed_executor_backend=args.distributed_executor_backend,
47 - otlp_traces_endpoint=args.otlp_traces_endpoint,
48 - enable_prefix_caching=args.enable_prefix_caching,
49 - )
26 + llm = LLM(**dataclasses.asdict(engine_args))
27
28 sampling_params = SamplingParams(
29 n=args.n,
@@ -125,19 +102,6 @@ if __name__ == '__main__':
102 parser = FlexibleArgumentParser(
103 description='Benchmark the latency of processing a single batch of '
104 'requests till completion.')
128 - parser.add_argument('--model', type=str, default='facebook/opt-125m')
129 - parser.add_argument('--speculative-model', type=str, default=None)
130 - parser.add_argument('--num-speculative-tokens', type=int, default=None)
131 - parser.add_argument('--speculative-draft-tensor-parallel-size',
132 - '-spec-draft-tp',
133 - type=int,
134 - default=None)
135 - parser.add_argument('--tokenizer', type=str, default=None)
136 - parser.add_argument('--quantization',
137 - '-q',
138 - choices=[*QUANTIZATION_METHODS, None],
139 - default=None)
140 - parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
105 parser.add_argument('--input-len', type=int, default=32)
106 parser.add_argument('--output-len', type=int, default=128)
107 parser.add_argument('--batch-size', type=int, default=8)
@@ -154,45 +118,6 @@ if __name__ == '__main__':
118 type=int,
119 default=30,
120 help='Number of iterations to run.')
157 - parser.add_argument('--trust-remote-code',
158 - action='store_true',
159 - help='trust remote code from huggingface')
160 - parser.add_argument(
161 - '--max-model-len',
162 - type=int,
163 - default=None,
164 - help='Maximum length of a sequence (including prompt and output). '
165 - 'If None, will be derived from the model.')
166 - parser.add_argument(
167 - '--dtype',
168 - type=str,
169 - default='auto',
170 - choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
171 - help='data type for model weights and activations. '
172 - 'The "auto" option will use FP16 precision '
173 - 'for FP32 and FP16 models, and BF16 precision '
174 - 'for BF16 models.')
175 - parser.add_argument('--enforce-eager',
176 - action='store_true',
177 - help='enforce eager mode and disable CUDA graph')
178 - parser.add_argument(
179 - '--kv-cache-dtype',
180 - type=str,
181 - choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
182 - default="auto",
183 - help='Data type for kv cache storage. If "auto", will use model '
184 - 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
185 - 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
186 - parser.add_argument(
187 - '--quantization-param-path',
188 - type=str,
189 - default=None,
190 - help='Path to the JSON file containing the KV cache scaling factors. '
191 - 'This should generally be supplied, when KV cache dtype is FP8. '
192 - 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
193 - 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
194 - 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
195 - 'instead supported for common inference criteria.')
121 parser.add_argument(
122 '--profile',
123 action='store_true',
@@ -203,78 +128,12 @@ if __name__ == '__main__':
128 default=None,
129 help=('path to save the pytorch profiler output. Can be visualized '
130 'with ui.perfetto.dev or Tensorboard.'))
206 - parser.add_argument("--device",
207 - type=str,
208 - default="auto",
209 - choices=DEVICE_OPTIONS,
210 - help='device type for vLLM execution')
211 - parser.add_argument('--block-size',
212 - type=int,
213 - default=16,
214 - help='block size of key/value cache')
215 - parser.add_argument(
216 - '--enable-chunked-prefill',
217 - action='store_true',
218 - help='If True, the prefill requests can be chunked based on the '
219 - 'max_num_batched_tokens')
220 - parser.add_argument("--enable-prefix-caching",
221 - action='store_true',
222 - help="Enable automatic prefix caching")
223 - parser.add_argument(
224 - "--ray-workers-use-nsight",
225 - action='store_true',
226 - help="If specified, use nsight to profile ray workers",
227 - )
228 - parser.add_argument('--download-dir',
229 - type=str,
230 - default=None,
231 - help='directory to download and load the weights, '
232 - 'default to the default cache dir of huggingface')
131 parser.add_argument(
132 '--output-json',
133 type=str,
134 default=None,
135 help='Path to save the latency results in JSON format.')
238 - parser.add_argument('--gpu-memory-utilization',
239 - type=float,
240 - default=0.9,
241 - help='the fraction of GPU memory to be used for '
242 - 'the model executor, which can range from 0 to 1.'
243 - 'If unspecified, will use the default value of 0.9.')
244 - parser.add_argument(
245 - '--load-format',
246 - type=str,
247 - default=EngineArgs.load_format,
248 - choices=[
249 - 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
250 - 'bitsandbytes'
251 - ],
252 - help='The format of the model weights to load.\n\n'
253 - '* "auto" will try to load the weights in the safetensors format '
254 - 'and fall back to the pytorch bin format if safetensors format '
255 - 'is not available.\n'
256 - '* "pt" will load the weights in the pytorch bin format.\n'
257 - '* "safetensors" will load the weights in the safetensors format.\n'
258 - '* "npcache" will load the weights in pytorch format and store '
259 - 'a numpy cache to speed up the loading.\n'
260 - '* "dummy" will initialize the weights with random values, '
261 - 'which is mainly for profiling.\n'
262 - '* "tensorizer" will load the weights using tensorizer from '
263 - 'CoreWeave. See the Tensorize vLLM Model script in the Examples'
264 - 'section for more information.\n'
265 - '* "bitsandbytes" will load the weights using bitsandbytes '
266 - 'quantization.\n')
267 - parser.add_argument(
268 - '--distributed-executor-backend',
269 - choices=['ray', 'mp'],
270 - default=None,
271 - help='Backend to use for distributed serving. When more than 1 GPU '
272 - 'is used, will be automatically set to "ray" if installed '
273 - 'or "mp" (multiprocessing) otherwise.')
274 - parser.add_argument(
275 - '--otlp-traces-endpoint',
276 - type=str,
277 - default=None,
278 - help='Target URL to which OpenTelemetry traces will be sent.')
136 +
137 + parser = EngineArgs.add_cli_args(parser)
138 args = parser.parse_args()
139 main(args)
benchmarks/benchmark_prefix_caching.py
+7 -17
@@ -25,6 +25,7 @@ ShareGPT example usage:
25 --input-length-range 128:256
26 """
27
28 +import dataclasses
29 import json
30 import random
31 import time
@@ -33,6 +34,7 @@ from typing import List, Optional, Tuple
34 from transformers import PreTrainedTokenizerBase
35
36 from vllm import LLM, SamplingParams
37 +from vllm.engine.arg_utils import EngineArgs
38 from vllm.utils import FlexibleArgumentParser
39
40 try:
@@ -129,12 +131,9 @@ def main(args):
131 filtered_datasets = [(PROMPT, prompt_len, args.output_len)
132 ] * args.num_prompts
133
132 - llm = LLM(model=args.model,
133 - tokenizer_mode='auto',
134 - trust_remote_code=True,
135 - enforce_eager=True,
136 - tensor_parallel_size=args.tensor_parallel_size,
137 - enable_prefix_caching=args.enable_prefix_caching)
134 + engine_args = EngineArgs.from_cli_args(args)
135 +
136 + llm = LLM(**dataclasses.asdict(engine_args))
137
138 sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
139
@@ -162,18 +161,11 @@ if __name__ == "__main__":
161 parser = FlexibleArgumentParser(
162 description=
163 'Benchmark the performance with or without automatic prefix caching.')
165 - parser.add_argument('--model',
166 - type=str,
167 - default='baichuan-inc/Baichuan2-13B-Chat')
164 parser.add_argument("--dataset-path",
165 type=str,
166 default=None,
167 help="Path to the dataset.")
172 - parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
168 parser.add_argument('--output-len', type=int, default=10)
174 - parser.add_argument('--enable-prefix-caching',
175 - action='store_true',
176 - help='enable prefix caching')
169 parser.add_argument('--num-prompts',
170 type=int,
171 default=1,
@@ -190,9 +182,7 @@ if __name__ == "__main__":
182 default='128:256',
183 help='Range of input lengths for sampling prompts,'
184 'specified as "min:max" (e.g., "128:256").')
193 - parser.add_argument("--seed",
194 - type=int,
195 - default=0,
196 - help='Random seed for reproducibility')
185 +
186 + parser = EngineArgs.add_cli_args(parser)
187 args = parser.parse_args()
188 main(args)
benchmarks/benchmark_prioritization.py
+9 -125
@@ -1,5 +1,6 @@
1 """Benchmark offline prioritization."""
2 import argparse
3 +import dataclasses
4 import json
5 import random
6 import time
@@ -7,7 +8,8 @@ from typing import List, Optional, Tuple
8
9 from transformers import AutoTokenizer, PreTrainedTokenizerBase
10
10 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11 +from vllm.engine.arg_utils import EngineArgs
12 +from vllm.utils import FlexibleArgumentParser
13
14
15 def sample_requests(
@@ -62,46 +64,11 @@ def sample_requests(
64
65 def run_vllm(
66 requests: List[Tuple[str, int, int]],
65 - model: str,
66 - tokenizer: str,
67 - quantization: Optional[str],
68 - tensor_parallel_size: int,
69 - seed: int,
67 n: int,
71 - trust_remote_code: bool,
72 - dtype: str,
73 - max_model_len: Optional[int],
74 - enforce_eager: bool,
75 - kv_cache_dtype: str,
76 - quantization_param_path: Optional[str],
77 - device: str,
78 - enable_prefix_caching: bool,
79 - enable_chunked_prefill: bool,
80 - max_num_batched_tokens: int,
81 - gpu_memory_utilization: float = 0.9,
82 - download_dir: Optional[str] = None,
68 + engine_args: EngineArgs,
69 ) -> float:
70 from vllm import LLM, SamplingParams
85 - llm = LLM(
86 - model=model,
87 - tokenizer=tokenizer,
88 - quantization=quantization,
89 - tensor_parallel_size=tensor_parallel_size,
90 - seed=seed,
91 - trust_remote_code=trust_remote_code,
92 - dtype=dtype,
93 - max_model_len=max_model_len,
94 - gpu_memory_utilization=gpu_memory_utilization,
95 - enforce_eager=enforce_eager,
96 - kv_cache_dtype=kv_cache_dtype,
97 - quantization_param_path=quantization_param_path,
98 - device=device,
99 - enable_prefix_caching=enable_prefix_caching,
100 - download_dir=download_dir,
101 - enable_chunked_prefill=enable_chunked_prefill,
102 - max_num_batched_tokens=max_num_batched_tokens,
103 - disable_log_stats=False,
104 - )
71 + llm = LLM(**dataclasses.asdict(engine_args))
72
73 # Add the requests to the engine.
74 prompts = []
@@ -142,16 +109,8 @@ def main(args: argparse.Namespace):
109 args.output_len)
110
111 if args.backend == "vllm":
145 - elapsed_time = run_vllm(requests, args.model, args.tokenizer,
146 - args.quantization, args.tensor_parallel_size,
147 - args.seed, args.n, args.trust_remote_code,
148 - args.dtype, args.max_model_len,
149 - args.enforce_eager, args.kv_cache_dtype,
150 - args.quantization_param_path, args.device,
151 - args.enable_prefix_caching,
152 - args.enable_chunked_prefill,
153 - args.max_num_batched_tokens,
154 - args.gpu_memory_utilization, args.download_dir)
112 + elapsed_time = run_vllm(requests, args.n,
113 + EngineArgs.from_cli_args(args))
114 else:
115 raise ValueError(f"Unknown backend: {args.backend}")
116 total_num_tokens = sum(prompt_len + output_len
@@ -173,7 +132,7 @@ def main(args: argparse.Namespace):
132
133
134 if __name__ == "__main__":
176 - parser = argparse.ArgumentParser(description="Benchmark the throughput.")
135 + parser = FlexibleArgumentParser(description="Benchmark the throughput.")
136 parser.add_argument("--backend",
137 type=str,
138 choices=["vllm", "hf", "mii"],
@@ -191,13 +150,6 @@ if __name__ == "__main__":
150 default=None,
151 help="Output length for each request. Overrides the "
152 "output length from the dataset.")
194 - parser.add_argument("--model", type=str, default="facebook/opt-125m")
195 - parser.add_argument("--tokenizer", type=str, default=None)
196 - parser.add_argument('--quantization',
197 - '-q',
198 - choices=[*QUANTIZATION_METHODS, None],
199 - default=None)
200 - parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
153 parser.add_argument("--n",
154 type=int,
155 default=1,
@@ -206,81 +158,13 @@ if __name__ == "__main__":
158 type=int,
159 default=200,
160 help="Number of prompts to process.")
209 - parser.add_argument("--seed", type=int, default=0)
210 - parser.add_argument('--trust-remote-code',
211 - action='store_true',
212 - help='trust remote code from huggingface')
213 - parser.add_argument(
214 - '--max-model-len',
215 - type=int,
216 - default=None,
217 - help='Maximum length of a sequence (including prompt and output). '
218 - 'If None, will be derived from the model.')
219 - parser.add_argument(
220 - '--dtype',
221 - type=str,
222 - default='auto',
223 - choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
224 - help='data type for model weights and activations. '
225 - 'The "auto" option will use FP16 precision '
226 - 'for FP32 and FP16 models, and BF16 precision '
227 - 'for BF16 models.')
228 - parser.add_argument('--gpu-memory-utilization',
229 - type=float,
230 - default=0.9,
231 - help='the fraction of GPU memory to be used for '
232 - 'the model executor, which can range from 0 to 1.'
233 - 'If unspecified, will use the default value of 0.9.')
234 - parser.add_argument("--enforce-eager",
235 - action="store_true",
236 - help="enforce eager execution")
237 - parser.add_argument(
238 - '--kv-cache-dtype',
239 - type=str,
240 - choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
241 - default="auto",
242 - help='Data type for kv cache storage. If "auto", will use model '
243 - 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
244 - 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
245 - parser.add_argument(
246 - '--quantization-param-path',
247 - type=str,
248 - default=None,
249 - help='Path to the JSON file containing the KV cache scaling factors. '
250 - 'This should generally be supplied, when KV cache dtype is FP8. '
251 - 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
252 - 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
253 - 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
254 - 'instead supported for common inference criteria.')
255 - parser.add_argument(
256 - "--device",
257 - type=str,
258 - default="cuda",
259 - choices=["cuda", "cpu"],
260 - help='device type for vLLM execution, supporting CUDA and CPU.')
261 - parser.add_argument(
262 - "--enable-prefix-caching",
263 - action='store_true',
264 - help="enable automatic prefix caching for vLLM backend.")
265 - parser.add_argument("--enable-chunked-prefill",
266 - action='store_true',
267 - help="enable chunked prefill for vLLM backend.")
268 - parser.add_argument('--max-num-batched-tokens',
269 - type=int,
270 - default=None,
271 - help='maximum number of batched tokens per '
272 - 'iteration')
273 - parser.add_argument('--download-dir',
274 - type=str,
275 - default=None,
276 - help='directory to download and load the weights, '
277 - 'default to the default cache dir of huggingface')
161 parser.add_argument(
162 '--output-json',
163 type=str,
164 default=None,
165 help='Path to save the throughput results in JSON format.')
166
167 + parser = EngineArgs.add_cli_args(parser)
168 args = parser.parse_args()
169 if args.tokenizer is None:
170 args.tokenizer = args.model
benchmarks/benchmark_throughput.py
+15 -222
@@ -1,5 +1,6 @@
1 """Benchmark offline inference throughput."""
2 import argparse
3 +import dataclasses
4 import json
5 import random
6 import time
@@ -11,10 +12,9 @@ from tqdm import tqdm
12 from transformers import (AutoModelForCausalLM, AutoTokenizer,
13 PreTrainedTokenizerBase)
14
14 -from vllm.engine.arg_utils import DEVICE_OPTIONS, AsyncEngineArgs, EngineArgs
15 +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
16 from vllm.entrypoints.openai.api_server import (
17 build_async_engine_client_from_engine_args)
17 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
18 from vllm.sampling_params import BeamSearchParams
19 from vllm.utils import FlexibleArgumentParser, merge_async_iterators
20
@@ -67,53 +67,11 @@ def sample_requests(
67
68 def run_vllm(
69 requests: List[Tuple[str, int, int]],
70 - model: str,
71 - tokenizer: str,
72 - quantization: Optional[str],
73 - tensor_parallel_size: int,
74 - seed: int,
70 n: int,
76 - trust_remote_code: bool,
77 - dtype: str,
78 - max_model_len: Optional[int],
79 - enforce_eager: bool,
80 - kv_cache_dtype: str,
81 - quantization_param_path: Optional[str],
82 - device: str,
83 - enable_prefix_caching: bool,
84 - enable_chunked_prefill: bool,
85 - max_num_batched_tokens: int,
86 - distributed_executor_backend: Optional[str],
87 - gpu_memory_utilization: float = 0.9,
88 - num_scheduler_steps: int = 1,
89 - download_dir: Optional[str] = None,
90 - load_format: str = EngineArgs.load_format,
91 - disable_async_output_proc: bool = False,
71 + engine_args: EngineArgs,
72 ) -> float:
73 from vllm import LLM, SamplingParams
94 - llm = LLM(
95 - model=model,
96 - tokenizer=tokenizer,
97 - quantization=quantization,
98 - tensor_parallel_size=tensor_parallel_size,
99 - seed=seed,
100 - trust_remote_code=trust_remote_code,
101 - dtype=dtype,
102 - max_model_len=max_model_len,
103 - gpu_memory_utilization=gpu_memory_utilization,
104 - enforce_eager=enforce_eager,
105 - kv_cache_dtype=kv_cache_dtype,
106 - quantization_param_path=quantization_param_path,
107 - device=device,
108 - enable_prefix_caching=enable_prefix_caching,
109 - download_dir=download_dir,
110 - enable_chunked_prefill=enable_chunked_prefill,
111 - max_num_batched_tokens=max_num_batched_tokens,
112 - distributed_executor_backend=distributed_executor_backend,
113 - load_format=load_format,
114 - num_scheduler_steps=num_scheduler_steps,
115 - disable_async_output_proc=disable_async_output_proc,
116 - )
74 + llm = LLM(**dataclasses.asdict(engine_args))
75
76 # Add the requests to the engine.
77 prompts: List[str] = []
@@ -155,56 +113,11 @@ def run_vllm(
113
114 async def run_vllm_async(
115 requests: List[Tuple[str, int, int]],
158 - model: str,
159 - tokenizer: str,
160 - quantization: Optional[str],
161 - tensor_parallel_size: int,
162 - seed: int,
116 n: int,
164 - trust_remote_code: bool,
165 - dtype: str,
166 - max_model_len: Optional[int],
167 - enforce_eager: bool,
168 - kv_cache_dtype: str,
169 - quantization_param_path: Optional[str],
170 - device: str,
171 - enable_prefix_caching: bool,
172 - enable_chunked_prefill: bool,
173 - max_num_batched_tokens: int,
174 - distributed_executor_backend: Optional[str],
175 - gpu_memory_utilization: float = 0.9,
176 - num_scheduler_steps: int = 1,
177 - download_dir: Optional[str] = None,
178 - load_format: str = EngineArgs.load_format,
179 - disable_async_output_proc: bool = False,
117 + engine_args: AsyncEngineArgs,
118 disable_frontend_multiprocessing: bool = False,
119 ) -> float:
120 from vllm import SamplingParams
183 - engine_args = AsyncEngineArgs(
184 - model=model,
185 - tokenizer=tokenizer,
186 - quantization=quantization,
187 - tensor_parallel_size=tensor_parallel_size,
188 - seed=seed,
189 - trust_remote_code=trust_remote_code,
190 - dtype=dtype,
191 - max_model_len=max_model_len,
192 - gpu_memory_utilization=gpu_memory_utilization,
193 - enforce_eager=enforce_eager,
194 - kv_cache_dtype=kv_cache_dtype,
195 - quantization_param_path=quantization_param_path,
196 - device=device,
197 - enable_prefix_caching=enable_prefix_caching,
198 - download_dir=download_dir,
199 - enable_chunked_prefill=enable_chunked_prefill,
200 - max_num_batched_tokens=max_num_batched_tokens,
201 - distributed_executor_backend=distributed_executor_backend,
202 - load_format=load_format,
203 - num_scheduler_steps=num_scheduler_steps,
204 - disable_async_output_proc=disable_async_output_proc,
205 - worker_use_ray=False,
206 - disable_log_requests=True,
207 - )
121
122 async with build_async_engine_client_from_engine_args(
123 engine_args, disable_frontend_multiprocessing) as llm:
@@ -328,23 +241,17 @@ def main(args: argparse.Namespace):
241 args.output_len)
242
243 if args.backend == "vllm":
331 - run_args = [
332 - requests, args.model, args.tokenizer, args.quantization,
333 - args.tensor_parallel_size, args.seed, args.n,
334 - args.trust_remote_code, args.dtype, args.max_model_len,
335 - args.enforce_eager, args.kv_cache_dtype,
336 - args.quantization_param_path, args.device,
337 - args.enable_prefix_caching, args.enable_chunked_prefill,
338 - args.max_num_batched_tokens, args.distributed_executor_backend,
339 - args.gpu_memory_utilization, args.num_scheduler_steps,
340 - args.download_dir, args.load_format, args.disable_async_output_proc
341 - ]
342 -
244 if args.async_engine:
344 - run_args.append(args.disable_frontend_multiprocessing)
345 - elapsed_time = uvloop.run(run_vllm_async(*run_args))
245 + elapsed_time = uvloop.run(
246 + run_vllm_async(
247 + requests,
248 + args.n,
249 + AsyncEngineArgs.from_cli_args(args),
250 + args.disable_frontend_multiprocessing,
251 + ))
252 else:
347 - elapsed_time = run_vllm(*run_args)
253 + elapsed_time = run_vllm(requests, args.n,
254 + EngineArgs.from_cli_args(args))
255 elif args.backend == "hf":
256 assert args.tensor_parallel_size == 1
257 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -391,13 +298,6 @@ if __name__ == "__main__":
298 default=None,
299 help="Output length for each request. Overrides the "
300 "output length from the dataset.")
394 - parser.add_argument("--model", type=str, default="facebook/opt-125m")
395 - parser.add_argument("--tokenizer", type=str, default=None)
396 - parser.add_argument('--quantization',
397 - '-q',
398 - choices=[*QUANTIZATION_METHODS, None],
399 - default=None)
400 - parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
301 parser.add_argument("--n",
302 type=int,
303 default=1,
@@ -406,123 +306,15 @@ if __name__ == "__main__":
306 type=int,
307 default=1000,
308 help="Number of prompts to process.")
409 - parser.add_argument("--seed", type=int, default=0)
309 parser.add_argument("--hf-max-batch-size",
310 type=int,
311 default=None,
312 help="Maximum batch size for HF backend.")
414 - parser.add_argument('--trust-remote-code',
415 - action='store_true',
416 - help='trust remote code from huggingface')
417 - parser.add_argument(
418 - '--max-model-len',
419 - type=int,
420 - default=None,
421 - help='Maximum length of a sequence (including prompt and output). '
422 - 'If None, will be derived from the model.')
423 - parser.add_argument(
424 - '--dtype',
425 - type=str,
426 - default='auto',
427 - choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
428 - help='data type for model weights and activations. '
429 - 'The "auto" option will use FP16 precision '
430 - 'for FP32 and FP16 models, and BF16 precision '
431 - 'for BF16 models.')
432 - parser.add_argument('--gpu-memory-utilization',
433 - type=float,
434 - default=0.9,
435 - help='the fraction of GPU memory to be used for '
436 - 'the model executor, which can range from 0 to 1.'
437 - 'If unspecified, will use the default value of 0.9.')
438 - parser.add_argument("--enforce-eager",
439 - action="store_true",
440 - help="enforce eager execution")
441 - parser.add_argument(
442 - '--kv-cache-dtype',
443 - type=str,
444 - choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
445 - default="auto",
446 - help='Data type for kv cache storage. If "auto", will use model '
447 - 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
448 - 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
449 - parser.add_argument(
450 - '--quantization-param-path',
451 - type=str,
452 - default=None,
453 - help='Path to the JSON file containing the KV cache scaling factors. '
454 - 'This should generally be supplied, when KV cache dtype is FP8. '
455 - 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
456 - 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
457 - 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
458 - 'instead supported for common inference criteria.')
459 - parser.add_argument("--device",
460 - type=str,
461 - default="auto",
462 - choices=DEVICE_OPTIONS,
463 - help='device type for vLLM execution')
464 - parser.add_argument(
465 - "--num-scheduler-steps",
466 - type=int,
467 - default=1,
468 - help="Maximum number of forward steps per scheduler call.")
469 - parser.add_argument(
470 - "--enable-prefix-caching",
471 - action='store_true',
472 - help="Enable automatic prefix caching for vLLM backend.")
473 - parser.add_argument("--enable-chunked-prefill",
474 - action='store_true',
475 - help="enable chunked prefill for vLLM backend.")
476 - parser.add_argument('--max-num-batched-tokens',
477 - type=int,
478 - default=None,
479 - help='maximum number of batched tokens per '
480 - 'iteration')
481 - parser.add_argument('--download-dir',
482 - type=str,
483 - default=None,
484 - help='directory to download and load the weights, '
485 - 'default to the default cache dir of huggingface')
313 parser.add_argument(
314 '--output-json',
315 type=str,
316 default=None,
317 help='Path to save the throughput results in JSON format.')
491 - parser.add_argument(
492 - '--distributed-executor-backend',
493 - choices=['ray', 'mp'],
494 - default=None,
495 - help='Backend to use for distributed serving. When more than 1 GPU '
496 - 'is used, will be automatically set to "ray" if installed '
497 - 'or "mp" (multiprocessing) otherwise.')
498 - parser.add_argument(
499 - '--load-format',
500 - type=str,
501 - default=EngineArgs.load_format,
502 - choices=[
503 - 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
504 - 'bitsandbytes'
505 - ],
506 - help='The format of the model weights to load.\n\n'
507 - '* "auto" will try to load the weights in the safetensors format '
508 - 'and fall back to the pytorch bin format if safetensors format '
509 - 'is not available.\n'
510 - '* "pt" will load the weights in the pytorch bin format.\n'
511 - '* "safetensors" will load the weights in the safetensors format.\n'
512 - '* "npcache" will load the weights in pytorch format and store '
513 - 'a numpy cache to speed up the loading.\n'
514 - '* "dummy" will initialize the weights with random values, '
515 - 'which is mainly for profiling.\n'
516 - '* "tensorizer" will load the weights using tensorizer from '
517 - 'CoreWeave. See the Tensorize vLLM Model script in the Examples'
518 - 'section for more information.\n'
519 - '* "bitsandbytes" will load the weights using bitsandbytes '
520 - 'quantization.\n')
521 - parser.add_argument(
522 - "--disable-async-output-proc",
523 - action='store_true',
524 - default=False,
525 - help="Disable async output processor for vLLM backend.")
318 parser.add_argument("--async-engine",
319 action='store_true',
320 default=False,
@@ -531,6 +323,7 @@ if __name__ == "__main__":
323 action='store_true',
324 default=False,
325 help="Disable decoupled async engine frontend.")
326 + parser = AsyncEngineArgs.add_cli_args(parser)
327 args = parser.parse_args()
328 if args.tokenizer is None:
329 args.tokenizer = args.model