A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Misc] Make benchmarks use EngineArgs (#9529)
Jeremy Arnold committed
Oct 22, 2024 at 17:40 UTC
cb6fdaa0a0b31985df4fa3ddf069c022c1faacb9
4 files changed
+38
-512
benchmarks/benchmark_latency.py
+7
-148
@@ -1,5 +1,6 @@
1
"""Benchmark the latency of processing a single batch of requests."""
2
import argparse
3
+import dataclasses
4
import json
5
import time
6
from pathlib import Path
@@ -10,43 +11,19 @@ import torch
11
from tqdm import tqdm
12
13
from vllm import LLM, SamplingParams
13
-from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14
+from vllm.engine.arg_utils import EngineArgs
15
from vllm.inputs import PromptType
15
-from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
from vllm.utils import FlexibleArgumentParser
17
18
19
def main(args: argparse.Namespace):
20
print(args)
21
22
+ engine_args = EngineArgs.from_cli_args(args)
23
+
24
# NOTE(woosuk): If the request cannot be processed in a single batch,
25
# the engine will automatically process the request in multiple batches.
24
- llm = LLM(
25
- model=args.model,
26
- speculative_model=args.speculative_model,
27
- num_speculative_tokens=args.num_speculative_tokens,
28
- speculative_draft_tensor_parallel_size=\
29
- args.speculative_draft_tensor_parallel_size,
30
- tokenizer=args.tokenizer,
31
- quantization=args.quantization,
32
- tensor_parallel_size=args.tensor_parallel_size,
33
- trust_remote_code=args.trust_remote_code,
34
- dtype=args.dtype,
35
- max_model_len=args.max_model_len,
36
- enforce_eager=args.enforce_eager,
37
- kv_cache_dtype=args.kv_cache_dtype,
38
- quantization_param_path=args.quantization_param_path,
39
- device=args.device,
40
- ray_workers_use_nsight=args.ray_workers_use_nsight,
41
- enable_chunked_prefill=args.enable_chunked_prefill,
42
- download_dir=args.download_dir,
43
- block_size=args.block_size,
44
- gpu_memory_utilization=args.gpu_memory_utilization,
45
- load_format=args.load_format,
46
- distributed_executor_backend=args.distributed_executor_backend,
47
- otlp_traces_endpoint=args.otlp_traces_endpoint,
48
- enable_prefix_caching=args.enable_prefix_caching,
49
- )
26
+ llm = LLM(**dataclasses.asdict(engine_args))
27
28
sampling_params = SamplingParams(
29
n=args.n,
@@ -125,19 +102,6 @@ if __name__ == '__main__':
102
parser = FlexibleArgumentParser(
103
description='Benchmark the latency of processing a single batch of '
104
'requests till completion.')
128
- parser.add_argument('--model', type=str, default='facebook/opt-125m')
129
- parser.add_argument('--speculative-model', type=str, default=None)
130
- parser.add_argument('--num-speculative-tokens', type=int, default=None)
131
- parser.add_argument('--speculative-draft-tensor-parallel-size',
132
- '-spec-draft-tp',
133
- type=int,
134
- default=None)
135
- parser.add_argument('--tokenizer', type=str, default=None)
136
- parser.add_argument('--quantization',
137
- '-q',
138
- choices=[*QUANTIZATION_METHODS, None],
139
- default=None)
140
- parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
105
parser.add_argument('--input-len', type=int, default=32)
106
parser.add_argument('--output-len', type=int, default=128)
107
parser.add_argument('--batch-size', type=int, default=8)
@@ -154,45 +118,6 @@ if __name__ == '__main__':
118
type=int,
119
default=30,
120
help='Number of iterations to run.')
157
- parser.add_argument('--trust-remote-code',
158
- action='store_true',
159
- help='trust remote code from huggingface')
160
- parser.add_argument(
161
- '--max-model-len',
162
- type=int,
163
- default=None,
164
- help='Maximum length of a sequence (including prompt and output). '
165
- 'If None, will be derived from the model.')
166
- parser.add_argument(
167
- '--dtype',
168
- type=str,
169
- default='auto',
170
- choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
171
- help='data type for model weights and activations. '
172
- 'The "auto" option will use FP16 precision '
173
- 'for FP32 and FP16 models, and BF16 precision '
174
- 'for BF16 models.')
175
- parser.add_argument('--enforce-eager',
176
- action='store_true',
177
- help='enforce eager mode and disable CUDA graph')
178
- parser.add_argument(
179
- '--kv-cache-dtype',
180
- type=str,
181
- choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
182
- default="auto",
183
- help='Data type for kv cache storage. If "auto", will use model '
184
- 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
185
- 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
186
- parser.add_argument(
187
- '--quantization-param-path',
188
- type=str,
189
- default=None,
190
- help='Path to the JSON file containing the KV cache scaling factors. '
191
- 'This should generally be supplied, when KV cache dtype is FP8. '
192
- 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
193
- 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
194
- 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
195
- 'instead supported for common inference criteria.')
121
parser.add_argument(
122
'--profile',
123
action='store_true',
@@ -203,78 +128,12 @@ if __name__ == '__main__':
128
default=None,
129
help=('path to save the pytorch profiler output. Can be visualized '
130
'with ui.perfetto.dev or Tensorboard.'))
206
- parser.add_argument("--device",
207
- type=str,
208
- default="auto",
209
- choices=DEVICE_OPTIONS,
210
- help='device type for vLLM execution')
211
- parser.add_argument('--block-size',
212
- type=int,
213
- default=16,
214
- help='block size of key/value cache')
215
- parser.add_argument(
216
- '--enable-chunked-prefill',
217
- action='store_true',
218
- help='If True, the prefill requests can be chunked based on the '
219
- 'max_num_batched_tokens')
220
- parser.add_argument("--enable-prefix-caching",
221
- action='store_true',
222
- help="Enable automatic prefix caching")
223
- parser.add_argument(
224
- "--ray-workers-use-nsight",
225
- action='store_true',
226
- help="If specified, use nsight to profile ray workers",
227
- )
228
- parser.add_argument('--download-dir',
229
- type=str,
230
- default=None,
231
- help='directory to download and load the weights, '
232
- 'default to the default cache dir of huggingface')
131
parser.add_argument(
132
'--output-json',
133
type=str,
134
default=None,
135
help='Path to save the latency results in JSON format.')
238
- parser.add_argument('--gpu-memory-utilization',
239
- type=float,
240
- default=0.9,
241
- help='the fraction of GPU memory to be used for '
242
- 'the model executor, which can range from 0 to 1.'
243
- 'If unspecified, will use the default value of 0.9.')
244
- parser.add_argument(
245
- '--load-format',
246
- type=str,
247
- default=EngineArgs.load_format,
248
- choices=[
249
- 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
250
- 'bitsandbytes'
251
- ],
252
- help='The format of the model weights to load.\n\n'
253
- '* "auto" will try to load the weights in the safetensors format '
254
- 'and fall back to the pytorch bin format if safetensors format '
255
- 'is not available.\n'
256
- '* "pt" will load the weights in the pytorch bin format.\n'
257
- '* "safetensors" will load the weights in the safetensors format.\n'
258
- '* "npcache" will load the weights in pytorch format and store '
259
- 'a numpy cache to speed up the loading.\n'
260
- '* "dummy" will initialize the weights with random values, '
261
- 'which is mainly for profiling.\n'
262
- '* "tensorizer" will load the weights using tensorizer from '
263
- 'CoreWeave. See the Tensorize vLLM Model script in the Examples'
264
- 'section for more information.\n'
265
- '* "bitsandbytes" will load the weights using bitsandbytes '
266
- 'quantization.\n')
267
- parser.add_argument(
268
- '--distributed-executor-backend',
269
- choices=['ray', 'mp'],
270
- default=None,
271
- help='Backend to use for distributed serving. When more than 1 GPU '
272
- 'is used, will be automatically set to "ray" if installed '
273
- 'or "mp" (multiprocessing) otherwise.')
274
- parser.add_argument(
275
- '--otlp-traces-endpoint',
276
- type=str,
277
- default=None,
278
- help='Target URL to which OpenTelemetry traces will be sent.')
136
+
137
+ parser = EngineArgs.add_cli_args(parser)
138
args = parser.parse_args()
139
main(args)
benchmarks/benchmark_prefix_caching.py
+7
-17
@@ -25,6 +25,7 @@ ShareGPT example usage:
25
--input-length-range 128:256
26
"""
27
28
+import dataclasses
29
import json
30
import random
31
import time
@@ -33,6 +34,7 @@ from typing import List, Optional, Tuple
34
from transformers import PreTrainedTokenizerBase
35
36
from vllm import LLM, SamplingParams
37
+from vllm.engine.arg_utils import EngineArgs
38
from vllm.utils import FlexibleArgumentParser
39
40
try:
@@ -129,12 +131,9 @@ def main(args):
131
filtered_datasets = [(PROMPT, prompt_len, args.output_len)
132
] * args.num_prompts
133
132
- llm = LLM(model=args.model,
133
- tokenizer_mode='auto',
134
- trust_remote_code=True,
135
- enforce_eager=True,
136
- tensor_parallel_size=args.tensor_parallel_size,
137
- enable_prefix_caching=args.enable_prefix_caching)
134
+ engine_args = EngineArgs.from_cli_args(args)
135
+
136
+ llm = LLM(**dataclasses.asdict(engine_args))
137
138
sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
139
@@ -162,18 +161,11 @@ if __name__ == "__main__":
161
parser = FlexibleArgumentParser(
162
description=
163
'Benchmark the performance with or without automatic prefix caching.')
165
- parser.add_argument('--model',
166
- type=str,
167
- default='baichuan-inc/Baichuan2-13B-Chat')
164
parser.add_argument("--dataset-path",
165
type=str,
166
default=None,
167
help="Path to the dataset.")
172
- parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
168
parser.add_argument('--output-len', type=int, default=10)
174
- parser.add_argument('--enable-prefix-caching',
175
- action='store_true',
176
- help='enable prefix caching')
169
parser.add_argument('--num-prompts',
170
type=int,
171
default=1,
@@ -190,9 +182,7 @@ if __name__ == "__main__":
182
default='128:256',
183
help='Range of input lengths for sampling prompts,'
184
'specified as "min:max" (e.g., "128:256").')
193
- parser.add_argument("--seed",
194
- type=int,
195
- default=0,
196
- help='Random seed for reproducibility')
185
+
186
+ parser = EngineArgs.add_cli_args(parser)
187
args = parser.parse_args()
188
main(args)
benchmarks/benchmark_prioritization.py
+9
-125
@@ -1,5 +1,6 @@
1
"""Benchmark offline prioritization."""
2
import argparse
3
+import dataclasses
4
import json
5
import random
6
import time
@@ -7,7 +8,8 @@ from typing import List, Optional, Tuple
8
9
from transformers import AutoTokenizer, PreTrainedTokenizerBase
10
10
-from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11
+from vllm.engine.arg_utils import EngineArgs
12
+from vllm.utils import FlexibleArgumentParser
13
14
15
def sample_requests(
@@ -62,46 +64,11 @@ def sample_requests(
64
65
def run_vllm(
66
requests: List[Tuple[str, int, int]],
65
- model: str,
66
- tokenizer: str,
67
- quantization: Optional[str],
68
- tensor_parallel_size: int,
69
- seed: int,
67
n: int,
71
- trust_remote_code: bool,
72
- dtype: str,
73
- max_model_len: Optional[int],
74
- enforce_eager: bool,
75
- kv_cache_dtype: str,
76
- quantization_param_path: Optional[str],
77
- device: str,
78
- enable_prefix_caching: bool,
79
- enable_chunked_prefill: bool,
80
- max_num_batched_tokens: int,
81
- gpu_memory_utilization: float = 0.9,
82
- download_dir: Optional[str] = None,
68
+ engine_args: EngineArgs,
69
) -> float:
70
from vllm import LLM, SamplingParams
85
- llm = LLM(
86
- model=model,
87
- tokenizer=tokenizer,
88
- quantization=quantization,
89
- tensor_parallel_size=tensor_parallel_size,
90
- seed=seed,
91
- trust_remote_code=trust_remote_code,
92
- dtype=dtype,
93
- max_model_len=max_model_len,
94
- gpu_memory_utilization=gpu_memory_utilization,
95
- enforce_eager=enforce_eager,
96
- kv_cache_dtype=kv_cache_dtype,
97
- quantization_param_path=quantization_param_path,
98
- device=device,
99
- enable_prefix_caching=enable_prefix_caching,
100
- download_dir=download_dir,
101
- enable_chunked_prefill=enable_chunked_prefill,
102
- max_num_batched_tokens=max_num_batched_tokens,
103
- disable_log_stats=False,
104
- )
71
+ llm = LLM(**dataclasses.asdict(engine_args))
72
73
# Add the requests to the engine.
74
prompts = []
@@ -142,16 +109,8 @@ def main(args: argparse.Namespace):
109
args.output_len)
110
111
if args.backend == "vllm":
145
- elapsed_time = run_vllm(requests, args.model, args.tokenizer,
146
- args.quantization, args.tensor_parallel_size,
147
- args.seed, args.n, args.trust_remote_code,
148
- args.dtype, args.max_model_len,
149
- args.enforce_eager, args.kv_cache_dtype,
150
- args.quantization_param_path, args.device,
151
- args.enable_prefix_caching,
152
- args.enable_chunked_prefill,
153
- args.max_num_batched_tokens,
154
- args.gpu_memory_utilization, args.download_dir)
112
+ elapsed_time = run_vllm(requests, args.n,
113
+ EngineArgs.from_cli_args(args))
114
else:
115
raise ValueError(f"Unknown backend: {args.backend}")
116
total_num_tokens = sum(prompt_len + output_len
@@ -173,7 +132,7 @@ def main(args: argparse.Namespace):
132
133
134
if __name__ == "__main__":
176
- parser = argparse.ArgumentParser(description="Benchmark the throughput.")
135
+ parser = FlexibleArgumentParser(description="Benchmark the throughput.")
136
parser.add_argument("--backend",
137
type=str,
138
choices=["vllm", "hf", "mii"],
@@ -191,13 +150,6 @@ if __name__ == "__main__":
150
default=None,
151
help="Output length for each request. Overrides the "
152
"output length from the dataset.")
194
- parser.add_argument("--model", type=str, default="facebook/opt-125m")
195
- parser.add_argument("--tokenizer", type=str, default=None)
196
- parser.add_argument('--quantization',
197
- '-q',
198
- choices=[*QUANTIZATION_METHODS, None],
199
- default=None)
200
- parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
153
parser.add_argument("--n",
154
type=int,
155
default=1,
@@ -206,81 +158,13 @@ if __name__ == "__main__":
158
type=int,
159
default=200,
160
help="Number of prompts to process.")
209
- parser.add_argument("--seed", type=int, default=0)
210
- parser.add_argument('--trust-remote-code',
211
- action='store_true',
212
- help='trust remote code from huggingface')
213
- parser.add_argument(
214
- '--max-model-len',
215
- type=int,
216
- default=None,
217
- help='Maximum length of a sequence (including prompt and output). '
218
- 'If None, will be derived from the model.')
219
- parser.add_argument(
220
- '--dtype',
221
- type=str,
222
- default='auto',
223
- choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
224
- help='data type for model weights and activations. '
225
- 'The "auto" option will use FP16 precision '
226
- 'for FP32 and FP16 models, and BF16 precision '
227
- 'for BF16 models.')
228
- parser.add_argument('--gpu-memory-utilization',
229
- type=float,
230
- default=0.9,
231
- help='the fraction of GPU memory to be used for '
232
- 'the model executor, which can range from 0 to 1.'
233
- 'If unspecified, will use the default value of 0.9.')
234
- parser.add_argument("--enforce-eager",
235
- action="store_true",
236
- help="enforce eager execution")
237
- parser.add_argument(
238
- '--kv-cache-dtype',
239
- type=str,
240
- choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
241
- default="auto",
242
- help='Data type for kv cache storage. If "auto", will use model '
243
- 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
244
- 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
245
- parser.add_argument(
246
- '--quantization-param-path',
247
- type=str,
248
- default=None,
249
- help='Path to the JSON file containing the KV cache scaling factors. '
250
- 'This should generally be supplied, when KV cache dtype is FP8. '
251
- 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
252
- 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
253
- 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
254
- 'instead supported for common inference criteria.')
255
- parser.add_argument(
256
- "--device",
257
- type=str,
258
- default="cuda",
259
- choices=["cuda", "cpu"],
260
- help='device type for vLLM execution, supporting CUDA and CPU.')
261
- parser.add_argument(
262
- "--enable-prefix-caching",
263
- action='store_true',
264
- help="enable automatic prefix caching for vLLM backend.")
265
- parser.add_argument("--enable-chunked-prefill",
266
- action='store_true',
267
- help="enable chunked prefill for vLLM backend.")
268
- parser.add_argument('--max-num-batched-tokens',
269
- type=int,
270
- default=None,
271
- help='maximum number of batched tokens per '
272
- 'iteration')
273
- parser.add_argument('--download-dir',
274
- type=str,
275
- default=None,
276
- help='directory to download and load the weights, '
277
- 'default to the default cache dir of huggingface')
161
parser.add_argument(
162
'--output-json',
163
type=str,
164
default=None,
165
help='Path to save the throughput results in JSON format.')
166
167
+ parser = EngineArgs.add_cli_args(parser)
168
args = parser.parse_args()
169
if args.tokenizer is None:
170
args.tokenizer = args.model
benchmarks/benchmark_throughput.py
+15
-222
@@ -1,5 +1,6 @@
1
"""Benchmark offline inference throughput."""
2
import argparse
3
+import dataclasses
4
import json
5
import random
6
import time
@@ -11,10 +12,9 @@ from tqdm import tqdm
12
from transformers import (AutoModelForCausalLM, AutoTokenizer,
13
PreTrainedTokenizerBase)
14
14
-from vllm.engine.arg_utils import DEVICE_OPTIONS, AsyncEngineArgs, EngineArgs
15
+from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
16
from vllm.entrypoints.openai.api_server import (
17
build_async_engine_client_from_engine_args)
17
-from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
18
from vllm.sampling_params import BeamSearchParams
19
from vllm.utils import FlexibleArgumentParser, merge_async_iterators
20
@@ -67,53 +67,11 @@ def sample_requests(
67
68
def run_vllm(
69
requests: List[Tuple[str, int, int]],
70
- model: str,
71
- tokenizer: str,
72
- quantization: Optional[str],
73
- tensor_parallel_size: int,
74
- seed: int,
70
n: int,
76
- trust_remote_code: bool,
77
- dtype: str,
78
- max_model_len: Optional[int],
79
- enforce_eager: bool,
80
- kv_cache_dtype: str,
81
- quantization_param_path: Optional[str],
82
- device: str,
83
- enable_prefix_caching: bool,
84
- enable_chunked_prefill: bool,
85
- max_num_batched_tokens: int,
86
- distributed_executor_backend: Optional[str],
87
- gpu_memory_utilization: float = 0.9,
88
- num_scheduler_steps: int = 1,
89
- download_dir: Optional[str] = None,
90
- load_format: str = EngineArgs.load_format,
91
- disable_async_output_proc: bool = False,
71
+ engine_args: EngineArgs,
72
) -> float:
73
from vllm import LLM, SamplingParams
94
- llm = LLM(
95
- model=model,
96
- tokenizer=tokenizer,
97
- quantization=quantization,
98
- tensor_parallel_size=tensor_parallel_size,
99
- seed=seed,
100
- trust_remote_code=trust_remote_code,
101
- dtype=dtype,
102
- max_model_len=max_model_len,
103
- gpu_memory_utilization=gpu_memory_utilization,
104
- enforce_eager=enforce_eager,
105
- kv_cache_dtype=kv_cache_dtype,
106
- quantization_param_path=quantization_param_path,
107
- device=device,
108
- enable_prefix_caching=enable_prefix_caching,
109
- download_dir=download_dir,
110
- enable_chunked_prefill=enable_chunked_prefill,
111
- max_num_batched_tokens=max_num_batched_tokens,
112
- distributed_executor_backend=distributed_executor_backend,
113
- load_format=load_format,
114
- num_scheduler_steps=num_scheduler_steps,
115
- disable_async_output_proc=disable_async_output_proc,
116
- )
74
+ llm = LLM(**dataclasses.asdict(engine_args))
75
76
# Add the requests to the engine.
77
prompts: List[str] = []
@@ -155,56 +113,11 @@ def run_vllm(
113
114
async def run_vllm_async(
115
requests: List[Tuple[str, int, int]],
158
- model: str,
159
- tokenizer: str,
160
- quantization: Optional[str],
161
- tensor_parallel_size: int,
162
- seed: int,
116
n: int,
164
- trust_remote_code: bool,
165
- dtype: str,
166
- max_model_len: Optional[int],
167
- enforce_eager: bool,
168
- kv_cache_dtype: str,
169
- quantization_param_path: Optional[str],
170
- device: str,
171
- enable_prefix_caching: bool,
172
- enable_chunked_prefill: bool,
173
- max_num_batched_tokens: int,
174
- distributed_executor_backend: Optional[str],
175
- gpu_memory_utilization: float = 0.9,
176
- num_scheduler_steps: int = 1,
177
- download_dir: Optional[str] = None,
178
- load_format: str = EngineArgs.load_format,
179
- disable_async_output_proc: bool = False,
117
+ engine_args: AsyncEngineArgs,
118
disable_frontend_multiprocessing: bool = False,
119
) -> float:
120
from vllm import SamplingParams
183
- engine_args = AsyncEngineArgs(
184
- model=model,
185
- tokenizer=tokenizer,
186
- quantization=quantization,
187
- tensor_parallel_size=tensor_parallel_size,
188
- seed=seed,
189
- trust_remote_code=trust_remote_code,
190
- dtype=dtype,
191
- max_model_len=max_model_len,
192
- gpu_memory_utilization=gpu_memory_utilization,
193
- enforce_eager=enforce_eager,
194
- kv_cache_dtype=kv_cache_dtype,
195
- quantization_param_path=quantization_param_path,
196
- device=device,
197
- enable_prefix_caching=enable_prefix_caching,
198
- download_dir=download_dir,
199
- enable_chunked_prefill=enable_chunked_prefill,
200
- max_num_batched_tokens=max_num_batched_tokens,
201
- distributed_executor_backend=distributed_executor_backend,
202
- load_format=load_format,
203
- num_scheduler_steps=num_scheduler_steps,
204
- disable_async_output_proc=disable_async_output_proc,
205
- worker_use_ray=False,
206
- disable_log_requests=True,
207
- )
121
122
async with build_async_engine_client_from_engine_args(
123
engine_args, disable_frontend_multiprocessing) as llm:
@@ -328,23 +241,17 @@ def main(args: argparse.Namespace):
241
args.output_len)
242
243
if args.backend == "vllm":
331
- run_args = [
332
- requests, args.model, args.tokenizer, args.quantization,
333
- args.tensor_parallel_size, args.seed, args.n,
334
- args.trust_remote_code, args.dtype, args.max_model_len,
335
- args.enforce_eager, args.kv_cache_dtype,
336
- args.quantization_param_path, args.device,
337
- args.enable_prefix_caching, args.enable_chunked_prefill,
338
- args.max_num_batched_tokens, args.distributed_executor_backend,
339
- args.gpu_memory_utilization, args.num_scheduler_steps,
340
- args.download_dir, args.load_format, args.disable_async_output_proc
341
- ]
342
-
244
if args.async_engine:
344
- run_args.append(args.disable_frontend_multiprocessing)
345
- elapsed_time = uvloop.run(run_vllm_async(*run_args))
245
+ elapsed_time = uvloop.run(
246
+ run_vllm_async(
247
+ requests,
248
+ args.n,
249
+ AsyncEngineArgs.from_cli_args(args),
250
+ args.disable_frontend_multiprocessing,
251
+ ))
252
else:
347
- elapsed_time = run_vllm(*run_args)
253
+ elapsed_time = run_vllm(requests, args.n,
254
+ EngineArgs.from_cli_args(args))
255
elif args.backend == "hf":
256
assert args.tensor_parallel_size == 1
257
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -391,13 +298,6 @@ if __name__ == "__main__":
298
default=None,
299
help="Output length for each request. Overrides the "
300
"output length from the dataset.")
394
- parser.add_argument("--model", type=str, default="facebook/opt-125m")
395
- parser.add_argument("--tokenizer", type=str, default=None)
396
- parser.add_argument('--quantization',
397
- '-q',
398
- choices=[*QUANTIZATION_METHODS, None],
399
- default=None)
400
- parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
301
parser.add_argument("--n",
302
type=int,
303
default=1,
@@ -406,123 +306,15 @@ if __name__ == "__main__":
306
type=int,
307
default=1000,
308
help="Number of prompts to process.")
409
- parser.add_argument("--seed", type=int, default=0)
309
parser.add_argument("--hf-max-batch-size",
310
type=int,
311
default=None,
312
help="Maximum batch size for HF backend.")
414
- parser.add_argument('--trust-remote-code',
415
- action='store_true',
416
- help='trust remote code from huggingface')
417
- parser.add_argument(
418
- '--max-model-len',
419
- type=int,
420
- default=None,
421
- help='Maximum length of a sequence (including prompt and output). '
422
- 'If None, will be derived from the model.')
423
- parser.add_argument(
424
- '--dtype',
425
- type=str,
426
- default='auto',
427
- choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
428
- help='data type for model weights and activations. '
429
- 'The "auto" option will use FP16 precision '
430
- 'for FP32 and FP16 models, and BF16 precision '
431
- 'for BF16 models.')
432
- parser.add_argument('--gpu-memory-utilization',
433
- type=float,
434
- default=0.9,
435
- help='the fraction of GPU memory to be used for '
436
- 'the model executor, which can range from 0 to 1.'
437
- 'If unspecified, will use the default value of 0.9.')
438
- parser.add_argument("--enforce-eager",
439
- action="store_true",
440
- help="enforce eager execution")
441
- parser.add_argument(
442
- '--kv-cache-dtype',
443
- type=str,
444
- choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
445
- default="auto",
446
- help='Data type for kv cache storage. If "auto", will use model '
447
- 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
448
- 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
449
- parser.add_argument(
450
- '--quantization-param-path',
451
- type=str,
452
- default=None,
453
- help='Path to the JSON file containing the KV cache scaling factors. '
454
- 'This should generally be supplied, when KV cache dtype is FP8. '
455
- 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
456
- 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
457
- 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
458
- 'instead supported for common inference criteria.')
459
- parser.add_argument("--device",
460
- type=str,
461
- default="auto",
462
- choices=DEVICE_OPTIONS,
463
- help='device type for vLLM execution')
464
- parser.add_argument(
465
- "--num-scheduler-steps",
466
- type=int,
467
- default=1,
468
- help="Maximum number of forward steps per scheduler call.")
469
- parser.add_argument(
470
- "--enable-prefix-caching",
471
- action='store_true',
472
- help="Enable automatic prefix caching for vLLM backend.")
473
- parser.add_argument("--enable-chunked-prefill",
474
- action='store_true',
475
- help="enable chunked prefill for vLLM backend.")
476
- parser.add_argument('--max-num-batched-tokens',
477
- type=int,
478
- default=None,
479
- help='maximum number of batched tokens per '
480
- 'iteration')
481
- parser.add_argument('--download-dir',
482
- type=str,
483
- default=None,
484
- help='directory to download and load the weights, '
485
- 'default to the default cache dir of huggingface')
313
parser.add_argument(
314
'--output-json',
315
type=str,
316
default=None,
317
help='Path to save the throughput results in JSON format.')
491
- parser.add_argument(
492
- '--distributed-executor-backend',
493
- choices=['ray', 'mp'],
494
- default=None,
495
- help='Backend to use for distributed serving. When more than 1 GPU '
496
- 'is used, will be automatically set to "ray" if installed '
497
- 'or "mp" (multiprocessing) otherwise.')
498
- parser.add_argument(
499
- '--load-format',
500
- type=str,
501
- default=EngineArgs.load_format,
502
- choices=[
503
- 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
504
- 'bitsandbytes'
505
- ],
506
- help='The format of the model weights to load.\n\n'
507
- '* "auto" will try to load the weights in the safetensors format '
508
- 'and fall back to the pytorch bin format if safetensors format '
509
- 'is not available.\n'
510
- '* "pt" will load the weights in the pytorch bin format.\n'
511
- '* "safetensors" will load the weights in the safetensors format.\n'
512
- '* "npcache" will load the weights in pytorch format and store '
513
- 'a numpy cache to speed up the loading.\n'
514
- '* "dummy" will initialize the weights with random values, '
515
- 'which is mainly for profiling.\n'
516
- '* "tensorizer" will load the weights using tensorizer from '
517
- 'CoreWeave. See the Tensorize vLLM Model script in the Examples'
518
- 'section for more information.\n'
519
- '* "bitsandbytes" will load the weights using bitsandbytes '
520
- 'quantization.\n')
521
- parser.add_argument(
522
- "--disable-async-output-proc",
523
- action='store_true',
524
- default=False,
525
- help="Disable async output processor for vLLM backend.")
318
parser.add_argument("--async-engine",
319
action='store_true',
320
default=False,
@@ -531,6 +323,7 @@ if __name__ == "__main__":
323
action='store_true',
324
default=False,
325
help="Disable decoupled async engine frontend.")
326
+ parser = AsyncEngineArgs.add_cli_args(parser)
327
args = parser.parse_args()
328
if args.tokenizer is None:
329
args.tokenizer = args.model