A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Fix missing docs and out of sync `EngineArgs` (#4219)
Co-authored-by: Harry Mellor <hmellor@oxts.com>
Harry Mellor committed
Apr 20, 2024 at 04:51 UTC
682789d4026429a04cc32acb88064265441080dd
3 files changed
+98
-197
docs/source/conf.py
+2
@@ -11,12 +11,14 @@
11
# documentation root, use os.path.abspath to make it absolute, like shown here.
12
13
import logging
14
+import os
15
import sys
16
from typing import List
17
18
from sphinx.ext import autodoc
19
20
logger = logging.getLogger(__name__)
21
+sys.path.append(os.path.abspath("../.."))
22
23
# -- Project information -----------------------------------------------------
24
docs/source/models/engine_args.rst
+9
-125
@@ -5,133 +5,17 @@ Engine Arguments
5
6
Below, you can find an explanation of every engine argument for vLLM:
7
8
-.. option:: --model <model_name_or_path>
9
-
10
- Name or path of the huggingface model to use.
11
-
12
-.. option:: --tokenizer <tokenizer_name_or_path>
13
-
14
- Name or path of the huggingface tokenizer to use.
15
-
16
-.. option:: --revision <revision>
17
-
18
- The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.
19
-
20
-.. option:: --tokenizer-revision <revision>
21
-
22
- The specific tokenizer version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.
23
-
24
-.. option:: --tokenizer-mode {auto,slow}
25
-
26
- The tokenizer mode.
27
-
28
- * "auto" will use the fast tokenizer if available.
29
- * "slow" will always use the slow tokenizer.
30
-
31
-.. option:: --trust-remote-code
32
-
33
- Trust remote code from huggingface.
34
-
35
-.. option:: --download-dir <directory>
36
-
37
- Directory to download and load the weights, default to the default cache dir of huggingface.
38
-
39
-.. option:: --load-format {auto,pt,safetensors,npcache,dummy,tensorizer}
40
-
41
- The format of the model weights to load.
42
-
43
- * "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available.
44
- * "pt" will load the weights in the pytorch bin format.
45
- * "safetensors" will load the weights in the safetensors format.
46
- * "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading.
47
- * "dummy" will initialize the weights with random values, mainly for profiling.
48
- * "tensorizer" will load serialized weights using `CoreWeave's Tensorizer model deserializer. <https://github.com/coreweave/tensorizer>`_ See `examples/tensorize_vllm_model.py <https://github.com/vllm-project/vllm/blob/main/examples/tensorize_vllm_model.py>`_ to serialize a vLLM model, and for more information.
49
-
50
-.. option:: --dtype {auto,half,float16,bfloat16,float,float32}
51
-
52
- Data type for model weights and activations.
53
-
54
- * "auto" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models.
55
- * "half" for FP16. Recommended for AWQ quantization.
56
- * "float16" is the same as "half".
57
- * "bfloat16" for a balance between precision and range.
58
- * "float" is shorthand for FP32 precision.
59
- * "float32" for FP32 precision.
60
-
61
-.. option:: --max-model-len <length>
62
-
63
- Model context length. If unspecified, will be automatically derived from the model config.
64
-
65
-.. option:: --worker-use-ray
66
-
67
- Use Ray for distributed serving, will be automatically set when using more than 1 GPU.
68
-
69
-.. option:: --pipeline-parallel-size (-pp) <size>
70
-
71
- Number of pipeline stages.
72
-
73
-.. option:: --tensor-parallel-size (-tp) <size>
74
-
75
- Number of tensor parallel replicas.
76
-
77
-.. option:: --max-parallel-loading-workers <workers>
78
-
79
- Load model sequentially in multiple batches, to avoid RAM OOM when using tensor parallel and large models.
80
-
81
-.. option:: --block-size {8,16,32}
82
-
83
- Token block size for contiguous chunks of tokens.
84
-
85
-.. option:: --enable-prefix-caching
86
-
87
- Enables automatic prefix caching
88
-
89
-.. option:: --seed <seed>
90
-
91
- Random seed for operations.
92
-
93
-.. option:: --swap-space <size>
94
-
95
- CPU swap space size (GiB) per GPU.
96
-
97
-.. option:: --gpu-memory-utilization <fraction>
98
-
99
- The fraction of GPU memory to be used for the model executor, which can range from 0 to 1.
100
- For example, a value of 0.5 would imply 50% GPU memory utilization.
101
- If unspecified, will use the default value of 0.9.
102
-
103
-.. option:: --max-num-batched-tokens <tokens>
104
-
105
- Maximum number of batched tokens per iteration.
106
-
107
-.. option:: --max-num-seqs <sequences>
108
-
109
- Maximum number of sequences per iteration.
110
-
111
-.. option:: --max-paddings <paddings>
112
-
113
- Maximum number of paddings in a batch.
114
-
115
-.. option:: --disable-log-stats
116
-
117
- Disable logging statistics.
118
-
119
-.. option:: --quantization (-q) {awq,squeezellm,None}
120
-
121
- Method used to quantize the weights.
8
+.. argparse::
9
+ :module: vllm.engine.arg_utils
10
+ :func: _engine_args_parser
11
+ :prog: -m vllm.entrypoints.openai.api_server
12
13
Async Engine Arguments
14
----------------------
125
-Below are the additional arguments related to the asynchronous engine:
126
-
127
-.. option:: --engine-use-ray
15
129
- Use Ray to start the LLM engine in a separate process as the server process.
130
-
131
-.. option:: --disable-log-requests
132
-
133
- Disable logging requests.
134
-
135
-.. option:: --max-log-len
16
+Below are the additional arguments related to the asynchronous engine:
17
137
- Max number of prompt characters or prompt ID numbers being printed in log. Defaults to unlimited.
\ No newline at end of file
18
+.. argparse::
19
+ :module: vllm.engine.arg_utils
20
+ :func: _async_engine_args_parser
21
+ :prog: -m vllm.entrypoints.openai.api_server
\ No newline at end of file
vllm/engine/arg_utils.py
+87
-72
@@ -82,57 +82,55 @@ class EngineArgs:
82
parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
83
"""Shared CLI arguments for vLLM engine."""
84
85
- # NOTE: If you update any of the arguments below, please also
86
- # make sure to update docs/source/models/engine_args.rst
87
-
85
# Model arguments
86
parser.add_argument(
87
'--model',
88
type=str,
89
default='facebook/opt-125m',
93
- help='name or path of the huggingface model to use')
90
+ help='Name or path of the huggingface model to use.')
91
parser.add_argument(
92
'--tokenizer',
93
type=str,
94
default=EngineArgs.tokenizer,
98
- help='name or path of the huggingface tokenizer to use')
95
+ help='Name or path of the huggingface tokenizer to use.')
96
parser.add_argument(
97
'--revision',
98
type=str,
99
default=None,
103
- help='the specific model version to use. It can be a branch '
100
+ help='The specific model version to use. It can be a branch '
101
'name, a tag name, or a commit id. If unspecified, will use '
102
'the default version.')
103
parser.add_argument(
104
'--code-revision',
105
type=str,
106
default=None,
110
- help='the specific revision to use for the model code on '
107
+ help='The specific revision to use for the model code on '
108
'Hugging Face Hub. It can be a branch name, a tag name, or a '
109
'commit id. If unspecified, will use the default version.')
110
parser.add_argument(
111
'--tokenizer-revision',
112
type=str,
113
default=None,
117
- help='the specific tokenizer version to use. It can be a branch '
114
+ help='The specific tokenizer version to use. It can be a branch '
115
'name, a tag name, or a commit id. If unspecified, will use '
116
'the default version.')
120
- parser.add_argument('--tokenizer-mode',
121
- type=str,
122
- default=EngineArgs.tokenizer_mode,
123
- choices=['auto', 'slow'],
124
- help='tokenizer mode. "auto" will use the fast '
125
- 'tokenizer if available, and "slow" will '
126
- 'always use the slow tokenizer.')
117
+ parser.add_argument(
118
+ '--tokenizer-mode',
119
+ type=str,
120
+ default=EngineArgs.tokenizer_mode,
121
+ choices=['auto', 'slow'],
122
+ help='The tokenizer mode.\n\n* "auto" will use the '
123
+ 'fast tokenizer if available.\n* "slow" will '
124
+ 'always use the slow tokenizer.')
125
parser.add_argument('--trust-remote-code',
126
action='store_true',
129
- help='trust remote code from huggingface')
127
+ help='Trust remote code from huggingface.')
128
parser.add_argument('--download-dir',
129
type=str,
130
default=EngineArgs.download_dir,
133
- help='directory to download and load the weights, '
131
+ help='Directory to download and load the weights, '
132
'default to the default cache dir of '
135
- 'huggingface')
133
+ 'huggingface.')
134
parser.add_argument(
135
'--load-format',
136
type=str,
@@ -140,19 +138,19 @@ class EngineArgs:
138
choices=[
139
'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer'
140
],
143
- help='The format of the model weights to load. '
144
- '"auto" will try to load the weights in the safetensors format '
141
+ help='The format of the model weights to load.\n\n'
142
+ '* "auto" will try to load the weights in the safetensors format '
143
'and fall back to the pytorch bin format if safetensors format '
146
- 'is not available. '
147
- '"pt" will load the weights in the pytorch bin format. '
148
- '"safetensors" will load the weights in the safetensors format. '
149
- '"npcache" will load the weights in pytorch format and store '
150
- 'a numpy cache to speed up the loading. '
151
- '"dummy" will initialize the weights with random values, '
152
- 'which is mainly for profiling.'
153
- '"tensorizer" will load the weights using tensorizer from CoreWeave'
154
- 'which assumes tensorizer_uri is set to the location of the '
155
- 'serialized weights.')
144
+ 'is not available.\n'
145
+ '* "pt" will load the weights in the pytorch bin format.\n'
146
+ '* "safetensors" will load the weights in the safetensors format.\n'
147
+ '* "npcache" will load the weights in pytorch format and store '
148
+ 'a numpy cache to speed up the loading.\n'
149
+ '* "dummy" will initialize the weights with random values, '
150
+ 'which is mainly for profiling.\n'
151
+ '* "tensorizer" will load the weights using tensorizer from '
152
+ 'CoreWeave which assumes tensorizer_uri is set to the location of '
153
+ 'the serialized weights.')
154
parser.add_argument(
155
'--dtype',
156
type=str,
@@ -160,10 +158,14 @@ class EngineArgs:
158
choices=[
159
'auto', 'half', 'float16', 'bfloat16', 'float', 'float32'
160
],
163
- help='data type for model weights and activations. '
164
- 'The "auto" option will use FP16 precision '
165
- 'for FP32 and FP16 models, and BF16 precision '
166
- 'for BF16 models.')
161
+ help='Data type for model weights and activations.\n\n'
162
+ '* "auto" will use FP16 precision for FP32 and FP16 models, and '
163
+ 'BF16 precision for BF16 models.\n'
164
+ '* "half" for FP16. Recommended for AWQ quantization.\n'
165
+ '* "float16" is the same as "half".\n'
166
+ '* "bfloat16" for a balance between precision and range.\n'
167
+ '* "float" is shorthand for FP32 precision.\n'
168
+ '* "float32" for FP32 precision.')
169
parser.add_argument(
170
'--kv-cache-dtype',
171
type=str,
@@ -172,7 +174,7 @@ class EngineArgs:
174
help='Data type for kv cache storage. If "auto", will use model '
175
'data type. FP8_E5M2 (without scaling) is only supported on cuda '
176
'version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
175
- 'supported for common inference criteria. ')
177
+ 'supported for common inference criteria.')
178
parser.add_argument(
179
'--quantization-param-path',
180
type=str,
@@ -183,58 +185,59 @@ class EngineArgs:
185
'default to 1.0, which may cause accuracy issues. '
186
'FP8_E5M2 (without scaling) is only supported on cuda version'
187
'greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
186
- 'supported for common inference criteria. ')
188
+ 'supported for common inference criteria.')
189
parser.add_argument('--max-model-len',
190
type=int,
191
default=EngineArgs.max_model_len,
190
- help='model context length. If unspecified, '
191
- 'will be automatically derived from the model.')
192
+ help='Model context length. If unspecified, will '
193
+ 'be automatically derived from the model config.')
194
parser.add_argument(
195
'--guided-decoding-backend',
196
type=str,
197
default='outlines',
198
choices=['outlines', 'lm-format-enforcer'],
199
help='Which engine will be used for guided decoding'
198
- ' (JSON schema / regex etc)')
200
+ ' (JSON schema / regex etc).')
201
# Parallel arguments
202
parser.add_argument('--worker-use-ray',
203
action='store_true',
202
- help='use Ray for distributed serving, will be '
203
- 'automatically set when using more than 1 GPU')
204
+ help='Use Ray for distributed serving, will be '
205
+ 'automatically set when using more than 1 GPU.')
206
parser.add_argument('--pipeline-parallel-size',
207
'-pp',
208
type=int,
209
default=EngineArgs.pipeline_parallel_size,
208
- help='number of pipeline stages')
210
+ help='Number of pipeline stages.')
211
parser.add_argument('--tensor-parallel-size',
212
'-tp',
213
type=int,
214
default=EngineArgs.tensor_parallel_size,
213
- help='number of tensor parallel replicas')
215
+ help='Number of tensor parallel replicas.')
216
parser.add_argument(
217
'--max-parallel-loading-workers',
218
type=int,
219
default=EngineArgs.max_parallel_loading_workers,
218
- help='load model sequentially in multiple batches, '
220
+ help='Load model sequentially in multiple batches, '
221
'to avoid RAM OOM when using tensor '
220
- 'parallel and large models')
222
+ 'parallel and large models.')
223
parser.add_argument(
224
'--ray-workers-use-nsight',
225
action='store_true',
224
- help='If specified, use nsight to profile ray workers')
226
+ help='If specified, use nsight to profile Ray workers.')
227
# KV cache arguments
228
parser.add_argument('--block-size',
229
type=int,
230
default=EngineArgs.block_size,
231
choices=[8, 16, 32, 128],
230
- help='token block size')
232
+ help='Token block size for contiguous chunks of '
233
+ 'tokens.')
234
235
parser.add_argument('--enable-prefix-caching',
236
action='store_true',
234
- help='Enables automatic prefix caching')
237
+ help='Enables automatic prefix caching.')
238
parser.add_argument('--use-v2-block-manager',
239
action='store_true',
237
- help='Use BlockSpaceMangerV2')
240
+ help='Use BlockSpaceMangerV2.')
241
parser.add_argument(
242
'--num-lookahead-slots',
243
type=int,
@@ -247,18 +250,19 @@ class EngineArgs:
250
parser.add_argument('--seed',
251
type=int,
252
default=EngineArgs.seed,
250
- help='random seed')
253
+ help='Random seed for operations.')
254
parser.add_argument('--swap-space',
255
type=int,
256
default=EngineArgs.swap_space,
254
- help='CPU swap space size (GiB) per GPU')
257
+ help='CPU swap space size (GiB) per GPU.')
258
parser.add_argument(
259
'--gpu-memory-utilization',
260
type=float,
261
default=EngineArgs.gpu_memory_utilization,
259
- help='the fraction of GPU memory to be used for '
260
- 'the model executor, which can range from 0 to 1.'
261
- 'If unspecified, will use the default value of 0.9.')
262
+ help='The fraction of GPU memory to be used for the model '
263
+ 'executor, which can range from 0 to 1. For example, a value of '
264
+ '0.5 would imply 50%% GPU memory utilization. If unspecified, '
265
+ 'will use the default value of 0.9.')
266
parser.add_argument(
267
'--num-gpu-blocks-override',
268
type=int,
@@ -268,21 +272,21 @@ class EngineArgs:
272
parser.add_argument('--max-num-batched-tokens',
273
type=int,
274
default=EngineArgs.max_num_batched_tokens,
271
- help='maximum number of batched tokens per '
272
- 'iteration')
275
+ help='Maximum number of batched tokens per '
276
+ 'iteration.')
277
parser.add_argument('--max-num-seqs',
278
type=int,
279
default=EngineArgs.max_num_seqs,
276
- help='maximum number of sequences per iteration')
280
+ help='Maximum number of sequences per iteration.')
281
parser.add_argument(
282
'--max-logprobs',
283
type=int,
284
default=EngineArgs.max_logprobs,
281
- help=('max number of log probs to return logprobs is specified in'
282
- ' SamplingParams'))
285
+ help=('Max number of log probs to return logprobs is specified in'
286
+ ' SamplingParams.'))
287
parser.add_argument('--disable-log-stats',
288
action='store_true',
285
- help='disable logging statistics')
289
+ help='Disable logging statistics.')
290
# Quantization settings.
291
parser.add_argument('--quantization',
292
'-q',
@@ -303,13 +307,13 @@ class EngineArgs:
307
parser.add_argument('--max-context-len-to-capture',
308
type=int,
309
default=EngineArgs.max_context_len_to_capture,
306
- help='maximum context length covered by CUDA '
310
+ help='Maximum context length covered by CUDA '
311
'graphs. When a sequence has context length '
312
'larger than this, we fall back to eager mode.')
313
parser.add_argument('--disable-custom-all-reduce',
314
action='store_true',
315
default=EngineArgs.disable_custom_all_reduce,
312
- help='See ParallelConfig')
316
+ help='See ParallelConfig.')
317
parser.add_argument('--tokenizer-pool-size',
318
type=int,
319
default=EngineArgs.tokenizer_pool_size,
@@ -402,7 +406,7 @@ class EngineArgs:
406
'--enable-chunked-prefill',
407
action='store_true',
408
help='If set, the prefill requests can be chunked based on the '
405
- 'max_num_batched_tokens')
409
+ 'max_num_batched_tokens.')
410
411
parser.add_argument(
412
'--speculative-model',
@@ -416,7 +420,7 @@ class EngineArgs:
420
type=int,
421
default=None,
422
help='The number of speculative tokens to sample from '
419
- 'the draft model in speculative decoding')
423
+ 'the draft model in speculative decoding.')
424
425
parser.add_argument('--model-loader-extra-config',
426
type=str,
@@ -534,20 +538,31 @@ class AsyncEngineArgs(EngineArgs):
538
max_log_len: Optional[int] = None
539
540
@staticmethod
537
- def add_cli_args(
538
- parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
539
- parser = EngineArgs.add_cli_args(parser)
541
+ def add_cli_args(parser: argparse.ArgumentParser,
542
+ async_args_only: bool = False) -> argparse.ArgumentParser:
543
+ if not async_args_only:
544
+ parser = EngineArgs.add_cli_args(parser)
545
parser.add_argument('--engine-use-ray',
546
action='store_true',
542
- help='use Ray to start the LLM engine in a '
547
+ help='Use Ray to start the LLM engine in a '
548
'separate process as the server process.')
549
parser.add_argument('--disable-log-requests',
550
action='store_true',
546
- help='disable logging requests')
551
+ help='Disable logging requests.')
552
parser.add_argument('--max-log-len',
553
type=int,
554
default=None,
550
- help='max number of prompt characters or prompt '
551
- 'ID numbers being printed in log. '
552
- 'Default: unlimited.')
555
+ help='Max number of prompt characters or prompt '
556
+ 'ID numbers being printed in log.'
557
+ '\n\nDefault: Unlimited')
558
return parser
559
+
560
+
561
+# These functions are used by sphinx to build the documentation
562
+def _engine_args_parser():
563
+ return EngineArgs.add_cli_args(argparse.ArgumentParser())
564
+
565
+
566
+def _async_engine_args_parser():
567
+ return AsyncEngineArgs.add_cli_args(argparse.ArgumentParser(),
568
+ async_args_only=True)