@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 682789d402
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Fix missing docs and out of sync `EngineArgs` (#4219)

Co-authored-by: Harry Mellor <hmellor@oxts.com>

Harry Mellor committed Apr 20, 2024 at 04:51 UTC 682789d4026429a04cc32acb88064265441080dd
3 files changed +98 -197
docs/source/conf.py
+2
@@ -11,12 +11,14 @@
11 # documentation root, use os.path.abspath to make it absolute, like shown here.
12
13 import logging
14 +import os
15 import sys
16 from typing import List
17
18 from sphinx.ext import autodoc
19
20 logger = logging.getLogger(__name__)
21 +sys.path.append(os.path.abspath("../.."))
22
23 # -- Project information -----------------------------------------------------
24
docs/source/models/engine_args.rst
+9 -125
@@ -5,133 +5,17 @@ Engine Arguments
5
6 Below, you can find an explanation of every engine argument for vLLM:
7
8 -.. option:: --model <model_name_or_path>
9 -
10 - Name or path of the huggingface model to use.
11 -
12 -.. option:: --tokenizer <tokenizer_name_or_path>
13 -
14 - Name or path of the huggingface tokenizer to use.
15 -
16 -.. option:: --revision <revision>
17 -
18 - The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.
19 -
20 -.. option:: --tokenizer-revision <revision>
21 -
22 - The specific tokenizer version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.
23 -
24 -.. option:: --tokenizer-mode {auto,slow}
25 -
26 - The tokenizer mode.
27 -
28 - * "auto" will use the fast tokenizer if available.
29 - * "slow" will always use the slow tokenizer.
30 -
31 -.. option:: --trust-remote-code
32 -
33 - Trust remote code from huggingface.
34 -
35 -.. option:: --download-dir <directory>
36 -
37 - Directory to download and load the weights, default to the default cache dir of huggingface.
38 -
39 -.. option:: --load-format {auto,pt,safetensors,npcache,dummy,tensorizer}
40 -
41 - The format of the model weights to load.
42 -
43 - * "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available.
44 - * "pt" will load the weights in the pytorch bin format.
45 - * "safetensors" will load the weights in the safetensors format.
46 - * "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading.
47 - * "dummy" will initialize the weights with random values, mainly for profiling.
48 - * "tensorizer" will load serialized weights using `CoreWeave's Tensorizer model deserializer. <https://github.com/coreweave/tensorizer>`_ See `examples/tensorize_vllm_model.py <https://github.com/vllm-project/vllm/blob/main/examples/tensorize_vllm_model.py>`_ to serialize a vLLM model, and for more information.
49 -
50 -.. option:: --dtype {auto,half,float16,bfloat16,float,float32}
51 -
52 - Data type for model weights and activations.
53 -
54 - * "auto" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models.
55 - * "half" for FP16. Recommended for AWQ quantization.
56 - * "float16" is the same as "half".
57 - * "bfloat16" for a balance between precision and range.
58 - * "float" is shorthand for FP32 precision.
59 - * "float32" for FP32 precision.
60 -
61 -.. option:: --max-model-len <length>
62 -
63 - Model context length. If unspecified, will be automatically derived from the model config.
64 -
65 -.. option:: --worker-use-ray
66 -
67 - Use Ray for distributed serving, will be automatically set when using more than 1 GPU.
68 -
69 -.. option:: --pipeline-parallel-size (-pp) <size>
70 -
71 - Number of pipeline stages.
72 -
73 -.. option:: --tensor-parallel-size (-tp) <size>
74 -
75 - Number of tensor parallel replicas.
76 -
77 -.. option:: --max-parallel-loading-workers <workers>
78 -
79 - Load model sequentially in multiple batches, to avoid RAM OOM when using tensor parallel and large models.
80 -
81 -.. option:: --block-size {8,16,32}
82 -
83 - Token block size for contiguous chunks of tokens.
84 -
85 -.. option:: --enable-prefix-caching
86 -
87 - Enables automatic prefix caching
88 -
89 -.. option:: --seed <seed>
90 -
91 - Random seed for operations.
92 -
93 -.. option:: --swap-space <size>
94 -
95 - CPU swap space size (GiB) per GPU.
96 -
97 -.. option:: --gpu-memory-utilization <fraction>
98 -
99 - The fraction of GPU memory to be used for the model executor, which can range from 0 to 1.
100 - For example, a value of 0.5 would imply 50% GPU memory utilization.
101 - If unspecified, will use the default value of 0.9.
102 -
103 -.. option:: --max-num-batched-tokens <tokens>
104 -
105 - Maximum number of batched tokens per iteration.
106 -
107 -.. option:: --max-num-seqs <sequences>
108 -
109 - Maximum number of sequences per iteration.
110 -
111 -.. option:: --max-paddings <paddings>
112 -
113 - Maximum number of paddings in a batch.
114 -
115 -.. option:: --disable-log-stats
116 -
117 - Disable logging statistics.
118 -
119 -.. option:: --quantization (-q) {awq,squeezellm,None}
120 -
121 - Method used to quantize the weights.
8 +.. argparse::
9 + :module: vllm.engine.arg_utils
10 + :func: _engine_args_parser
11 + :prog: -m vllm.entrypoints.openai.api_server
12
13 Async Engine Arguments
14 ----------------------
125 -Below are the additional arguments related to the asynchronous engine:
126 -
127 -.. option:: --engine-use-ray
15
129 - Use Ray to start the LLM engine in a separate process as the server process.
130 -
131 -.. option:: --disable-log-requests
132 -
133 - Disable logging requests.
134 -
135 -.. option:: --max-log-len
16 +Below are the additional arguments related to the asynchronous engine:
17
137 - Max number of prompt characters or prompt ID numbers being printed in log. Defaults to unlimited.
\ No newline at end of file
18 +.. argparse::
19 + :module: vllm.engine.arg_utils
20 + :func: _async_engine_args_parser
21 + :prog: -m vllm.entrypoints.openai.api_server
\ No newline at end of file
vllm/engine/arg_utils.py
+87 -72
@@ -82,57 +82,55 @@ class EngineArgs:
82 parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
83 """Shared CLI arguments for vLLM engine."""
84
85 - # NOTE: If you update any of the arguments below, please also
86 - # make sure to update docs/source/models/engine_args.rst
87 -
85 # Model arguments
86 parser.add_argument(
87 '--model',
88 type=str,
89 default='facebook/opt-125m',
93 - help='name or path of the huggingface model to use')
90 + help='Name or path of the huggingface model to use.')
91 parser.add_argument(
92 '--tokenizer',
93 type=str,
94 default=EngineArgs.tokenizer,
98 - help='name or path of the huggingface tokenizer to use')
95 + help='Name or path of the huggingface tokenizer to use.')
96 parser.add_argument(
97 '--revision',
98 type=str,
99 default=None,
103 - help='the specific model version to use. It can be a branch '
100 + help='The specific model version to use. It can be a branch '
101 'name, a tag name, or a commit id. If unspecified, will use '
102 'the default version.')
103 parser.add_argument(
104 '--code-revision',
105 type=str,
106 default=None,
110 - help='the specific revision to use for the model code on '
107 + help='The specific revision to use for the model code on '
108 'Hugging Face Hub. It can be a branch name, a tag name, or a '
109 'commit id. If unspecified, will use the default version.')
110 parser.add_argument(
111 '--tokenizer-revision',
112 type=str,
113 default=None,
117 - help='the specific tokenizer version to use. It can be a branch '
114 + help='The specific tokenizer version to use. It can be a branch '
115 'name, a tag name, or a commit id. If unspecified, will use '
116 'the default version.')
120 - parser.add_argument('--tokenizer-mode',
121 - type=str,
122 - default=EngineArgs.tokenizer_mode,
123 - choices=['auto', 'slow'],
124 - help='tokenizer mode. "auto" will use the fast '
125 - 'tokenizer if available, and "slow" will '
126 - 'always use the slow tokenizer.')
117 + parser.add_argument(
118 + '--tokenizer-mode',
119 + type=str,
120 + default=EngineArgs.tokenizer_mode,
121 + choices=['auto', 'slow'],
122 + help='The tokenizer mode.\n\n* "auto" will use the '
123 + 'fast tokenizer if available.\n* "slow" will '
124 + 'always use the slow tokenizer.')
125 parser.add_argument('--trust-remote-code',
126 action='store_true',
129 - help='trust remote code from huggingface')
127 + help='Trust remote code from huggingface.')
128 parser.add_argument('--download-dir',
129 type=str,
130 default=EngineArgs.download_dir,
133 - help='directory to download and load the weights, '
131 + help='Directory to download and load the weights, '
132 'default to the default cache dir of '
135 - 'huggingface')
133 + 'huggingface.')
134 parser.add_argument(
135 '--load-format',
136 type=str,
@@ -140,19 +138,19 @@ class EngineArgs:
138 choices=[
139 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer'
140 ],
143 - help='The format of the model weights to load. '
144 - '"auto" will try to load the weights in the safetensors format '
141 + help='The format of the model weights to load.\n\n'
142 + '* "auto" will try to load the weights in the safetensors format '
143 'and fall back to the pytorch bin format if safetensors format '
146 - 'is not available. '
147 - '"pt" will load the weights in the pytorch bin format. '
148 - '"safetensors" will load the weights in the safetensors format. '
149 - '"npcache" will load the weights in pytorch format and store '
150 - 'a numpy cache to speed up the loading. '
151 - '"dummy" will initialize the weights with random values, '
152 - 'which is mainly for profiling.'
153 - '"tensorizer" will load the weights using tensorizer from CoreWeave'
154 - 'which assumes tensorizer_uri is set to the location of the '
155 - 'serialized weights.')
144 + 'is not available.\n'
145 + '* "pt" will load the weights in the pytorch bin format.\n'
146 + '* "safetensors" will load the weights in the safetensors format.\n'
147 + '* "npcache" will load the weights in pytorch format and store '
148 + 'a numpy cache to speed up the loading.\n'
149 + '* "dummy" will initialize the weights with random values, '
150 + 'which is mainly for profiling.\n'
151 + '* "tensorizer" will load the weights using tensorizer from '
152 + 'CoreWeave which assumes tensorizer_uri is set to the location of '
153 + 'the serialized weights.')
154 parser.add_argument(
155 '--dtype',
156 type=str,
@@ -160,10 +158,14 @@ class EngineArgs:
158 choices=[
159 'auto', 'half', 'float16', 'bfloat16', 'float', 'float32'
160 ],
163 - help='data type for model weights and activations. '
164 - 'The "auto" option will use FP16 precision '
165 - 'for FP32 and FP16 models, and BF16 precision '
166 - 'for BF16 models.')
161 + help='Data type for model weights and activations.\n\n'
162 + '* "auto" will use FP16 precision for FP32 and FP16 models, and '
163 + 'BF16 precision for BF16 models.\n'
164 + '* "half" for FP16. Recommended for AWQ quantization.\n'
165 + '* "float16" is the same as "half".\n'
166 + '* "bfloat16" for a balance between precision and range.\n'
167 + '* "float" is shorthand for FP32 precision.\n'
168 + '* "float32" for FP32 precision.')
169 parser.add_argument(
170 '--kv-cache-dtype',
171 type=str,
@@ -172,7 +174,7 @@ class EngineArgs:
174 help='Data type for kv cache storage. If "auto", will use model '
175 'data type. FP8_E5M2 (without scaling) is only supported on cuda '
176 'version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
175 - 'supported for common inference criteria. ')
177 + 'supported for common inference criteria.')
178 parser.add_argument(
179 '--quantization-param-path',
180 type=str,
@@ -183,58 +185,59 @@ class EngineArgs:
185 'default to 1.0, which may cause accuracy issues. '
186 'FP8_E5M2 (without scaling) is only supported on cuda version'
187 'greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
186 - 'supported for common inference criteria. ')
188 + 'supported for common inference criteria.')
189 parser.add_argument('--max-model-len',
190 type=int,
191 default=EngineArgs.max_model_len,
190 - help='model context length. If unspecified, '
191 - 'will be automatically derived from the model.')
192 + help='Model context length. If unspecified, will '
193 + 'be automatically derived from the model config.')
194 parser.add_argument(
195 '--guided-decoding-backend',
196 type=str,
197 default='outlines',
198 choices=['outlines', 'lm-format-enforcer'],
199 help='Which engine will be used for guided decoding'
198 - ' (JSON schema / regex etc)')
200 + ' (JSON schema / regex etc).')
201 # Parallel arguments
202 parser.add_argument('--worker-use-ray',
203 action='store_true',
202 - help='use Ray for distributed serving, will be '
203 - 'automatically set when using more than 1 GPU')
204 + help='Use Ray for distributed serving, will be '
205 + 'automatically set when using more than 1 GPU.')
206 parser.add_argument('--pipeline-parallel-size',
207 '-pp',
208 type=int,
209 default=EngineArgs.pipeline_parallel_size,
208 - help='number of pipeline stages')
210 + help='Number of pipeline stages.')
211 parser.add_argument('--tensor-parallel-size',
212 '-tp',
213 type=int,
214 default=EngineArgs.tensor_parallel_size,
213 - help='number of tensor parallel replicas')
215 + help='Number of tensor parallel replicas.')
216 parser.add_argument(
217 '--max-parallel-loading-workers',
218 type=int,
219 default=EngineArgs.max_parallel_loading_workers,
218 - help='load model sequentially in multiple batches, '
220 + help='Load model sequentially in multiple batches, '
221 'to avoid RAM OOM when using tensor '
220 - 'parallel and large models')
222 + 'parallel and large models.')
223 parser.add_argument(
224 '--ray-workers-use-nsight',
225 action='store_true',
224 - help='If specified, use nsight to profile ray workers')
226 + help='If specified, use nsight to profile Ray workers.')
227 # KV cache arguments
228 parser.add_argument('--block-size',
229 type=int,
230 default=EngineArgs.block_size,
231 choices=[8, 16, 32, 128],
230 - help='token block size')
232 + help='Token block size for contiguous chunks of '
233 + 'tokens.')
234
235 parser.add_argument('--enable-prefix-caching',
236 action='store_true',
234 - help='Enables automatic prefix caching')
237 + help='Enables automatic prefix caching.')
238 parser.add_argument('--use-v2-block-manager',
239 action='store_true',
237 - help='Use BlockSpaceMangerV2')
240 + help='Use BlockSpaceMangerV2.')
241 parser.add_argument(
242 '--num-lookahead-slots',
243 type=int,
@@ -247,18 +250,19 @@ class EngineArgs:
250 parser.add_argument('--seed',
251 type=int,
252 default=EngineArgs.seed,
250 - help='random seed')
253 + help='Random seed for operations.')
254 parser.add_argument('--swap-space',
255 type=int,
256 default=EngineArgs.swap_space,
254 - help='CPU swap space size (GiB) per GPU')
257 + help='CPU swap space size (GiB) per GPU.')
258 parser.add_argument(
259 '--gpu-memory-utilization',
260 type=float,
261 default=EngineArgs.gpu_memory_utilization,
259 - help='the fraction of GPU memory to be used for '
260 - 'the model executor, which can range from 0 to 1.'
261 - 'If unspecified, will use the default value of 0.9.')
262 + help='The fraction of GPU memory to be used for the model '
263 + 'executor, which can range from 0 to 1. For example, a value of '
264 + '0.5 would imply 50%% GPU memory utilization. If unspecified, '
265 + 'will use the default value of 0.9.')
266 parser.add_argument(
267 '--num-gpu-blocks-override',
268 type=int,
@@ -268,21 +272,21 @@ class EngineArgs:
272 parser.add_argument('--max-num-batched-tokens',
273 type=int,
274 default=EngineArgs.max_num_batched_tokens,
271 - help='maximum number of batched tokens per '
272 - 'iteration')
275 + help='Maximum number of batched tokens per '
276 + 'iteration.')
277 parser.add_argument('--max-num-seqs',
278 type=int,
279 default=EngineArgs.max_num_seqs,
276 - help='maximum number of sequences per iteration')
280 + help='Maximum number of sequences per iteration.')
281 parser.add_argument(
282 '--max-logprobs',
283 type=int,
284 default=EngineArgs.max_logprobs,
281 - help=('max number of log probs to return logprobs is specified in'
282 - ' SamplingParams'))
285 + help=('Max number of log probs to return logprobs is specified in'
286 + ' SamplingParams.'))
287 parser.add_argument('--disable-log-stats',
288 action='store_true',
285 - help='disable logging statistics')
289 + help='Disable logging statistics.')
290 # Quantization settings.
291 parser.add_argument('--quantization',
292 '-q',
@@ -303,13 +307,13 @@ class EngineArgs:
307 parser.add_argument('--max-context-len-to-capture',
308 type=int,
309 default=EngineArgs.max_context_len_to_capture,
306 - help='maximum context length covered by CUDA '
310 + help='Maximum context length covered by CUDA '
311 'graphs. When a sequence has context length '
312 'larger than this, we fall back to eager mode.')
313 parser.add_argument('--disable-custom-all-reduce',
314 action='store_true',
315 default=EngineArgs.disable_custom_all_reduce,
312 - help='See ParallelConfig')
316 + help='See ParallelConfig.')
317 parser.add_argument('--tokenizer-pool-size',
318 type=int,
319 default=EngineArgs.tokenizer_pool_size,
@@ -402,7 +406,7 @@ class EngineArgs:
406 '--enable-chunked-prefill',
407 action='store_true',
408 help='If set, the prefill requests can be chunked based on the '
405 - 'max_num_batched_tokens')
409 + 'max_num_batched_tokens.')
410
411 parser.add_argument(
412 '--speculative-model',
@@ -416,7 +420,7 @@ class EngineArgs:
420 type=int,
421 default=None,
422 help='The number of speculative tokens to sample from '
419 - 'the draft model in speculative decoding')
423 + 'the draft model in speculative decoding.')
424
425 parser.add_argument('--model-loader-extra-config',
426 type=str,
@@ -534,20 +538,31 @@ class AsyncEngineArgs(EngineArgs):
538 max_log_len: Optional[int] = None
539
540 @staticmethod
537 - def add_cli_args(
538 - parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
539 - parser = EngineArgs.add_cli_args(parser)
541 + def add_cli_args(parser: argparse.ArgumentParser,
542 + async_args_only: bool = False) -> argparse.ArgumentParser:
543 + if not async_args_only:
544 + parser = EngineArgs.add_cli_args(parser)
545 parser.add_argument('--engine-use-ray',
546 action='store_true',
542 - help='use Ray to start the LLM engine in a '
547 + help='Use Ray to start the LLM engine in a '
548 'separate process as the server process.')
549 parser.add_argument('--disable-log-requests',
550 action='store_true',
546 - help='disable logging requests')
551 + help='Disable logging requests.')
552 parser.add_argument('--max-log-len',
553 type=int,
554 default=None,
550 - help='max number of prompt characters or prompt '
551 - 'ID numbers being printed in log. '
552 - 'Default: unlimited.')
555 + help='Max number of prompt characters or prompt '
556 + 'ID numbers being printed in log.'
557 + '\n\nDefault: Unlimited')
558 return parser
559 +
560 +
561 +# These functions are used by sphinx to build the documentation
562 +def _engine_args_parser():
563 + return EngineArgs.add_cli_args(argparse.ArgumentParser())
564 +
565 +
566 +def _async_engine_args_parser():
567 + return AsyncEngineArgs.add_cli_args(argparse.ArgumentParser(),
568 + async_args_only=True)