A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[DOCS] Add engine args documentation (#1741)
Casper committed
Nov 22, 2023 at 21:31 UTC
a921d8be9dd8b98795b4d8076f3af4f48dc3d24d
3 files changed
+119
docs/source/index.rst
+1
@@ -73,6 +73,7 @@ Documentation
73
74
models/supported_models
75
models/adding_model
76
+ models/engine_args
77
78
.. toctree::
79
:maxdepth: 1
docs/source/models/engine_args.rst
new
+114
@@ -0,0 +1,114 @@
1
+.. _engine_args:
2
+
3
+Engine Arguments
4
+================
5
+
6
+Below, you can find an explanation of every engine argument for vLLM:
7
+
8
+.. option:: --model <model_name_or_path>
9
+
10
+ Name or path of the huggingface model to use.
11
+
12
+.. option:: --tokenizer <tokenizer_name_or_path>
13
+
14
+ Name or path of the huggingface tokenizer to use.
15
+
16
+.. option:: --revision <revision>
17
+
18
+ The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.
19
+
20
+.. option:: --tokenizer-revision <revision>
21
+
22
+ The specific tokenizer version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.
23
+
24
+.. option:: --tokenizer-mode {auto,slow}
25
+
26
+ The tokenizer mode.
27
+
28
+ * "auto" will use the fast tokenizer if available.
29
+ * "slow" will always use the slow tokenizer.
30
+
31
+.. option:: --trust-remote-code
32
+
33
+ Trust remote code from huggingface.
34
+
35
+.. option:: --download-dir <directory>
36
+
37
+ Directory to download and load the weights, default to the default cache dir of huggingface.
38
+
39
+.. option:: --load-format {auto,pt,safetensors,npcache,dummy}
40
+
41
+ The format of the model weights to load.
42
+
43
+ * "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available.
44
+ * "pt" will load the weights in the pytorch bin format.
45
+ * "safetensors" will load the weights in the safetensors format.
46
+ * "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading.
47
+ * "dummy" will initialize the weights with random values, mainly for profiling.
48
+
49
+.. option:: --dtype {auto,half,float16,bfloat16,float,float32}
50
+
51
+ Data type for model weights and activations.
52
+
53
+ * "auto" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models.
54
+ * "half" for FP16. Recommended for AWQ quantization.
55
+ * "float16" is the same as "half".
56
+ * "bfloat16" for a balance between precision and range.
57
+ * "float" is shorthand for FP32 precision.
58
+ * "float32" for FP32 precision.
59
+
60
+.. option:: --max-model-len <length>
61
+
62
+ Model context length. If unspecified, will be automatically derived from the model config.
63
+
64
+.. option:: --worker-use-ray
65
+
66
+ Use Ray for distributed serving, will be automatically set when using more than 1 GPU.
67
+
68
+.. option:: --pipeline-parallel-size (-pp) <size>
69
+
70
+ Number of pipeline stages.
71
+
72
+.. option:: --tensor-parallel-size (-tp) <size>
73
+
74
+ Number of tensor parallel replicas.
75
+
76
+.. option:: --max-parallel-loading-workers <workers>
77
+
78
+ Load model sequentially in multiple batches, to avoid RAM OOM when using tensor parallel and large models.
79
+
80
+.. option:: --block-size {8,16,32}
81
+
82
+ Token block size for contiguous chunks of tokens.
83
+
84
+.. option:: --seed <seed>
85
+
86
+ Random seed for operations.
87
+
88
+.. option:: --swap-space <size>
89
+
90
+ CPU swap space size (GiB) per GPU.
91
+
92
+.. option:: --gpu-memory-utilization <percentage>
93
+
94
+ The percentage of GPU memory to be used for the model executor.
95
+
96
+.. option:: --max-num-batched-tokens <tokens>
97
+
98
+ Maximum number of batched tokens per iteration.
99
+
100
+.. option:: --max-num-seqs <sequences>
101
+
102
+ Maximum number of sequences per iteration.
103
+
104
+.. option:: --max-paddings <paddings>
105
+
106
+ Maximum number of paddings in a batch.
107
+
108
+.. option:: --disable-log-stats
109
+
110
+ Disable logging statistics.
111
+
112
+.. option:: --quantization (-q) {awq,squeezellm,None}
113
+
114
+ Method used to quantize the weights.
vllm/engine/arg_utils.py
+4
@@ -42,6 +42,10 @@ class EngineArgs:
42
def add_cli_args(
43
parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
44
"""Shared CLI arguments for vLLM engine."""
45
+
46
+ # NOTE: If you update any of the arguments below, please also
47
+ # make sure to update docs/source/models/engine_args.rst
48
+
49
# Model arguments
50
parser.add_argument(
51
'--model',