A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Doc] Improve documentation for multimodal CLI args (#16960)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Apr 22, 2025 at 16:35 UTC
8f7bace7c3cfb2836ea56722c66277270d7254c6
2 files changed
+17
-10
vllm/config.py
+9
-4
@@ -54,13 +54,15 @@ if TYPE_CHECKING:
54
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
55
BaseTokenizerGroup)
56
57
- Config = TypeVar("Config", bound=DataclassInstance)
57
+ ConfigType = type[DataclassInstance]
58
else:
59
QuantizationConfig = None
60
- Config = TypeVar("Config")
60
+ ConfigType = type
61
62
logger = init_logger(__name__)
63
64
+ConfigT = TypeVar("ConfigT", bound=ConfigType)
65
+
66
# This value is chosen to have a balance between ITL and TTFT. Note it is
67
# not optimized for throughput.
68
_DEFAULT_MAX_NUM_BATCHED_TOKENS = 2048
@@ -162,7 +164,7 @@ def get_attr_docs(cls: type[Any]) -> dict[str, str]:
164
return out
165
166
165
-def config(cls: type[Config]) -> type[Config]:
167
+def config(cls: ConfigT) -> ConfigT:
168
"""
169
A decorator that ensures all fields in a dataclass have default values
170
and that each field has a docstring.
@@ -181,7 +183,7 @@ def config(cls: type[Config]) -> type[Config]:
183
return cls
184
185
184
-def get_field(cls: type[Config], name: str) -> Field:
186
+def get_field(cls: ConfigType, name: str) -> Field:
187
"""Get the default factory field of a dataclass by name. Used for getting
188
default factory fields in `EngineArgs`."""
189
if not is_dataclass(cls):
@@ -2749,6 +2751,9 @@ class MultiModalConfig:
2751
The maximum number of input items allowed per prompt for each modality.
2752
This should be a JSON string that will be parsed into a dictionary.
2753
Defaults to 1 (V0) or 999 (V1) for each modality.
2754
+
2755
+ For example, to allow up to 16 images and 2 videos per prompt:
2756
+ ``{"images": 16, "videos": 2}``
2757
"""
2758
2759
def compute_hash(self) -> str:
vllm/engine/arg_utils.py
+8
-6
@@ -17,7 +17,7 @@ from typing_extensions import TypeIs
17
import vllm.envs as envs
18
from vllm import version
19
from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
20
- Config, ConfigFormat, DecodingConfig, Device,
20
+ ConfigFormat, ConfigType, DecodingConfig, Device,
21
DeviceConfig, DistributedExecutorBackend, HfOverrides,
22
KVTransferConfig, LoadConfig, LoadFormat, LoRAConfig,
23
ModelConfig, ModelImpl, MultiModalConfig,
@@ -304,7 +304,7 @@ class EngineArgs:
304
"""Check if the class is a custom type."""
305
return cls.__module__ != "builtins"
306
307
- def get_kwargs(cls: type[Config]) -> dict[str, Any]:
307
+ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
308
cls_docs = get_attr_docs(cls)
309
kwargs = {}
310
for field in fields(cls):
@@ -678,13 +678,15 @@ class EngineArgs:
678
'--mm-processor-kwargs',
679
default=None,
680
type=json.loads,
681
- help=('Overrides for the multimodal input mapping/processing, '
682
- 'e.g., image processor. For example: ``{"num_crops": 4}``.'))
681
+ help=('Overrides for the multi-modal processor obtained from '
682
+ '``AutoProcessor.from_pretrained``. The available overrides '
683
+ 'depend on the model that is being run.'
684
+ 'For example, for Phi-3-Vision: ``{"num_crops": 4}``.'))
685
parser.add_argument(
686
'--disable-mm-preprocessor-cache',
687
action='store_true',
686
- help='If true, then disables caching of the multi-modal '
687
- 'preprocessor/mapper. (not recommended)')
688
+ help='If True, disable caching of the processed multi-modal '
689
+ 'inputs.')
690
691
# LoRA related configs
692
parser.add_argument('--enable-lora',