@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e78587a64c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 13:30 UTC

Improve-mm-and-pooler-and-decoding-configs (#16789)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Apr 18, 2025 at 06:13 UTC e78587a64ccf236c8506c8b536965ac74560ab48

14 files changed +84 -78

docs/source/models/supported_models.md

+1 -1

@@ -788,7 +788,7 @@ llm = LLM(

788 Online serving:

789

790 ```bash

791 -vllm serve Qwen/Qwen2-VL-7B-Instruct --limit-mm-per-prompt image=4

791 +vllm serve Qwen/Qwen2-VL-7B-Instruct --limit-mm-per-prompt '{"image":4}'

792 ```

793

794 **This is no longer required if you are using vLLM V1.**

docs/source/serving/multimodal_inputs.md

+1 -1

@@ -228,7 +228,7 @@ First, launch the OpenAI-compatible server:

228

229 ```bash

230 vllm serve microsoft/Phi-3.5-vision-instruct --task generate \

231 - --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2

231 + --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt '{"image":2}'

232 ```

233

234 Then, you can use the OpenAI client as follows:

examples/offline_inference/mistral-small.py

+2 -2

@@ -16,11 +16,11 @@ from vllm.sampling_params import SamplingParams

16 # # Mistral format

17 # vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \

18 # --tokenizer-mode mistral --config-format mistral --load-format mistral \

19 -# --limit-mm-per-prompt 'image=4' --max-model-len 16384

19 +# --limit-mm-per-prompt '{"image":4}' --max-model-len 16384

20 #

21 # # HF format

22 # vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \

23 -# --limit-mm-per-prompt 'image=4' --max-model-len 16384

23 +# --limit-mm-per-prompt '{"image":4}' --max-model-len 16384

24 # ```

25 #

26 # - Client:

examples/online_serving/openai_chat_completion_client_for_multimodal.py

+1 -1

@@ -9,7 +9,7 @@ vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja

10 (multi-image inference with Phi-3.5-vision-instruct)

11 vllm serve microsoft/Phi-3.5-vision-instruct --task generate \

12 - --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2

12 + --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt '{"image":2}'

14 (audio inference with Ultravox)

15 vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b --max-model-len 4096

tests/engine/test_arg_utils.py

@@ -24,6 +24,10 @@ from vllm.utils import FlexibleArgumentParser

24 }),

25 ])

26 def test_limit_mm_per_prompt_parser(arg, expected):

27 + """This functionality is deprecated and will be removed in the future.

28 + This argument should be passed as JSON string instead.

29 +

30 + TODO: Remove with nullable_kvs."""

31 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())

32 if arg is None:

33 args = parser.parse_args([])

tests/entrypoints/openai/test_audio.py

+1 -1

@@ -27,7 +27,7 @@ def server():

27 "--enforce-eager",

28 "--trust-remote-code",

29 "--limit-mm-per-prompt",

30 - f"audio={MAXIMUM_AUDIOS}",

30 + str({"audio": MAXIMUM_AUDIOS}),

31 ]

33 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

tests/entrypoints/openai/test_video.py

+1 -1

@@ -31,7 +31,7 @@ def server():

31 "--enforce-eager",

32 "--trust-remote-code",

33 "--limit-mm-per-prompt",

34 - f"video={MAXIMUM_VIDEOS}",

34 + str({"video": MAXIMUM_VIDEOS}),

35 ]

37 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

tests/entrypoints/openai/test_vision.py

+1 -1

@@ -35,7 +35,7 @@ def server():

35 "--enforce-eager",

36 "--trust-remote-code",

37 "--limit-mm-per-prompt",

38 - f"image={MAXIMUM_IMAGES}",

38 + str({"image": MAXIMUM_IMAGES}),

39 ]

41 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

tests/entrypoints/openai/test_vision_embedding.py

+1 -1

@@ -37,7 +37,7 @@ def server():

37 "--enforce-eager",

38 "--trust-remote-code",

39 "--limit-mm-per-prompt",

40 - f"image={MAXIMUM_IMAGES}",

40 + str({"image": MAXIMUM_IMAGES}),

41 "--chat-template",

42 str(vlm2vec_jinja_path),

43 ]

tests/models/decoder_only/audio_language/test_ultravox.py

+3 -3

@@ -48,9 +48,9 @@ def audio(request):

48 ])

49 def server(request, audio_assets):

50 args = [

51 - "--dtype=bfloat16", "--max-model-len=4096", "--enforce-eager",

52 - f"--limit-mm-per-prompt=audio={len(audio_assets)}",

53 - "--trust-remote-code"

51 + "--dtype", "bfloat16", "--max-model-len", "4096", "--enforce-eager",

52 + "--limit-mm-per-prompt",

53 + str({"audio": len(audio_assets)}), "--trust-remote-code"

54 ] + [

55 f"--{key.replace('_','-')}={value}"

56 for key, value in request.param.items()

vllm/config.py

+28 -21

@@ -17,7 +17,7 @@ from dataclasses import (MISSING, dataclass, field, fields, is_dataclass,

17 from importlib.util import find_spec

18 from pathlib import Path

19 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Final, Literal,

20 - Optional, Protocol, TypeVar, Union)

20 + Optional, Protocol, TypeVar, Union, get_args)

22 import torch

23 from pydantic import BaseModel, Field, PrivateAttr

@@ -2725,6 +2725,7 @@ class PromptAdapterConfig:

2725 self.prompt_adapter_dtype)

2726

2727

2728 +@config

2729 @dataclass

2730 class MultiModalConfig:

2731 """Controls the behavior of multimodal models."""

@@ -2732,6 +2733,8 @@ class MultiModalConfig:

2733 limit_per_prompt: Mapping[str, int] = field(default_factory=dict)

2734 """

2735 The maximum number of input items allowed per prompt for each modality.

2736 + This should be a JSON string that will be parsed into a dictionary.

2737 + Defaults to 1 (V0) or 999 (V1) for each modality.

2738 """

2739

2740 def compute_hash(self) -> str:

@@ -2753,24 +2756,20 @@ class MultiModalConfig:

2756 usedforsecurity=False).hexdigest()

2757 return hash_str

2758

2756 - def get_default_limit_per_prompt(self) -> int:

2757 - """

2758 - Return the default number of input items allowed per prompt

2759 - for any modality if not specified by the user.

2760 - """

2761 - return 999 if envs.VLLM_USE_V1 else 1

2762 -

2759 def get_limit_per_prompt(self, modality: str) -> int:

2760 """

2761 Get the maximum number of input items allowed per prompt

2762 for the given modality.

2763 """

2768 - default = self.get_default_limit_per_prompt()

2769 - return self.limit_per_prompt.get(modality, default)

2764 + return self.limit_per_prompt.get(

2765 + modality,

2766 + 999 if envs.VLLM_USE_V1 else 1,

2767 + )

2768

2769 # TODO: Add configs to init vision tower or not.

2770

2771

2772 +@config

2773 @dataclass

2774 class PoolerConfig:

2775 """Controls the behavior of output pooling in pooling models."""

@@ -3095,15 +3094,28 @@ def get_served_model_name(model: str,

3094 return served_model_name

3095

3096

3097 +GuidedDecodingBackendV0 = Literal["auto", "outlines", "lm-format-enforcer",

3098 + "xgrammar"]

3099 +GuidedDecodingBackendV1 = Literal["auto", "xgrammar", "guidance"]

3100 +

3101 +

3102 +@config

3103 @dataclass

3104 class DecodingConfig:

3100 - """Dataclass which contains the decoding strategy of the engine"""

3105 + """Dataclass which contains the decoding strategy of the engine."""

3106

3102 - # Which guided decoding algo to use.

3103 - # 'outlines' / 'lm-format-enforcer' / 'xgrammar'

3104 - guided_decoding_backend: str = "auto" if envs.VLLM_USE_V1 else "xgrammar"

3107 + guided_decoding_backend: Union[

3108 + GuidedDecodingBackendV0,

3109 + GuidedDecodingBackendV1] = "auto" if envs.VLLM_USE_V1 else "xgrammar"

3110 + """Which engine will be used for guided decoding (JSON schema / regex etc)

3111 + by default. With "auto", we will make opinionated choices based on request

3112 + contents and what the backend libraries currently support, so the behavior

3113 + is subject to change in each release."""

3114

3115 reasoning_backend: Optional[str] = None

3116 + """Select the reasoning parser depending on the model that you're using.

3117 + This is used to parse the reasoning content into OpenAI API format.

3118 + Required for `--enable-reasoning`."""

3119

3120 def compute_hash(self) -> str:

3121 """

@@ -3125,17 +3137,12 @@ class DecodingConfig:

3137 return hash_str

3138

3139 def __post_init__(self):

3128 - v0_valid_guided_backends = [

3129 - 'outlines', 'lm-format-enforcer', 'xgrammar', 'auto'

3130 - ]

3131 - v1_valid_guided_backends = ['xgrammar', 'guidance', 'auto']

3132 -

3140 backend = GuidedDecodingParams(

3141 backend=self.guided_decoding_backend).backend_name

3142 if envs.VLLM_USE_V1:

3136 - valid_guided_backends = v1_valid_guided_backends

3143 + valid_guided_backends = get_args(GuidedDecodingBackendV1)

3144 else:

3138 - valid_guided_backends = v0_valid_guided_backends

3145 + valid_guided_backends = get_args(GuidedDecodingBackendV0)

3146 if backend not in valid_guided_backends:

3147 raise ValueError(f"Invalid guided_decoding_backend '{backend}',"

3148 f" must be one of {valid_guided_backends}")

vllm/engine/arg_utils.py

+32 -40

@@ -20,11 +20,12 @@ from vllm.config import (CacheConfig, CompilationConfig, Config, ConfigFormat,

20 DecodingConfig, Device, DeviceConfig,

21 DistributedExecutorBackend, HfOverrides,

22 KVTransferConfig, LoadConfig, LoadFormat, LoRAConfig,

23 - ModelConfig, ModelImpl, ObservabilityConfig,

24 - ParallelConfig, PoolerConfig, PoolType,

25 - PromptAdapterConfig, SchedulerConfig, SchedulerPolicy,

26 - SpeculativeConfig, TaskOption, TokenizerPoolConfig,

27 - VllmConfig, get_attr_docs, get_field)

23 + ModelConfig, ModelImpl, MultiModalConfig,

24 + ObservabilityConfig, ParallelConfig, PoolerConfig,

25 + PoolType, PromptAdapterConfig, SchedulerConfig,

26 + SchedulerPolicy, SpeculativeConfig, TaskOption,

27 + TokenizerPoolConfig, VllmConfig, get_attr_docs,

28 + get_field)

29 from vllm.executor.executor_base import ExecutorBase

30 from vllm.logger import init_logger

31 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS

@@ -190,7 +191,8 @@ class EngineArgs:

191 TokenizerPoolConfig.pool_type

192 tokenizer_pool_extra_config: dict[str, Any] = \

193 get_field(TokenizerPoolConfig, "extra_config")

193 - limit_mm_per_prompt: Optional[Mapping[str, int]] = None

194 + limit_mm_per_prompt: Mapping[str, int] = \

195 + get_field(MultiModalConfig, "limit_per_prompt")

196 mm_processor_kwargs: Optional[Dict[str, Any]] = None

197 disable_mm_preprocessor_cache: bool = False

198 enable_lora: bool = False

@@ -252,7 +254,7 @@ class EngineArgs:

254

255 additional_config: Optional[Dict[str, Any]] = None

256 enable_reasoning: Optional[bool] = None

255 - reasoning_parser: Optional[str] = None

257 + reasoning_parser: Optional[str] = DecodingConfig.reasoning_backend

258 use_tqdm_on_load: bool = LoadConfig.use_tqdm_on_load

259

260 def __post_init__(self):

@@ -478,18 +480,22 @@ class EngineArgs:

480 'Examples:\n'

481 '- 1k → 1000\n'

482 '- 1K → 1024\n')

481 - parser.add_argument(

483 +

484 + # Guided decoding arguments

485 + guided_decoding_kwargs = get_kwargs(DecodingConfig)

486 + guided_decoding_group = parser.add_argument_group(

487 + title="DecodingConfig",

488 + description=DecodingConfig.__doc__,

489 + )

490 + guided_decoding_group.add_argument(

491 '--guided-decoding-backend',

483 - type=str,

484 - default=DecodingConfig.guided_decoding_backend,

485 - help='Which engine will be used for guided decoding'

486 - ' (JSON schema / regex etc) by default. Currently support '

487 - 'https://github.com/mlc-ai/xgrammar and '

488 - 'https://github.com/guidance-ai/llguidance.'

489 - 'Valid backend values are "xgrammar", "guidance", and "auto". '

490 - 'With "auto", we will make opinionated choices based on request '

491 - 'contents and what the backend libraries currently support, so '

492 - 'the behavior is subject to change in each release.')

492 + **guided_decoding_kwargs["guided_decoding_backend"])

493 + guided_decoding_group.add_argument(

494 + "--reasoning-parser",

495 + # This choices is a special case because it's not static

496 + choices=list(ReasoningParserManager.reasoning_parsers),

497 + **guided_decoding_kwargs["reasoning_backend"])

498 +

499 parser.add_argument(

500 '--logits-processor-pattern',

501 type=optional_str,

@@ -697,18 +703,14 @@ class EngineArgs:

703 **tokenizer_kwargs["extra_config"])

704

705 # Multimodal related configs

700 - parser.add_argument(

701 - '--limit-mm-per-prompt',

702 - type=nullable_kvs,

703 - default=EngineArgs.limit_mm_per_prompt,

704 - # The default value is given in

705 - # MultiModalConfig.get_default_limit_per_prompt

706 - help=('For each multimodal plugin, limit how many '

707 - 'input instances to allow for each prompt. '

708 - 'Expects a comma-separated list of items, '

709 - 'e.g.: `image=16,video=2` allows a maximum of 16 '

710 - 'images and 2 videos per prompt. Defaults to '

711 - '1 (V0) or 999 (V1) for each modality.'))

706 + multimodal_kwargs = get_kwargs(MultiModalConfig)

707 + multimodal_group = parser.add_argument_group(

708 + title="MultiModalConfig",

709 + description=MultiModalConfig.__doc__,

710 + )

711 + multimodal_group.add_argument('--limit-mm-per-prompt',

712 + **multimodal_kwargs["limit_per_prompt"])

713 +

714 parser.add_argument(

715 '--mm-processor-kwargs',

716 default=None,

@@ -1018,16 +1020,6 @@ class EngineArgs:

1020 "If enabled, the model will be able to generate reasoning content."

1021 )

1022

1021 - parser.add_argument(

1022 - "--reasoning-parser",

1023 - type=str,

1024 - choices=list(ReasoningParserManager.reasoning_parsers),

1025 - default=None,

1026 - help=

1027 - "Select the reasoning parser depending on the model that you're "

1028 - "using. This is used to parse the reasoning content into OpenAI "

1029 - "API format. Required for ``--enable-reasoning``.")

1030 -

1023 parser.add_argument(

1024 "--disable-cascade-attn",

1025 action="store_true",

vllm/multimodal/processing.py

+4 -2

@@ -1,4 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2 +import json

3 import re

4 import sys

5 from abc import ABC, abstractmethod

@@ -1117,8 +1118,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1118

1119 if num_items > allowed_limit:

1120 raise ValueError(

1120 - f"You set or defaulted to {modality}={allowed_limit} "

1121 - f"in --limit-mm-per-prompt`, but passed {num_items} "

1121 + "You set or defaulted to "

1122 + f"'{json.dumps({modality: allowed_limit})}' in "

1123 + f"`--limit-mm-per-prompt`, but passed {num_items} "

1124 f"{modality} items in the same prompt.")

1125

1126 return mm_items

vllm/multimodal/registry.py

+4 -3

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

3 import functools

4 +import json

5 from collections import UserDict

6 from collections.abc import Mapping, Sequence

7 from dataclasses import dataclass

@@ -194,9 +195,9 @@ class MultiModalRegistry:

195 max_items = self._limits_by_model[model_config][data_key]

196 if num_items > max_items:

197 raise ValueError(

197 - f"You set {data_key}={max_items} (or defaulted to 1) in "

198 - f"`--limit-mm-per-prompt`, but found {num_items} items "

199 - "in the same prompt.")

198 + f"You set '{json.dumps({data_key: max_items})}' (or "

199 + "defaulted to 1) in `--limit-mm-per-prompt`, but found "

200 + f"{num_items} items in the same prompt.")

201

202 input_dict = plugin.map_input(model_config, data_value,

203 mm_processor_kwargs)