@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2e7cbf2d7d

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

[Frontend] Support configurable mm placeholder strings & flexible video sampling policies via CLI flags. (#20105)

Signed-off-by: Chenheli Hua <huachenheli@outlook.com>

Chenheli Hua committed Jul 1, 2025 at 23:34 UTC 2e7cbf2d7da1030d141dd32e0271697b49b84e54

12 files changed +199 -29

tests/async_engine/test_async_llm_engine.py

+4 -2

@@ -6,8 +6,8 @@ import os

6 import uuid

7 from asyncio import CancelledError

8 from copy import copy

9 -from dataclasses import dataclass

10 -from typing import Optional

9 +from dataclasses import dataclass, field

10 +from typing import Any, Optional

12 import pytest

13 import pytest_asyncio

@@ -32,6 +32,8 @@ class RequestOutput:

32 @dataclass

33 class MockModelConfig:

34 use_async_output_proc = True

35 + media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

36 + mm_placeholder_str_override: dict[str, str] = field(default_factory=dict)

39 class MockEngine:

tests/engine/test_arg_utils.py

+52

@@ -231,6 +231,58 @@ def test_limit_mm_per_prompt_parser(arg, expected):

231 assert args.limit_mm_per_prompt == expected

232

233

234 +@pytest.mark.parametrize(

235 + ("arg", "expected"),

236 + [

237 + (None, dict()),

238 + ('{"video": {"num_frames": 123} }', {

239 + "video": {

240 + "num_frames": 123

241 + }

242 + }),

243 + (

244 + '{"video": {"num_frames": 123, "fps": 1.0, "foo": "bar"}, "image": {"foo": "bar"} }', # noqa

245 + {

246 + "video": {

247 + "num_frames": 123,

248 + "fps": 1.0,

249 + "foo": "bar"

250 + },

251 + "image": {

252 + "foo": "bar"

253 + }

254 + }),

255 + ])

256 +def test_media_io_kwargs_parser(arg, expected):

257 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())

258 + if arg is None:

259 + args = parser.parse_args([])

260 + else:

261 + args = parser.parse_args(["--media-io-kwargs", arg])

262 +

263 + assert args.media_io_kwargs == expected

264 +

265 +

266 +@pytest.mark.parametrize(("arg", "expected"), [

267 + (None, dict()),

268 + ('{"video":"<|video_placeholder|>"}', {

269 + "video": "<|video_placeholder|>"

270 + }),

271 + ('{"video":"<|video_placeholder|>", "image": "<|image_placeholder|>"}', {

272 + "video": "<|video_placeholder|>",

273 + "image": "<|image_placeholder|>"

274 + }),

275 +])

276 +def test_mm_placeholder_str_override_parser(arg, expected):

277 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())

278 + if arg is None:

279 + args = parser.parse_args([])

280 + else:

281 + args = parser.parse_args(["--mm-placeholder-str-override", arg])

282 +

283 + assert args.mm_placeholder_str_override == expected

284 +

285 +

286 def test_compilation_config():

287 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())

288

tests/entrypoints/openai/test_serving_chat.py

+4 -2

@@ -3,8 +3,8 @@

4 import asyncio

5 from contextlib import suppress

6 -from dataclasses import dataclass

7 -from typing import Optional

6 +from dataclasses import dataclass, field

7 +from typing import Any, Optional

8 from unittest.mock import MagicMock

10 from vllm.config import MultiModalConfig

@@ -40,6 +40,8 @@ class MockModelConfig:

40 allowed_local_media_path: str = ""

41 encoder_config = None

42 generation_config: str = "auto"

43 + media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

44 + mm_placeholder_str_override: dict[str, str] = field(default_factory=dict)

46 def get_diff_sampling_param(self):

47 return self.diff_sampling_param or {}

tests/multimodal/test_utils.py

+6 -6

@@ -167,14 +167,14 @@ async def test_fetch_image_error_conversion():

167 @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)

168 @pytest.mark.parametrize("num_frames", [-1, 32, 1800])

169 async def test_fetch_video_http(video_url: str, num_frames: int):

170 - connector = MediaConnector()

170 + connector = MediaConnector(

171 + media_io_kwargs={"video": {

172 + "num_frames": num_frames,

173 + }})

174

172 - video_sync = connector.fetch_video(video_url, num_frames=num_frames)

173 - video_async = await connector.fetch_video_async(video_url,

174 - num_frames=num_frames)

175 - # Check that the video frames are equal and metadata are same

175 + video_sync = connector.fetch_video(video_url)

176 + video_async = await connector.fetch_video_async(video_url)

177 assert np.array_equal(video_sync[0], video_async[0])

177 - assert video_sync[1] == video_async[1]

178

179

180 # Used for the next two tests related to `merge_and_sort_multimodal_metadata`.

tests/multimodal/test_video.py

+47 -1

@@ -4,7 +4,10 @@ import numpy as np

4 import numpy.typing as npt

5 import pytest

7 -from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader

7 +from vllm import envs

8 +from vllm.multimodal.image import ImageMediaIO

9 +from vllm.multimodal.video import (VIDEO_LOADER_REGISTRY, VideoLoader,

10 + VideoMediaIO)

12 NUM_FRAMES = 10

13 FAKE_OUTPUT_1 = np.random.rand(NUM_FRAMES, 1280, 720, 3)

@@ -40,3 +43,46 @@ def test_video_loader_registry():

43 def test_video_loader_type_doesnt_exist():

44 with pytest.raises(AssertionError):

45 VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")

46 +

47 +

48 +@VIDEO_LOADER_REGISTRY.register("assert_10_frames_1_fps")

49 +class Assert10Frames1FPSVideoLoader(VideoLoader):

50 +

51 + @classmethod

52 + def load_bytes(cls,

53 + data: bytes,

54 + num_frames: int = -1,

55 + fps: float = -1.0,

56 + **kwargs) -> npt.NDArray:

57 + assert num_frames == 10, "bad num_frames"

58 + assert fps == 1.0, "bad fps"

59 + return FAKE_OUTPUT_2

60 +

61 +

62 +def test_video_media_io_kwargs():

63 + envs.VLLM_VIDEO_LOADER_BACKEND = "assert_10_frames_1_fps"

64 + imageio = ImageMediaIO()

65 +

66 + # Verify that different args pass/fail assertions as expected.

67 + videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 1.0})

68 + _ = videoio.load_bytes(b"test")

69 +

70 + videoio = VideoMediaIO(

71 + imageio, **{

72 + "num_frames": 10,

73 + "fps": 1.0,

74 + "not_used": "not_used"

75 + })

76 + _ = videoio.load_bytes(b"test")

77 +

78 + with pytest.raises(AssertionError, match="bad num_frames"):

79 + videoio = VideoMediaIO(imageio, **{})

80 + _ = videoio.load_bytes(b"test")

81 +

82 + with pytest.raises(AssertionError, match="bad num_frames"):

83 + videoio = VideoMediaIO(imageio, **{"num_frames": 9, "fps": 1.0})

84 + _ = videoio.load_bytes(b"test")

85 +

86 + with pytest.raises(AssertionError, match="bad fps"):

87 + videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 2.0})

88 + _ = videoio.load_bytes(b"test")

vllm/config.py

+16

@@ -346,6 +346,12 @@ class ModelConfig:

346 limit_mm_per_prompt: dict[str, int] = field(default_factory=dict)

347 """Maximum number of data items per modality per prompt. Only applicable

348 for multimodal models."""

349 + media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

350 + """Additional args passed to process media inputs, keyed by modalities.

351 + For example, to set num_frames for video, set

352 + `--media-io-kwargs '{"video": {"num_frames": 40} }'` """

353 + mm_placeholder_str_override: dict[str, str] = field(default_factory=dict)

354 + """Optionally override placeholder string for given modalities."""

355 use_async_output_proc: bool = True

356 """Whether to use async output processor."""

357 config_format: Union[str, ConfigFormat] = ConfigFormat.AUTO.value

@@ -694,6 +700,8 @@ class ModelConfig:

700 if self.registry.is_multimodal_model(self.architectures):

701 return MultiModalConfig(

702 limit_per_prompt=self.limit_mm_per_prompt,

703 + media_io_kwargs=self.media_io_kwargs,

704 + mm_placeholder_str_override=self.mm_placeholder_str_override,

705 mm_processor_kwargs=self.mm_processor_kwargs,

706 disable_mm_preprocessor_cache=self.

707 disable_mm_preprocessor_cache)

@@ -3063,6 +3071,14 @@ class MultiModalConfig:

3071 `{"images": 16, "videos": 2}`

3072 """

3073

3074 + media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

3075 + """Additional args passed to process media inputs, keyed by modalities.

3076 + For example, to set num_frames for video, set

3077 + `--media-io-kwargs '{"video": {"num_frames": 40} }'` """

3078 +

3079 + mm_placeholder_str_override: dict[str, str] = field(default_factory=dict)

3080 + """Optionally override placeholder string for given modalities."""

3081 +

3082 mm_processor_kwargs: Optional[dict[str, object]] = None

3083 """

3084 Overrides for the multi-modal processor obtained from

vllm/engine/arg_utils.py

+12

@@ -369,6 +369,11 @@ class EngineArgs:

369 get_field(TokenizerPoolConfig, "extra_config")

370 limit_mm_per_prompt: dict[str, int] = \

371 get_field(MultiModalConfig, "limit_per_prompt")

372 + media_io_kwargs: dict[str, dict[str,

373 + Any]] = get_field(MultiModalConfig,

374 + "media_io_kwargs")

375 + mm_placeholder_str_override: dict[str, str] = \

376 + get_field(MultiModalConfig, "mm_placeholder_str_override")

377 mm_processor_kwargs: Optional[Dict[str, Any]] = \

378 MultiModalConfig.mm_processor_kwargs

379 disable_mm_preprocessor_cache: bool = \

@@ -745,6 +750,11 @@ class EngineArgs:

750 )

751 multimodal_group.add_argument("--limit-mm-per-prompt",

752 **multimodal_kwargs["limit_per_prompt"])

753 + multimodal_group.add_argument("--media-io-kwargs",

754 + **multimodal_kwargs["media_io_kwargs"])

755 + multimodal_group.add_argument(

756 + "--mm-placeholder-str-override",

757 + **multimodal_kwargs["mm_placeholder_str_override"])

758 multimodal_group.add_argument(

759 "--mm-processor-kwargs",

760 **multimodal_kwargs["mm_processor_kwargs"])

@@ -969,6 +979,8 @@ class EngineArgs:

979 enable_prompt_embeds=self.enable_prompt_embeds,

980 served_model_name=self.served_model_name,

981 limit_mm_per_prompt=self.limit_mm_per_prompt,

982 + media_io_kwargs=self.media_io_kwargs,

983 + mm_placeholder_str_override=self.mm_placeholder_str_override,

984 use_async_output_proc=not self.disable_async_output_proc,

985 config_format=self.config_format,

986 mm_processor_kwargs=self.mm_processor_kwargs,

vllm/entrypoints/chat_utils.py

+8 -3

@@ -507,6 +507,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

507

508 def _placeholder_str(self, modality: ModalityStr,

509 current_count: int) -> Optional[str]:

510 + if modality in self._model_config.mm_placeholder_str_override:

511 + return self._model_config.mm_placeholder_str_override[modality]

512 +

513 # TODO: Let user specify how to insert image tokens into prompt

514 # (similar to chat template)

515 hf_config = self._model_config.hf_config

@@ -725,6 +728,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):

728 self._tracker = tracker

729

730 self._connector = MediaConnector(

731 + media_io_kwargs=self._tracker._model_config.media_io_kwargs,

732 allowed_local_media_path=tracker.allowed_local_media_path,

733 )

734

@@ -763,7 +767,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):

767 return self.parse_audio(audio_url)

768

769 def parse_video(self, video_url: str) -> None:

766 - video = self._connector.fetch_video(video_url)

770 + video = self._connector.fetch_video(video_url=video_url)

771

772 placeholder = self._tracker.add("video", video)

773 self._add_placeholder(placeholder)

@@ -776,7 +780,8 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

780

781 self._tracker = tracker

782 self._connector = MediaConnector(

779 - allowed_local_media_path=tracker.allowed_local_media_path,

783 + media_io_kwargs=self._tracker._model_config.media_io_kwargs,

784 + allowed_local_media_path=tracker.allowed_local_media_path

785 )

786

787 def parse_image(self, image_url: str) -> None:

@@ -818,7 +823,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):

823 return self.parse_audio(audio_url)

824

825 def parse_video(self, video_url: str) -> None:

821 - video = self._connector.fetch_video_async(video_url)

826 + video = self._connector.fetch_video_async(video_url=video_url)

827

828 placeholder = self._tracker.add("video", video)

829 self._add_placeholder(placeholder)

vllm/multimodal/audio.py

+10

@@ -83,6 +83,16 @@ class AudioResampler:

84 class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):

86 + def __init__(self, **kwargs) -> None:

87 + super().__init__()

88 +

89 + # `kwargs` contains custom arguments from

90 + # --media-io-kwargs for this modality.

91 + # They can be passed to the underlying

92 + # media loaders (e.g. custom implementations)

93 + # for flexible control.

94 + self.kwargs = kwargs

95 +

96 def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:

97 return librosa.load(BytesIO(data), sr=None)

vllm/multimodal/image.py

+7 -1

@@ -44,10 +44,16 @@ def convert_image_mode(image: Image.Image, to_mode: str):

45 class ImageMediaIO(MediaIO[Image.Image]):

47 - def __init__(self, *, image_mode: str = "RGB") -> None:

47 + def __init__(self, image_mode: str = "RGB", **kwargs) -> None:

48 super().__init__()

50 self.image_mode = image_mode

51 + # `kwargs` contains custom arguments from

52 + # --media-io-kwargs for this modality.

53 + # They can be passed to the underlying

54 + # media loaders (e.g. custom implementations)

55 + # for flexible control.

56 + self.kwargs = kwargs

58 def load_bytes(self, data: bytes) -> Image.Image:

59 image = Image.open(BytesIO(data))

vllm/multimodal/utils.py

+17 -10

@@ -38,12 +38,15 @@ class MediaConnector:

39 def __init__(

40 self,

41 + media_io_kwargs: Optional[dict[str, dict[str, Any]]] = None,

42 connection: HTTPConnection = global_http_connection,

43 *,

44 allowed_local_media_path: str = "",

45 ) -> None:

46 super().__init__()

48 + self.media_io_kwargs: dict[str, dict[

49 + str, Any]] = media_io_kwargs if media_io_kwargs else {}

50 self.connection = connection

52 if allowed_local_media_path:

@@ -149,7 +152,7 @@ class MediaConnector:

152 """

153 Load audio from a URL.

154 """

152 - audio_io = AudioMediaIO()

155 + audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))

156

157 return self.load_from_url(

158 audio_url,

@@ -164,7 +167,7 @@ class MediaConnector:

167 """

168 Asynchronously fetch audio from a URL.

169 """

167 - audio_io = AudioMediaIO()

170 + audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))

171

172 return await self.load_from_url_async(

173 audio_url,

@@ -183,7 +186,8 @@ class MediaConnector:

186

187 By default, the image is converted into RGB format.

188 """

186 - image_io = ImageMediaIO(image_mode=image_mode)

189 + image_io = ImageMediaIO(image_mode=image_mode,

190 + **self.media_io_kwargs.get("image", {}))

191

192 try:

193 return self.load_from_url(

@@ -206,7 +210,8 @@ class MediaConnector:

210

211 By default, the image is converted into RGB format.

212 """

209 - image_io = ImageMediaIO(image_mode=image_mode)

213 + image_io = ImageMediaIO(image_mode=image_mode,

214 + **self.media_io_kwargs.get("image", {}))

215

216 try:

217 return await self.load_from_url_async(

@@ -223,13 +228,14 @@ class MediaConnector:

228 video_url: str,

229 *,

230 image_mode: str = "RGB",

226 - num_frames: int = 32,

231 ) -> npt.NDArray:

232 """

233 Load video from a HTTP or base64 data URL.

234 """

231 - image_io = ImageMediaIO(image_mode=image_mode)

232 - video_io = VideoMediaIO(image_io, num_frames=num_frames)

235 + image_io = ImageMediaIO(image_mode=image_mode,

236 + **self.media_io_kwargs.get("image", {}))

237 + video_io = VideoMediaIO(image_io,

238 + **self.media_io_kwargs.get("video", {}))

239

240 return self.load_from_url(

241 video_url,

@@ -242,15 +248,16 @@ class MediaConnector:

248 video_url: str,

249 *,

250 image_mode: str = "RGB",

245 - num_frames: int = 32,

251 ) -> npt.NDArray:

252 """

253 Asynchronously load video from a HTTP or base64 data URL.

254

255 By default, the image is converted into RGB format.

256 """

252 - image_io = ImageMediaIO(image_mode=image_mode)

253 - video_io = VideoMediaIO(image_io, num_frames=num_frames)

257 + image_io = ImageMediaIO(image_mode=image_mode,

258 + **self.media_io_kwargs.get("image", {}))

259 + video_io = VideoMediaIO(image_io,

260 + **self.media_io_kwargs.get("video", {}))

261

262 return await self.load_from_url_async(

263 video_url,

vllm/multimodal/video.py

+16 -4

@@ -54,7 +54,10 @@ class VideoLoader:

55 @classmethod

56 @abstractmethod

57 - def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:

57 + def load_bytes(cls,

58 + data: bytes,

59 + num_frames: int = -1,

60 + **kwargs) -> npt.NDArray:

61 raise NotImplementedError

@@ -102,7 +105,8 @@ class OpenCVVideoBackend(VideoLoader):

105 @classmethod

106 def load_bytes(cls,

107 data: bytes,

105 - num_frames: int = -1) -> tuple[npt.NDArray, dict]:

108 + num_frames: int = -1,

109 + **kwargs) -> npt.NDArray:

110 import cv2

111

112 backend = cls().get_cv2_video_api()

@@ -159,18 +163,26 @@ class VideoMediaIO(MediaIO[npt.NDArray]):

163 def __init__(

164 self,

165 image_io: ImageMediaIO,

162 - *,

166 num_frames: int = 32,

167 + **kwargs,

168 ) -> None:

169 super().__init__()

170

171 self.image_io = image_io

172 self.num_frames = num_frames

173 + # `kwargs` contains custom arguments from

174 + # --media-io-kwargs for this modality.

175 + # They can be passed to the underlying

176 + # media loaders (e.g. custom implementations)

177 + # for flexible control.

178 + self.kwargs = kwargs

179 video_loader_backend = envs.VLLM_VIDEO_LOADER_BACKEND

180 self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)

181

182 def load_bytes(self, data: bytes) -> npt.NDArray:

173 - return self.video_loader.load_bytes(data, self.num_frames)

183 + return self.video_loader.load_bytes(data,

184 + num_frames=self.num_frames,

185 + **self.kwargs)

186

187 def load_base64(self, media_type: str, data: str) -> npt.NDArray:

188 if media_type.lower() == "video/jpeg":