@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 803d5c35f3

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 17:15 UTC

[V1] Override `mm_counts` for dummy data creation (#15703)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 30, 2025 at 18:20 UTC 803d5c35f3e8a6547ff7c6e6c322e54cbfec8444

9 files changed +114 -93

tests/models/decoder_only/vision_language/test_models.py

+2 -26

@@ -385,18 +385,7 @@ VLM_TEST_SETTINGS = {

385 ),

386 "minicpmo_26": VLMTestInfo(

387 models=["openbmb/MiniCPM-o-2_6"],

388 - test_type=(VLMTestType.IMAGE),

389 - prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501

390 - img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",

391 - max_model_len=4096,

392 - max_num_seqs=2,

393 - get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501

394 - hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,

395 - patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,

396 - ),

397 - "minicpmo_26_multi_image": VLMTestInfo(

398 - models=["openbmb/MiniCPM-o-2_6"],

399 - test_type=(VLMTestType.MULTI_IMAGE),

388 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),

389 prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501

390 img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",

391 max_model_len=4096,

@@ -404,22 +393,10 @@ VLM_TEST_SETTINGS = {

393 get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501

394 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,

395 patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,

407 - marks=[large_gpu_mark(min_gb=32)],

396 ),

397 "minicpmv_26": VLMTestInfo(

398 models=["openbmb/MiniCPM-V-2_6"],

411 - test_type=(VLMTestType.IMAGE),

412 - prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501

413 - img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",

414 - max_model_len=4096,

415 - max_num_seqs=2,

416 - get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501

417 - hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,

418 - patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,

419 - ),

420 - "minicpmv_26_multi_image": VLMTestInfo(

421 - models=["openbmb/MiniCPM-V-2_6"],

422 - test_type=(VLMTestType.MULTI_IMAGE),

399 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),

400 prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501

401 img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",

402 max_model_len=4096,

@@ -427,7 +404,6 @@ VLM_TEST_SETTINGS = {

404 get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501

405 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,

406 patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,

430 - marks=[large_gpu_mark(min_gb=32)],

407 ),

408 "molmo": VLMTestInfo(

409 models=["allenai/Molmo-7B-D-0924"],

vllm/model_executor/models/llava_next_video.py

+9 -5

@@ -71,7 +71,8 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):

71 max_video_tokens = self.get_num_video_tokens(

72 image_width=target_width,

73 image_height=target_height,

74 - num_frames=self.get_num_frames_with_most_features(seq_len),

74 + num_frames=self.get_num_frames_with_most_features(

75 + seq_len, mm_counts),

76 )

78 return {"video": max_video_tokens}

@@ -130,9 +131,12 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):

131

132 return num_frames

133

133 - def get_num_frames_with_most_features(self, seq_len: int) -> int:

134 - mm_config = self.ctx.get_mm_config()

135 - max_videos = mm_config.get_limit_per_prompt("video")

134 + def get_num_frames_with_most_features(

135 + self,

136 + seq_len: int,

137 + mm_counts: Mapping[str, int],

138 + ) -> int:

139 + max_videos = mm_counts.get("video", 0)

140

141 max_total_frames = self._get_max_video_frames(seq_len)

142

@@ -155,7 +159,7 @@ class LlavaNextVideoDummyInputsBuilder(

159 target_width, target_height = \

160 self.info.get_image_size_with_most_features()

161 target_num_frames = \

158 - self.info.get_num_frames_with_most_features(seq_len)

162 + self.info.get_num_frames_with_most_features(seq_len, mm_counts)

163

164 mm_data = {

165 "video":

vllm/model_executor/models/llava_onevision.py

+17 -8

@@ -108,7 +108,7 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):

108 ) -> Mapping[str, int]:

109 return {

110 "image": self.get_max_image_tokens(),

111 - "video": self.get_max_video_tokens(seq_len),

111 + "video": self.get_max_video_tokens(seq_len, mm_counts),

112 }

113

114 # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86

@@ -202,10 +202,13 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):

202

203 return num_frames

204

205 - def get_num_frames_with_most_features(self, seq_len: int) -> int:

206 - mm_config = self.ctx.get_mm_config()

207 - max_images = mm_config.get_limit_per_prompt("image")

208 - max_videos = mm_config.get_limit_per_prompt("video")

205 + def get_num_frames_with_most_features(

206 + self,

207 + seq_len: int,

208 + mm_counts: Mapping[str, int],

209 + ) -> int:

210 + max_images = mm_counts.get("image", 0)

211 + max_videos = mm_counts.get("video", 0)

212

213 max_image_tokens = self.get_max_image_tokens() * max_images

214 max_total_frames = self._get_max_video_frames(seq_len -

@@ -215,13 +218,18 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):

218

219 return max(max_frames_per_video, 1)

220

218 - def get_max_video_tokens(self, seq_len: int) -> int:

221 + def get_max_video_tokens(

222 + self,

223 + seq_len: int,

224 + mm_counts: Mapping[str, int],

225 + ) -> int:

226 target_width, target_height = self.get_image_size_with_most_features()

227

228 return self.get_num_video_tokens(

229 image_width=target_width,

230 image_height=target_height,

224 - num_frames=self.get_num_frames_with_most_features(seq_len),

231 + num_frames=self.get_num_frames_with_most_features(

232 + seq_len, mm_counts),

233 )

234

235

@@ -243,7 +251,8 @@ class LlavaOnevisionDummyInputsBuilder(

251 target_width, target_height = \

252 self.info.get_image_size_with_most_features()

253 target_num_frames = \

246 - self.info.get_num_frames_with_most_features(seq_len)

254 + self.info.get_num_frames_with_most_features(seq_len,

255 + mm_counts)

256

257 mm_data = {

258 "image":

vllm/model_executor/models/minicpmo.py

+15 -11

@@ -43,7 +43,8 @@ from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,

43 from vllm.multimodal.processing import PromptReplacement, PromptUpdate

44 from vllm.multimodal.profiling import ProcessorInputs

46 -from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder,

46 +from .minicpmv import (_MAX_FRAMES_PER_VIDEO, MiniCPMV2_6,

47 + MiniCPMVDummyInputsBuilder,

48 MiniCPMVMultiModalDataParser,

49 MiniCPMVMultiModalProcessor, MiniCPMVProcessingInfo,

50 _minicpmv_field_config)

@@ -203,8 +204,8 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):

204 return 30

205

206 def get_max_audio_tokens(self) -> int:

206 - return self.get_max_audio_tokens_per_chunk(

207 - ) * self.get_max_audio_chunks_with_most_features()

207 + num_chunks = self.get_max_audio_chunks_with_most_features()

208 + return self.get_max_audio_tokens_per_chunk() * num_chunks

209

210 def get_audio_len_by_num_chunks(self, num_chunks: int) -> int:

211 sampling_rate = self.get_default_audio_sampling_rate()

@@ -212,21 +213,24 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):

213 num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk() - 2

214 return int(num_chunks * sampling_rate / num_tokens_per_chunk) + 1

215

215 - def get_num_frames_with_most_features(self, seq_len: int) -> int:

216 - mm_config = self.ctx.get_mm_config()

217 - max_images = mm_config.get_limit_per_prompt("image")

218 - max_videos = mm_config.get_limit_per_prompt("video")

219 - max_audios = mm_config.get_limit_per_prompt("audio")

216 + def get_num_frames_with_most_features(

217 + self,

218 + seq_len: int,

219 + mm_counts: Mapping[str, int],

220 + ) -> int:

221 + max_images = mm_counts.get("image", 0)

222 + max_videos = mm_counts.get("video", 0)

223 + max_audios = mm_counts.get("audio", 0)

224

225 max_image_tokens = self.get_max_image_tokens() * max_images

226 max_audio_tokens = self.get_max_audio_tokens() * max_audios

227 max_total_frames = self.get_max_video_frames(seq_len -

228 max_image_tokens -

229 max_audio_tokens)

230 + max_frames_per_video = min(max_total_frames // max(max_videos, 1),

231 + _MAX_FRAMES_PER_VIDEO)

232

227 - num_frames = max(max_total_frames // max(max_videos, 1), 1)

228 -

229 - return num_frames

233 + return max(max_frames_per_video, 1)

234

235

236 class MiniCPMODummyInputsBuilder(

vllm/model_executor/models/minicpmv.py

+24 -12

@@ -69,6 +69,9 @@ from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,

69 merge_multimodal_embeddings)

70 from .vision import scatter_patch_features, select_patch_features

72 +# For profile run

73 +_MAX_FRAMES_PER_VIDEO = 16

74 +

76 class MiniCPMVImagePixelInputs(TypedDict):

77 type: Literal["pixel_values"]

@@ -369,7 +372,8 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):

372 ) -> Mapping[str, int]:

373 mm_max_tokens = {"image": self.get_max_image_tokens()}

374 if self.get_model_version() == (2, 6):

372 - mm_max_tokens["video"] = self.get_max_video_tokens(seq_len)

375 + mm_max_tokens["video"] = self.get_max_video_tokens(

376 + seq_len, mm_counts)

377

378 return mm_max_tokens

379

@@ -432,9 +436,14 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):

436 use_image_id=False,

437 )

438

435 - def get_max_video_tokens(self, seq_len: int) -> int:

436 - return self.get_max_video_frame_tokens(

437 - ) * self.get_num_frames_with_most_features(seq_len)

439 + def get_max_video_tokens(

440 + self,

441 + seq_len: int,

442 + mm_counts: Mapping[str, int],

443 + ) -> int:

444 + num_frames = self.get_num_frames_with_most_features(seq_len, mm_counts)

445 + num_video_tokens_total = self.get_max_video_frame_tokens() * num_frames

446 + return num_video_tokens_total

447

448 def get_video_max_slice_num(self) -> int:

449 return 1

@@ -449,18 +458,21 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):

458 num_frames = max_tokens // num_frame_tokens

459 return num_frames

460

452 - def get_num_frames_with_most_features(self, seq_len: int) -> int:

453 - mm_config = self.ctx.get_mm_config()

454 - max_images = mm_config.get_limit_per_prompt("image")

455 - max_videos = mm_config.get_limit_per_prompt("video")

461 + def get_num_frames_with_most_features(

462 + self,

463 + seq_len: int,

464 + mm_counts: Mapping[str, int],

465 + ) -> int:

466 + max_images = mm_counts.get("image", 0)

467 + max_videos = mm_counts.get("video", 0)

468

469 max_image_tokens = self.get_max_image_tokens() * max_images

470 max_total_frames = self.get_max_video_frames(seq_len -

471 max_image_tokens)

472 + max_frames_per_video = min(max_total_frames // max(max_videos, 1),

473 + _MAX_FRAMES_PER_VIDEO)

474

461 - num_frames = max(max_total_frames // max(max_videos, 1), 1)

462 -

463 - return num_frames

475 + return max(max_frames_per_video, 1)

476

477

478 _I = TypeVar("_I",

@@ -483,7 +495,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

495 video_width, video_height = \

496 self.info.get_video_frame_size_with_most_features()

497 num_video_frames = \

486 - self.info.get_num_frames_with_most_features(seq_len)

498 + self.info.get_num_frames_with_most_features(seq_len, mm_counts)

499

500 mm_data = {

501 "image":

vllm/model_executor/models/qwen2_vl.py

+17 -9

@@ -806,7 +806,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):

806 max_pixels: Optional[int] = None,

807 size: Optional[dict[str, int]] = None,

808 **kwargs: object,

809 - ):

809 + ) -> Qwen2VLImageProcessor:

810 return cached_image_processor_from_config(

811 self.ctx.model_config,

812 **self._get_image_processor_kwargs(min_pixels=min_pixels,

@@ -825,7 +825,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):

825 ) -> Mapping[str, int]:

826 return {

827 "image": self.get_max_image_tokens(),

828 - "video": self.get_max_video_tokens(seq_len),

828 + "video": self.get_max_video_tokens(seq_len, mm_counts),

829 }

830

831 def _get_vision_info(

@@ -941,10 +941,13 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):

941

942 return num_frames

943

944 - def get_num_frames_with_most_features(self, seq_len: int) -> int:

945 - mm_config = self.ctx.get_mm_config()

946 - max_images = mm_config.get_limit_per_prompt("image")

947 - max_videos = mm_config.get_limit_per_prompt("video")

944 + def get_num_frames_with_most_features(

945 + self,

946 + seq_len: int,

947 + mm_counts: Mapping[str, int],

948 + ) -> int:

949 + max_images = mm_counts.get("image", 0)

950 + max_videos = mm_counts.get("video", 0)

951

952 max_image_tokens = self.get_max_image_tokens() * max_images

953 max_total_frames = self._get_max_video_frames(seq_len -

@@ -954,13 +957,18 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):

957

958 return max(max_frames_per_video, 1)

959

957 - def get_max_video_tokens(self, seq_len: int) -> int:

960 + def get_max_video_tokens(

961 + self,

962 + seq_len: int,

963 + mm_counts: Mapping[str, int],

964 + ) -> int:

965 target_width, target_height = self.get_image_size_with_most_features()

966

967 return self.get_num_video_tokens(

968 image_width=target_width,

969 image_height=target_height,

963 - num_frames=self.get_num_frames_with_most_features(seq_len),

970 + num_frames=self.get_num_frames_with_most_features(

971 + seq_len, mm_counts),

972 image_processor=None,

973 )

974

@@ -982,7 +990,7 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):

990 target_width, target_height = \

991 self.info.get_image_size_with_most_features()

992 target_num_frames = \

985 - self.info.get_num_frames_with_most_features(seq_len)

993 + self.info.get_num_frames_with_most_features(seq_len, mm_counts)

994

995 mm_data = {

996 "image":

vllm/multimodal/profiling.py

+21 -9

@@ -3,7 +3,7 @@

3 from abc import ABC, abstractmethod

4 from collections.abc import Mapping

5 from dataclasses import dataclass, field

6 -from typing import Generic, NamedTuple, TypeVar, cast

6 +from typing import Generic, NamedTuple, Optional, TypeVar, cast

8 import numpy as np

9 import numpy.typing as npt

@@ -160,17 +160,19 @@ class MultiModalProfiler(Generic[_I]):

160 def get_and_validate_mm_inputs(

161 self,

162 seq_len: int,

163 + mm_counts: Optional[Mapping[str, int]] = None,

164 ) -> tuple[MultiModalInputs, Mapping[str, int]]:

164 - mm_counts = self.get_mm_limits()

165 + if mm_counts is None:

166 + mm_counts = self.get_mm_limits()

167

168 info = self.processing_info

169 mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(

170 seq_len, mm_counts)

171

170 - if mm_counts.keys() != mm_max_tokens_per_item.keys():

172 + if mm_counts.keys() - mm_max_tokens_per_item.keys():

173 raise AssertionError(

174 "The keys returned by `get_supported_mm_limits` "

173 - f"({set(mm_counts.keys())}) should be the same as those "

175 + f"({set(mm_counts.keys())}) should be a subset of those "

176 "returned by `get_mm_max_tokens_per_item` "

177 f"({set(mm_max_tokens_per_item.keys())})")

178

@@ -193,8 +195,12 @@ class MultiModalProfiler(Generic[_I]):

195 "tokens.")

196 return mm_inputs, total_placeholders_by_modality

197

196 - def get_encoder_dummy_data(self, seq_len: int) -> DummyEncoderData:

197 - mm_inputs, _ = self.get_and_validate_mm_inputs(seq_len)

198 + def get_encoder_dummy_data(

199 + self,

200 + seq_len: int,

201 + mm_counts: Optional[Mapping[str, int]] = None,

202 + ) -> DummyEncoderData:

203 + mm_inputs, _ = self.get_and_validate_mm_inputs(seq_len, mm_counts)

204 mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)

205

206 # For encoder-decoder models, use encoder prompt token ids instead of

@@ -207,9 +213,15 @@ class MultiModalProfiler(Generic[_I]):

213

214 return DummyEncoderData(encoder_prompt_token_ids)

215

210 - def get_decoder_dummy_data(self, seq_len: int) -> DummyDecoderData:

211 - (mm_inputs, total_placeholders_by_modality

212 - ) = self.get_and_validate_mm_inputs(seq_len)

216 + def get_decoder_dummy_data(

217 + self,

218 + seq_len: int,

219 + mm_counts: Optional[Mapping[str, int]] = None,

220 + ) -> DummyDecoderData:

221 + (

222 + mm_inputs,

223 + total_placeholders_by_modality,

224 + ) = self.get_and_validate_mm_inputs(seq_len, mm_counts)

225

226 prompt_token_ids = mm_inputs["prompt_token_ids"]

227 total_len = len(prompt_token_ids)

vllm/multimodal/registry.py

+4 -2

@@ -458,6 +458,7 @@ class MultiModalRegistry:

458 self,

459 model_config: "ModelConfig",

460 seq_len: int,

461 + mm_counts: Optional[Mapping[str, int]] = None,

462 ) -> DummyDecoderData:

463 """

464 Create dummy data for profiling the memory usage of a model.

@@ -466,7 +467,7 @@ class MultiModalRegistry:

467 """

468 processor = self.create_processor(model_config, disable_cache=True)

469 profiler = MultiModalProfiler(processor)

469 - dummy_data = profiler.get_decoder_dummy_data(seq_len)

470 + dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts)

471

472 # Having more tokens is over-conservative but otherwise fine

473 token_ids = dummy_data.prompt_token_ids

@@ -481,6 +482,7 @@ class MultiModalRegistry:

482 self,

483 model_config: "ModelConfig",

484 seq_len: int,

485 + mm_counts: Optional[Mapping[str, int]] = None,

486 ) -> DummyEncoderData:

487 """

488 Create dummy data for profiling the memory usage of a model.

@@ -489,7 +491,7 @@ class MultiModalRegistry:

491 """

492 processor = self.create_processor(model_config, disable_cache=True)

493 profiler = MultiModalProfiler(processor)

492 - dummy_data = profiler.get_encoder_dummy_data(seq_len)

494 + dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts)

495

496 # Having more tokens is over-conservative but otherwise fine

497 token_ids = dummy_data.prompt_token_ids

vllm/v1/worker/gpu_model_runner.py

+5 -11

@@ -1470,19 +1470,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):

1470 encoder_budget, max_num_mm_items, dummy_data_modality)

1471

1472 # Create dummy batch of multimodal inputs.

1473 - dummy_request_data = self.mm_registry.get_decoder_dummy_data(

1473 + dummy_mm_kwargs = self.mm_registry.get_decoder_dummy_data(

1474 model_config=self.model_config,

1475 seq_len=self.max_num_tokens,

1476 - )

1477 - dummy_mm_data = dummy_request_data.multi_modal_data

1478 -

1479 - # Dummy data definition may contain multiple multimodal items

1480 - # (e.g, multiple images) for a single request, therefore here we

1481 - # always replicate first item by max_num_mm_items times since in V1

1482 - # they are scheduled to be processed separately.

1483 - dummy_mm_item = dummy_mm_data.get_item(

1484 - modality=dummy_data_modality, item_index=0)

1485 - dummy_mm_kwargs = MultiModalKwargs.from_items([dummy_mm_item])

1476 + mm_counts={

1477 + dummy_data_modality: 1

1478 + },

1479 + ).multi_modal_data

1480

1481 batched_dummy_mm_inputs = MultiModalKwargs.batch(

1482 [dummy_mm_kwargs] * max_num_mm_items)