@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9515c20868
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 10:45 UTC

[Misc] Clean up processing logic (#37541)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 19, 2026 at 21:30 UTC 9515c208684c7d289b7c482e3ee2d201d9a5c497
9 files changed +456 -658
vllm/model_executor/models/ernie45_vl.py
+20 -36
@@ -1221,49 +1221,33 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
1221 num_videos: int,
1222 overrides: VideoDummyOptions | None = None,
1223 ):
1224 - if overrides:
1225 - if overrides.num_frames:
1226 - if overrides.num_frames > num_frames:
1227 - logger.warning(
1228 - "video.num_frames override (%d) exceeds model's "
1229 - "maximum number of frames (%d), will be ignored",
1230 - overrides.num_frames,
1231 - num_frames,
1232 - )
1233 - num_frames = min(num_frames, overrides.num_frames)
1234 - if overrides.width:
1235 - if overrides.width > width:
1236 - logger.warning(
1237 - "video.width override (%d) exceeds model's "
1238 - "maximum width (%d), will be ignored",
1239 - overrides.width,
1240 - width,
1241 - )
1242 - width = min(width, overrides.width)
1243 - if overrides.height:
1244 - if overrides.height > height:
1245 - logger.warning(
1246 - "video.height override (%d) exceeds model's "
1247 - "maximum height (%d), will be ignored",
1248 - overrides.height,
1249 - height,
1250 - )
1251 - height = min(height, overrides.height)
1252 - num_frames = max(num_frames, 2) # ernie4.5-vl requires at least 2 frames
1224 + # ernie4.5-vl requires at least 2 frames
1225 + num_frames = max(num_frames, 2)
1226 + if overrides and overrides.num_frames:
1227 + overrides.num_frames = max(overrides.num_frames, 2)
1228 +
1229 + videos = super()._get_dummy_videos(
1230 + width=width,
1231 + height=height,
1232 + num_frames=num_frames,
1233 + num_videos=num_videos,
1234 + overrides=overrides,
1235 + )
1236 + videos = [v.copy() for v in videos]
1237
1254 - video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)
1238 video_items = []
1256 - for i in range(num_videos):
1239 + for video in videos:
1240 + video_num_frames = video.shape[0]
1241 video_metadata = {
1242 "fps": 2.0,
1259 - "duration": num_frames / 2.0,
1260 - "total_num_frames": num_frames,
1261 - "frames_indices": [i for i in range(num_frames)],
1243 + "duration": video_num_frames / 2.0,
1244 + "total_num_frames": video_num_frames,
1245 + "frames_indices": list(range(video_num_frames)),
1246 "video_backend": "opencv",
1247 "do_sample_frames": False,
1248 }
1265 - video_item = (video.copy(), video_metadata)
1266 - video_items.append(video_item)
1249 + video_items.append((video, video_metadata))
1250 +
1251 return video_items
1252
1253
vllm/model_executor/models/glm4_1v.py
+19 -36
@@ -1206,49 +1206,32 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1206 num_videos: int,
1207 overrides: VideoDummyOptions | None = None,
1208 ) -> list[VideoItem]:
1209 - if overrides:
1210 - if overrides.num_frames:
1211 - if overrides.num_frames > num_frames:
1212 - logger.warning(
1213 - "video.num_frames override (%d) exceeds model's "
1214 - "maximum number of frames (%d), will be ignored",
1215 - overrides.num_frames,
1216 - num_frames,
1217 - )
1218 - num_frames = min(num_frames, overrides.num_frames)
1219 - if overrides.width:
1220 - if overrides.width > width:
1221 - logger.warning(
1222 - "video.width override (%d) exceeds model's "
1223 - "maximum width (%d), will be ignored",
1224 - overrides.width,
1225 - width,
1226 - )
1227 - width = min(width, overrides.width)
1228 - if overrides.height:
1229 - if overrides.height > height:
1230 - logger.warning(
1231 - "video.height override (%d) exceeds model's "
1232 - "maximum height (%d), will be ignored",
1233 - overrides.height,
1234 - height,
1235 - )
1236 - height = min(height, overrides.height)
1209 + # GLM 4.6V requires at least 2 frames
1210 + num_frames = max(num_frames, 2)
1211 + if overrides and overrides.num_frames:
1212 + overrides.num_frames = max(overrides.num_frames, 2)
1213 +
1214 + videos = super()._get_dummy_videos(
1215 + width=width,
1216 + height=height,
1217 + num_frames=num_frames,
1218 + num_videos=num_videos,
1219 + overrides=overrides,
1220 + )
1221 + videos = [v.copy() for v in videos]
1222
1238 - num_frames = max(num_frames, 2) # GLM 4.6V requires 2 frames
1239 - video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)
1223 video_items = []
1241 - for i in range(num_videos):
1224 + for video in videos:
1225 + video_num_frames = video.shape[0]
1226 video_metadata = {
1227 "fps": 2.0,
1244 - "duration": num_frames / 2.0,
1245 - "total_num_frames": num_frames,
1246 - "frames_indices": [i for i in range(num_frames)],
1228 + "duration": video_num_frames / 2.0,
1229 + "total_num_frames": video_num_frames,
1230 + "frames_indices": list(range(video_num_frames)),
1231 "video_backend": "opencv",
1232 "do_sample_frames": False,
1233 }
1250 - video_item = (video.copy(), video_metadata)
1251 - video_items.append(video_item)
1234 + video_items.append((video, video_metadata))
1235
1236 return video_items
1237
vllm/model_executor/models/h2ovl.py
+10 -17
@@ -8,14 +8,13 @@
8 # Copyright (c) 2024 H2O.AI
9 # Licensed under Apache 2.0 License [see LICENSE for details]
10 # --------------------------------------------------------
11 -from collections.abc import Mapping, Sequence
11
12 import torch
13 from transformers import PretrainedConfig
14
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.multimodal import MULTIMODAL_REGISTRY
18 -from vllm.multimodal.inputs import MultiModalKwargsItems
17 +from vllm.multimodal.inputs import BatchedTensorInputs
18 from vllm.multimodal.parse import (
19 ImageEmbeddingItems,
20 ImageProcessorItems,
@@ -25,7 +24,6 @@ from vllm.multimodal.processing.processor import (
24 MultiModalProcessingInfo,
25 ProcessorInputs,
26 PromptReplacement,
28 - PromptUpdate,
27 TimingContext,
28 )
29 from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor
@@ -86,15 +84,12 @@ class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
84
85
86 class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingInfo]):
89 - def _get_prompt_updates(
87 + def _get_prompt_repl_image(
88 self,
89 mm_items: MultiModalDataItems,
92 - hf_processor_mm_kwargs: Mapping[str, object],
93 - out_mm_kwargs: MultiModalKwargsItems,
94 - ) -> Sequence[PromptUpdate]:
95 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
96 -
97 - out_mm_data = out_mm_kwargs.get_data()
90 + hf_processor: H2OVLProcessor,
91 + out_mm_data: BatchedTensorInputs,
92 + ):
93 if "image_num_patches" in out_mm_data:
94 image_num_patches = out_mm_data["image_num_patches"]
95 assert isinstance(image_num_patches, torch.Tensor)
@@ -130,13 +125,11 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
125
126 return hf_processor.get_image_repl(num_patches, num_features=feature_size)
127
133 - return [
134 - PromptReplacement(
135 - modality="image",
136 - target="<image>",
137 - replacement=get_replacement_internvl,
138 - )
139 - ]
128 + return PromptReplacement(
129 + modality="image",
130 + target="<image>",
131 + replacement=get_replacement_internvl,
132 + )
133
134 def _cached_apply_hf_processor(
135 self,
vllm/model_executor/models/internvl.py
+70 -51
@@ -27,6 +27,7 @@ from vllm.model_executor.models.intern_vit import (
27 from vllm.model_executor.models.module_mapping import MultiModelKeys
28 from vllm.multimodal import MULTIMODAL_REGISTRY
29 from vllm.multimodal.inputs import (
30 + BatchedTensorInputs,
31 MultiModalDataDict,
32 MultiModalFieldConfig,
33 MultiModalKwargsItems,
@@ -238,11 +239,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
239
240 return processed_outputs
241
241 - def _get_mm_fields_config(
242 - self,
243 - hf_inputs: BatchFeature,
244 - hf_processor_mm_kwargs: Mapping[str, object],
245 - ) -> Mapping[str, MultiModalFieldConfig]:
242 + def _get_image_fields_config(self, hf_inputs: BatchFeature):
243 image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
244 num_images = len(image_num_patches)
245
@@ -255,15 +252,19 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
252 image_token_id=MultiModalFieldConfig.shared("image", num_images),
253 )
254
258 - def _get_prompt_updates(
255 + def _get_mm_fields_config(
256 self,
260 - mm_items: MultiModalDataItems,
257 + hf_inputs: BatchFeature,
258 hf_processor_mm_kwargs: Mapping[str, object],
262 - out_mm_kwargs: MultiModalKwargsItems,
263 - ) -> Sequence[PromptUpdate]:
264 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
259 + ) -> Mapping[str, MultiModalFieldConfig]:
260 + return self._get_image_fields_config(hf_inputs)
261
266 - out_mm_data = out_mm_kwargs.get_data()
262 + def _get_prompt_repl_image(
263 + self,
264 + mm_items: MultiModalDataItems,
265 + hf_processor: InternVLProcessor,
266 + out_mm_data: BatchedTensorInputs,
267 + ):
268 if "image_num_patches" in out_mm_data:
269 image_num_patches = out_mm_data["image_num_patches"]
270 assert isinstance(image_num_patches, torch.Tensor)
@@ -296,12 +297,23 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
297
298 return hf_processor.get_image_repl(num_patches, num_features=feature_size)
299
300 + return PromptReplacement(
301 + modality="image",
302 + target="<image>",
303 + replacement=get_replacement_internvl,
304 + )
305 +
306 + def _get_prompt_updates(
307 + self,
308 + mm_items: MultiModalDataItems,
309 + hf_processor_mm_kwargs: Mapping[str, object],
310 + out_mm_kwargs: MultiModalKwargsItems,
311 + ) -> Sequence[PromptUpdate]:
312 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
313 + out_mm_data = out_mm_kwargs.get_data()
314 +
315 return [
300 - PromptReplacement(
301 - modality="image",
302 - target="<image>",
303 - replacement=get_replacement_internvl,
304 - )
316 + self._get_prompt_repl_image(mm_items, hf_processor, out_mm_data),
317 ]
318
319
@@ -455,44 +467,35 @@ class InternVLMultiModalProcessor(
467
468 return processed_outputs
469
470 + def _get_video_fields_config(self, hf_inputs: BatchFeature):
471 + video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
472 + num_videos = len(video_num_patches)
473 +
474 + return dict(
475 + pixel_values_flat_video=MultiModalFieldConfig.flat_from_sizes(
476 + "video", video_num_patches
477 + ),
478 + video_num_patches=MultiModalFieldConfig.batched("video"),
479 + video_token_id=MultiModalFieldConfig.shared("video", num_videos),
480 + )
481 +
482 def _get_mm_fields_config(
483 self,
484 hf_inputs: BatchFeature,
485 hf_processor_mm_kwargs: Mapping[str, object],
486 ) -> Mapping[str, MultiModalFieldConfig]:
463 - image_fields = super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs)
487 + fields = self._get_image_fields_config(hf_inputs)
488 if self.info.ctx_video_token:
465 - video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
466 - num_videos = len(video_num_patches)
467 - video_fields = dict(
468 - pixel_values_flat_video=MultiModalFieldConfig.flat_from_sizes(
469 - "video", video_num_patches
470 - ),
471 - video_num_patches=MultiModalFieldConfig.batched("video"),
472 - video_token_id=MultiModalFieldConfig.shared("video", num_videos),
473 - )
474 - else:
475 - video_fields = {}
489 + fields |= self._get_video_fields_config(hf_inputs)
490
477 - return image_fields | video_fields
491 + return fields
492
479 - def _get_prompt_updates(
493 + def _get_prompt_repl_video(
494 self,
495 mm_items: MultiModalDataItems,
482 - hf_processor_mm_kwargs: Mapping[str, object],
483 - out_mm_kwargs: MultiModalKwargsItems,
484 - ) -> Sequence[PromptUpdate]:
485 - prompt_repl = super()._get_prompt_updates(
486 - mm_items=mm_items,
487 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
488 - out_mm_kwargs=out_mm_kwargs,
489 - )
490 - if self.info.ctx_video_token is None:
491 - return prompt_repl
492 -
493 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
494 -
495 - out_mm_data = out_mm_kwargs.get_data()
496 + hf_processor: InternVLProcessor,
497 + out_mm_data: BatchedTensorInputs,
498 + ):
499 if "video_num_patches" in out_mm_data:
500 video_num_patches = out_mm_data["video_num_patches"]
501 assert isinstance(video_num_patches, torch.Tensor)
@@ -507,14 +510,30 @@ class InternVLMultiModalProcessor(
510
511 return hf_processor.get_video_repl(num_patches)
512
510 - return [
511 - *prompt_repl,
512 - PromptReplacement(
513 - modality="video",
514 - target="<video>",
515 - replacement=get_video_replacement_internvl,
516 - ),
513 + return PromptReplacement(
514 + modality="video",
515 + target="<video>",
516 + replacement=get_video_replacement_internvl,
517 + )
518 +
519 + def _get_prompt_updates(
520 + self,
521 + mm_items: MultiModalDataItems,
522 + hf_processor_mm_kwargs: Mapping[str, object],
523 + out_mm_kwargs: MultiModalKwargsItems,
524 + ) -> Sequence[PromptUpdate]:
525 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
526 + out_mm_data = out_mm_kwargs.get_data()
527 +
528 + prompt_repls = [
529 + self._get_prompt_repl_image(mm_items, hf_processor, out_mm_data),
530 ]
531 + if self.info.ctx_video_token is not None:
532 + prompt_repls.append(
533 + self._get_prompt_repl_video(mm_items, hf_processor, out_mm_data)
534 + )
535 +
536 + return prompt_repls
537
538
539 @MULTIMODAL_REGISTRY.register_processor(
vllm/model_executor/models/molmo2.py
+17 -7
@@ -1913,22 +1913,32 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
1913 height: int,
1914 num_frames: int,
1915 num_videos: int,
1916 + overrides: VideoDummyOptions | None = None,
1917 ) -> list[VideoItem]:
1917 - video = np.full((num_frames, height, width, 3), 255, dtype=np.uint8)
1918 + videos = super()._get_dummy_videos(
1919 + width=width,
1920 + height=height,
1921 + num_frames=num_frames,
1922 + num_videos=num_videos,
1923 + overrides=overrides,
1924 + )
1925 + videos = [v.copy() for v in videos]
1926 +
1927 video_items = []
1919 - for i in range(num_videos):
1928 + for video in videos:
1929 + video_num_frames = video.shape[0]
1930 video_metadata = {
1931 "fps": 2.0,
1922 - "duration": num_frames / 2.0,
1923 - "total_num_frames": num_frames,
1924 - "frames_indices": list(range(num_frames)),
1932 + "duration": video_num_frames / 2.0,
1933 + "total_num_frames": video_num_frames,
1934 + "frames_indices": list(range(video_num_frames)),
1935 "video_backend": "decord",
1936 "do_sample_frames": False,
1937 "height": height,
1938 "width": width,
1939 }
1930 - video_item = (video.copy(), video_metadata)
1931 - video_items.append(video_item)
1940 + video_items.append((video, video_metadata))
1941 +
1942 return video_items
1943
1944
vllm/model_executor/models/nano_nemotron_vl.py
+277 -332
@@ -10,10 +10,9 @@
10 import copy
11 import math
12 import warnings
13 -from abc import abstractmethod
13 from collections.abc import Iterable, Mapping, Sequence
14 from functools import cached_property
16 -from typing import Annotated, Literal, TypeAlias, TypeVar
15 +from typing import Annotated, Literal, TypeAlias
16
17 import torch
18 import torch.nn as nn
@@ -47,6 +46,7 @@ from vllm.multimodal.evs import (
46 )
47 from vllm.multimodal.inputs import (
48 AudioItem,
49 + BatchedTensorInputs,
50 MultiModalDataDict,
51 MultiModalFieldConfig,
52 MultiModalInputs,
@@ -196,21 +196,58 @@ NanoNemotronVLVideoInputs: TypeAlias = (
196 )
197
198
199 -class BaseNanoNemotronVLProcessingInfo(BaseProcessingInfo):
200 - """Basic image-only ProcessingInfo for InternVL-style models."""
199 +class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
200 + def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
201 + return self.ctx.init_processor(
202 + NanoNemotronVLProcessor,
203 + config=self.get_hf_config(),
204 + tokenizer=self.get_tokenizer(),
205 + video_token=self.get_video_token(),
206 + video_pruning_rate=self.get_video_pruning_rate(),
207 + max_model_len=self.ctx.model_config.max_model_len,
208 + **kwargs,
209 + )
210
202 - @abstractmethod
203 - def get_hf_processor(
204 - self,
205 - **kwargs: object,
206 - ) -> BaseNanoNemotronVLProcessor:
207 - raise NotImplementedError
211 + @cached_property
212 + def is_dynamic_tiler(self) -> bool:
213 + return self.get_hf_processor().dynamic_tiler is not None
214 +
215 + @cached_property
216 + def supports_video(self):
217 + return self.get_hf_processor().supports_video
218 +
219 + def get_video_token(self) -> str | None:
220 + return IMG_CONTEXT
221 +
222 + def get_video_pruning_rate(self) -> float | None:
223 + return self.ctx.get_mm_config().video_pruning_rate
224 +
225 + @property
226 + def audio_extractor(self) -> ParakeetExtractor | None:
227 + return self.get_hf_processor().audio_extractor
228
229 def get_default_tok_params(self) -> TokenizeParams:
230 return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
231
232 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
213 - return {"image": None}
233 + image_limit = {"image": None}
234 + video_limit = {"video": None} if self.supports_video else {}
235 + audio_limit = {"audio": None} if self.audio_extractor is not None else {}
236 + return {**image_limit, **video_limit, **audio_limit}
237 +
238 + def get_data_parser(self):
239 + target_sr = None
240 + target_channels = None
241 + if extractor := self.audio_extractor:
242 + target_sr = extractor.sampling_rate
243 + target_channels = 1
244 +
245 + return MultiModalDataParser(
246 + video_needs_metadata=True,
247 + target_sr=target_sr,
248 + target_channels=target_channels,
249 + expected_hidden_size=self._get_expected_hidden_size(),
250 + )
251
252 def get_image_size_with_most_features(self, max_num_tiles: int) -> ImageSize:
253 processor = self.get_hf_processor()
@@ -248,46 +285,6 @@ class BaseNanoNemotronVLProcessingInfo(BaseProcessingInfo):
285 max_num_tiles=max_num_tiles,
286 )
287
251 -
252 -_I = TypeVar("_I", bound=BaseNanoNemotronVLProcessingInfo)
253 -
254 -
255 -class NanoNemotronVLProcessingInfo(BaseNanoNemotronVLProcessingInfo):
256 - """ProcessingInfo extended for video processing"""
257 -
258 - @property
259 - def supports_video(self):
260 - return self.get_hf_processor().supports_video
261 -
262 - @property
263 - def audio_extractor(self) -> ParakeetExtractor | None:
264 - return self.get_hf_processor().audio_extractor
265 -
266 - def get_data_parser(self):
267 - target_sr = None
268 - target_channels = None
269 - if extractor := self.audio_extractor:
270 - target_sr = extractor.sampling_rate
271 - target_channels = 1
272 -
273 - return MultiModalDataParser(
274 - video_needs_metadata=True,
275 - target_sr=target_sr,
276 - target_channels=target_channels,
277 - expected_hidden_size=self._get_expected_hidden_size(),
278 - )
279 -
280 - def get_supported_mm_limits(self):
281 - video_limit = {"video": None} if self.supports_video else {}
282 - audio_limit = {"audio": None} if self.audio_extractor is not None else {}
283 - return {**super().get_supported_mm_limits(), **video_limit, **audio_limit}
284 -
285 - def get_video_token(self) -> str | None:
286 - return IMG_CONTEXT
287 -
288 - def get_video_pruning_rate(self) -> float | None:
289 - return self.ctx.get_mm_config().video_pruning_rate
290 -
288 def get_num_frames_with_most_features(
289 self,
290 seq_len: int,
@@ -306,31 +303,12 @@ class NanoNemotronVLProcessingInfo(BaseNanoNemotronVLProcessingInfo):
303 max_frames_per_video = max_tubelets_per_video * T
304 return max(max_frames_per_video, 1)
305
309 - def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
310 - return self.ctx.init_processor(
311 - NanoNemotronVLProcessor,
312 - config=self.get_hf_config(),
313 - tokenizer=self.get_tokenizer(),
314 - video_token=self.get_video_token(),
315 - video_pruning_rate=self.get_video_pruning_rate(),
316 - max_model_len=self.ctx.model_config.max_model_len,
317 - **kwargs,
318 - )
319 -
320 -
321 -class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
322 - """Basic image-only MultiModalProcessor for InternVL-style models."""
323 -
324 - @cached_property
325 - def is_dynamic_tiler(self) -> bool:
326 - return self.info.get_hf_processor().dynamic_tiler is not None
306
328 - def _get_mm_fields_config(
329 - self,
330 - hf_inputs: BatchFeature,
331 - hf_processor_mm_kwargs: Mapping[str, object],
332 - ) -> Mapping[str, MultiModalFieldConfig]:
333 - if self.is_dynamic_tiler:
307 +class NanoNemotronVLMultiModalProcessor(
308 + BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
309 +):
310 + def _get_image_fields_config(self, hf_inputs: BatchFeature):
311 + if self.info.is_dynamic_tiler:
312 pixel_values_flat = MultiModalFieldConfig.batched("image")
313 else:
314 image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -346,15 +324,50 @@ class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
324 imgs_sizes=MultiModalFieldConfig.batched("image"),
325 )
326
349 - def _get_prompt_updates(
327 + def _get_video_fields_config(self, hf_inputs: BatchFeature):
328 + video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
329 +
330 + return dict(
331 + pixel_values_flat_video=MultiModalFieldConfig.flat_from_sizes(
332 + "video", video_num_patches
333 + ),
334 + video_num_patches=MultiModalFieldConfig.batched("video"),
335 + frames_indices=MultiModalFieldConfig.batched("video"),
336 + frame_duration_ms=MultiModalFieldConfig.batched("video"),
337 + )
338 +
339 + def _get_audio_fields_config(self, hf_inputs: BatchFeature):
340 + audio_num_clips = torch.as_tensor(hf_inputs["audio_num_clips"])
341 +
342 + return dict(
343 + input_audio_features=MultiModalFieldConfig.flat_from_sizes(
344 + "audio", audio_num_clips
345 + ),
346 + feature_attention_mask=MultiModalFieldConfig.flat_from_sizes(
347 + "audio", audio_num_clips
348 + ),
349 + audio_num_clips=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
350 + )
351 +
352 + def _get_mm_fields_config(
353 self,
351 - mm_items: MultiModalDataItems,
354 + hf_inputs: BatchFeature,
355 hf_processor_mm_kwargs: Mapping[str, object],
353 - out_mm_kwargs: MultiModalKwargsItems,
354 - ) -> Sequence[PromptUpdate]:
355 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
356 + ) -> Mapping[str, MultiModalFieldConfig]:
357 + fields = self._get_image_fields_config(hf_inputs)
358 + if self.info.supports_video:
359 + fields |= self._get_video_fields_config(hf_inputs)
360 + if self.info.audio_extractor:
361 + fields |= self._get_audio_fields_config(hf_inputs)
362
357 - out_mm_data = out_mm_kwargs.get_data()
363 + return fields
364 +
365 + def _get_prompt_repl_image(
366 + self,
367 + mm_items: MultiModalDataItems,
368 + hf_processor: NanoNemotronVLProcessor,
369 + out_mm_data: BatchedTensorInputs,
370 + ):
371 if "image_num_patches" in out_mm_data:
372 image_num_patches = out_mm_data["image_num_patches"]
373 assert isinstance(image_num_patches, torch.Tensor)
@@ -365,7 +378,7 @@ class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
378 else:
379 image_num_patches = []
380
368 - def get_replacement_custom(item_idx: int):
381 + def get_image_replacement(item_idx: int):
382 images = mm_items.get_items(
383 "image", (ImageEmbeddingItems, ImageProcessorItems)
384 )
@@ -377,10 +390,7 @@ class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
390 feature_size = tiler.get_cached_feature_size(image)
391 else:
392 image_size = images.get_image_size(item_idx)
380 - # Extract max_num_tiles from kwargs, default to 12
381 - max_num_tiles = hf_processor_mm_kwargs.get(
382 - "max_num_tiles", hf_processor.max_num_tiles
383 - )
393 + max_num_tiles = hf_processor.max_num_tiles
394 feature_size = hf_processor.get_num_image_tokens(
395 image_width=image_size.width,
396 image_height=image_size.height,
@@ -398,19 +408,126 @@ class NanoNemotronBaseVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
408
409 return hf_processor.get_image_repl(feature_size, num_patches)
410
401 - return [
402 - PromptReplacement(
403 - modality="image",
404 - target="<image>",
405 - replacement=get_replacement_custom,
411 + return PromptReplacement(
412 + modality="image",
413 + target="<image>",
414 + replacement=get_image_replacement,
415 + )
416 +
417 + def _get_prompt_repl_video(
418 + self,
419 + mm_items: MultiModalDataItems,
420 + hf_processor: NanoNemotronVLProcessor,
421 + out_mm_data: BatchedTensorInputs,
422 + ):
423 + if "video_num_patches" in out_mm_data:
424 + video_num_patches = out_mm_data["video_num_patches"]
425 + assert isinstance(video_num_patches, torch.Tensor)
426 + video_num_patches = video_num_patches.tolist()
427 + else:
428 + video_num_patches = []
429 +
430 + def get_video_replacement(item_idx: int):
431 + video, metadata = mm_items["video"][item_idx]
432 + patch_size = hf_processor.config.patch_size
433 + downsample_ratio = hf_processor.config.downsample_ratio
434 + target_patches = hf_processor.video_target_num_patches
435 +
436 + if target_patches is not None and video is not None and video.shape[0] > 0:
437 + orig_h, orig_w = video.shape[1], video.shape[2]
438 + _, _, feature_size = get_video_target_size_and_feature_size(
439 + orig_w=orig_w,
440 + orig_h=orig_h,
441 + target_patches=target_patches,
442 + maintain_aspect_ratio=hf_processor.video_maintain_aspect_ratio,
443 + patch_size=patch_size,
444 + downsample_ratio=downsample_ratio,
445 + )
446 + else:
447 + feature_size = hf_processor.num_image_token
448 + num_patches = video_num_patches[item_idx]
449 + if num_patches is not None:
450 + assert isinstance(num_patches, int)
451 +
452 + T = hf_processor.video_temporal_patch_size
453 + if T > 1 and num_patches is not None:
454 + num_tubelets = math.ceil(num_patches / T)
455 + else:
456 + num_tubelets = num_patches
457 +
458 + video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
459 + if video_pruning_rate is not None and video_pruning_rate > 0.0:
460 + # Start of EVS-specific code
461 + num_tokens = compute_retained_tokens_count(
462 + tokens_per_frame=feature_size,
463 + num_frames=num_tubelets,
464 + q=video_pruning_rate,
465 + )
466 + # Here we just need placeholders that won't actually be replaced -
467 + # we just need to make sure the total number of tokens is correct
468 + # assign all tokens to the first frame
469 + tokens_per_frame = [num_tokens] + [0] * (num_tubelets - 1)
470 +
471 + # End of EVS-specific code
472 + else:
473 + tokens_per_frame = [feature_size] * num_tubelets
474 +
475 + frame_duration_ms = int(1000 / metadata["fps"])
476 + return hf_processor.get_video_repl(
477 + tokens_per_frame=tokens_per_frame,
478 + frames_indices=metadata["frames_indices"],
479 + frame_duration_ms=frame_duration_ms,
480 + tokenizer=hf_processor.tokenizer,
481 + img_start_token_ids=hf_processor._img_start_token_ids,
482 + img_end_token_ids=hf_processor._img_end_token_ids,
483 + img_context_token_ids=hf_processor._img_context_token_ids,
484 + video_temporal_patch_size=T,
485 )
407 - ]
486
487 + return PromptReplacement(
488 + modality="video",
489 + target="<video>",
490 + replacement=get_video_replacement,
491 + )
492
410 -class NanoNemotronVLMultiModalProcessor(
411 - NanoNemotronBaseVLMultiModalProcessor[NanoNemotronVLProcessingInfo]
412 -):
413 - """MultiModalProcessor extended for video support"""
493 + def _get_prompt_repl_audio(
494 + self,
495 + mm_items: MultiModalDataItems,
496 + hf_processor: NanoNemotronVLProcessor,
497 + out_mm_data: BatchedTensorInputs,
498 + ):
499 + def get_audio_replacement(item_idx: int):
500 + audios = mm_items.get_items("audio", AudioProcessorItems)
501 + return hf_processor.get_audio_repl(audios.get(item_idx))
502 +
503 + return PromptReplacement(
504 + modality="audio",
505 + target=AUDIO_CONTEXT,
506 + replacement=get_audio_replacement,
507 + )
508 +
509 + def _get_prompt_updates(
510 + self,
511 + mm_items: MultiModalDataItems,
512 + hf_processor_mm_kwargs: Mapping[str, object],
513 + out_mm_kwargs: MultiModalKwargsItems,
514 + ) -> Sequence[PromptUpdate]:
515 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
516 + out_mm_data = out_mm_kwargs.get_data()
517 +
518 + prompt_repls = [
519 + self._get_prompt_repl_image(mm_items, hf_processor, out_mm_data),
520 + ]
521 + if self.info.supports_video:
522 + prompt_repls.append(
523 + self._get_prompt_repl_video(mm_items, hf_processor, out_mm_data)
524 + )
525 + if self.info.audio_extractor:
526 + prompt_repls.append(
527 + self._get_prompt_repl_audio(mm_items, hf_processor, out_mm_data)
528 + )
529 +
530 + return prompt_repls
531
532 def _extract_audio_from_videos(
533 self,
@@ -456,38 +573,29 @@ class NanoNemotronVLMultiModalProcessor(
573
574 def apply(
575 self,
459 - processor_inputs: ProcessorInputs,
460 - timing_ctx: TimingContext | None = None,
576 + inputs: ProcessorInputs,
577 + timing_ctx: TimingContext,
578 ) -> MultiModalInputs:
462 - if (hf_processor_mm_kwargs := processor_inputs.hf_processor_mm_kwargs) is None:
463 - hf_processor_mm_kwargs = {}
464 -
579 use_audio_in_video = bool(
466 - hf_processor_mm_kwargs.get("use_audio_in_video", False)
580 + inputs.hf_processor_mm_kwargs.get("use_audio_in_video", False)
581 )
468 -
469 - hf_processor_mm_kwargs = {
470 - k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
582 + inputs.hf_processor_mm_kwargs = {
583 + k: v
584 + for k, v in inputs.hf_processor_mm_kwargs.items()
585 + if k != "use_audio_in_video"
586 }
587
473 - processor_inputs.hf_processor_mm_kwargs = hf_processor_mm_kwargs
474 -
588 if not (
589 use_audio_in_video
477 - and "video" in processor_inputs.mm_data_items
478 - and "audio" not in processor_inputs.mm_data_items
590 + and "video" in inputs.mm_data_items
591 + and "audio" not in inputs.mm_data_items
592 ):
480 - return super().apply(
481 - processor_inputs,
482 - timing_ctx,
483 - )
593 + return super().apply(inputs, timing_ctx)
594
485 - mm_items, audio_items = self._extract_audio_from_videos(
486 - processor_inputs.mm_data_items
487 - )
488 - processor_inputs.mm_data_items = mm_items
595 + mm_items, audio_items = self._extract_audio_from_videos(inputs.mm_data_items)
596 + inputs.mm_data_items = mm_items
597
490 - prompt = processor_inputs.prompt
598 + prompt = inputs.prompt
599 tokenizer = self.info.get_tokenizer()
600 if not isinstance(prompt, str):
601 prompt = tokenizer.decode(prompt, skip_special_tokens=False)
@@ -495,10 +603,10 @@ class NanoNemotronVLMultiModalProcessor(
603 for _ in audio_items:
604 prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)
605
498 - processor_inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)
606 + inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)
607
500 - if processor_inputs.tokenization_kwargs is None:
501 - processor_inputs.tokenization_kwargs = {}
608 + if inputs.tokenization_kwargs is None:
609 + inputs.tokenization_kwargs = {}
610
611 # Bypass the cached path: the HF processor must receive the
612 # prompt (with injected <so_embedding>) and the audio data
@@ -507,18 +615,16 @@ class NanoNemotronVLMultiModalProcessor(
615 prompt_ids,
616 mm_info,
617 is_update_applied,
510 - ) = self._apply_hf_processor(
511 - processor_inputs,
512 - timing_ctx=timing_ctx,
513 - )
514 -
515 - prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
516 - mm_items=mm_items,
517 - prompt_ids=prompt_ids,
518 - mm_kwargs=mm_info.kwargs,
519 - mm_prompt_updates=mm_info.prompt_updates,
520 - is_update_applied=is_update_applied,
521 - )
618 + ) = self._apply_hf_processor(inputs, timing_ctx)
619 +
620 + with timing_ctx.record("apply_prompt_updates"):
621 + prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
622 + mm_items=mm_items,
623 + prompt_ids=prompt_ids,
624 + mm_kwargs=mm_info.kwargs,
625 + mm_prompt_updates=mm_info.prompt_updates,
626 + is_update_applied=is_update_applied,
627 + )
628
629 mm_placeholder_ranges = {
630 modality: [item.to_range() for item in placeholders]
@@ -533,202 +639,17 @@ class NanoNemotronVLMultiModalProcessor(
639 mm_placeholders=mm_placeholder_ranges,
640 )
641
536 - def _get_mm_fields_config(
537 - self,
538 - hf_inputs: BatchFeature,
539 - hf_processor_mm_kwargs: Mapping[str, object],
540 - ) -> Mapping[str, MultiModalFieldConfig]:
541 - image_fields = super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs)
542 - if self.info.supports_video:
543 - video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
544 -
545 - video_fields = dict(
546 - pixel_values_flat_video=MultiModalFieldConfig.flat_from_sizes(
547 - "video", video_num_patches
548 - ),
549 - video_num_patches=MultiModalFieldConfig.batched("video"),
550 - frames_indices=MultiModalFieldConfig.batched("video"),
551 - frame_duration_ms=MultiModalFieldConfig.batched("video"),
552 - )
553 - else:
554 - video_fields = {}
555 -
556 - if self.info.audio_extractor is not None:
557 - audio_num_clips = torch.as_tensor(hf_inputs["audio_num_clips"])
558 - audio_fields = dict(
559 - input_audio_features=MultiModalFieldConfig.flat_from_sizes(
560 - "audio", audio_num_clips
561 - ),
562 - feature_attention_mask=MultiModalFieldConfig.flat_from_sizes(
563 - "audio", audio_num_clips
564 - ),
565 - audio_num_clips=MultiModalFieldConfig.batched(
566 - "audio", keep_on_cpu=True
567 - ),
568 - )
569 - else:
570 - audio_fields = {}
571 -
572 - return image_fields | video_fields | audio_fields
573 -
574 - def _get_prompt_updates(
575 - self,
576 - mm_items: MultiModalDataItems,
577 - hf_processor_mm_kwargs: Mapping[str, object],
578 - out_mm_kwargs: MultiModalKwargsItems,
579 - ) -> Sequence[PromptUpdate]:
580 - prompt_repl = super()._get_prompt_updates(
581 - mm_items=mm_items,
582 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
583 - out_mm_kwargs=out_mm_kwargs,
584 - )
585 -
586 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
587 -
588 - out_mm_data = out_mm_kwargs.get_data()
589 - if "video_num_patches" in out_mm_data:
590 - video_num_patches = out_mm_data["video_num_patches"]
591 - assert isinstance(video_num_patches, torch.Tensor)
592 - video_num_patches = video_num_patches.tolist()
593 - else:
594 - video_num_patches = []
595 -
596 - def get_video_replacement_internvl(item_idx: int):
597 - video, metadata = mm_items["video"][item_idx]
598 - patch_size = hf_processor.config.patch_size
599 - downsample_ratio = hf_processor.config.downsample_ratio
600 - target_patches = hf_processor.video_target_num_patches
601 -
602 - if target_patches is not None and video is not None and video.shape[0] > 0:
603 - orig_h, orig_w = video.shape[1], video.shape[2]
604 - _, _, feature_size = get_video_target_size_and_feature_size(
605 - orig_w=orig_w,
606 - orig_h=orig_h,
607 - target_patches=target_patches,
608 - maintain_aspect_ratio=hf_processor.video_maintain_aspect_ratio,
609 - patch_size=patch_size,
610 - downsample_ratio=downsample_ratio,
611 - )
612 - else:
613 - feature_size = hf_processor.num_image_token
614 - num_patches = video_num_patches[item_idx]
615 - if num_patches is not None:
616 - assert isinstance(num_patches, int)
617 -
618 - T = hf_processor.video_temporal_patch_size
619 - if T > 1 and num_patches is not None:
620 - num_tubelets = math.ceil(num_patches / T)
621 - else:
622 - num_tubelets = num_patches
623 -
624 - video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
625 - if video_pruning_rate is not None and video_pruning_rate > 0.0:
626 - # Start of EVS-specific code
627 - num_tokens = compute_retained_tokens_count(
628 - tokens_per_frame=feature_size,
629 - num_frames=num_tubelets,
630 - q=video_pruning_rate,
631 - )
632 - # Here we just need placeholders that won't actually be replaced -
633 - # we just need to make sure the total number of tokens is correct
634 - # assign all tokens to the first frame
635 - tokens_per_frame = [num_tokens] + [0] * (num_tubelets - 1)
636 -
637 - # End of EVS-specific code
638 - else:
639 - tokens_per_frame = [feature_size] * num_tubelets
640 -
641 - frame_duration_ms = int(1000 / metadata["fps"])
642 - return hf_processor.get_video_repl(
643 - tokens_per_frame=tokens_per_frame,
644 - frames_indices=metadata["frames_indices"],
645 - frame_duration_ms=frame_duration_ms,
646 - tokenizer=hf_processor.tokenizer,
647 - img_start_token_ids=hf_processor._img_start_token_ids,
648 - img_end_token_ids=hf_processor._img_end_token_ids,
649 - img_context_token_ids=hf_processor._img_context_token_ids,
650 - video_temporal_patch_size=T,
651 - )
652 -
653 - if self.info.supports_video:
654 - prompt_repl = [
655 - *prompt_repl,
656 - PromptReplacement(
657 - modality="video",
658 - target="<video>",
659 - replacement=get_video_replacement_internvl,
660 - ),
661 - ]
662 -
663 - def get_audio_replacement(item_idx: int):
664 - audios = mm_items.get_items("audio", AudioProcessorItems)
665 - return hf_processor.get_audio_repl(audios.get(item_idx))
666 -
667 - if self.info.audio_extractor is not None:
668 - prompt_repl = [
669 - *prompt_repl,
670 - PromptReplacement(
671 - modality="audio",
672 - target=AUDIO_CONTEXT,
673 - replacement=get_audio_replacement,
674 - ),
675 - ]
676 -
677 - return prompt_repl
678 -
679 -
680 -class NanoNemotronVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
681 - """Basic image-only DummyInputsBuilder for InternVL-style models."""
682 -
683 - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
684 - num_images = mm_counts.get("image", 0)
685 -
686 - return "<image>" * num_images
687 -
688 - def get_dummy_mm_data(
689 - self,
690 - seq_len: int,
691 - mm_counts: Mapping[str, int],
692 - mm_options: Mapping[str, BaseDummyOptions],
693 - ) -> MultiModalDataDict:
694 - num_images = mm_counts.get("image", 0)
695 - processor = self.info.get_hf_processor()
696 - if tiler := processor.dynamic_tiler:
697 - budget = tiler.max_num_tokens_available(text_prompt_length=num_images)
698 - target_width, target_height = (
699 - tiler.width_and_height_for_max_num_tokens_available(budget)
700 - )
701 - else:
702 - max_num_tiles = 12
703 - target_width, target_height = self.info.get_image_size_with_most_features(
704 - max_num_tiles
705 - )
706 -
707 - image_overrides = mm_options.get("image")
708 -
709 - return {
710 - "image": self._get_dummy_images(
711 - width=target_width,
712 - height=target_height,
713 - num_images=num_images,
714 - overrides=image_overrides,
715 - )
716 - }
717 -
642
643 class NanoNemotronVLDummyInputsBuilder(
720 - NanoNemotronVLDummyInputsBuilder[NanoNemotronVLProcessingInfo]
644 + BaseDummyInputsBuilder[NanoNemotronVLProcessingInfo]
645 ):
722 - """DummyInputsBuilder extended for video support"""
723 -
646 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
647 + num_images = mm_counts.get("image", 0)
648 num_videos = mm_counts.get("video", 0)
649 num_audios = mm_counts.get("audio", 0)
650
651 return (
729 - super().get_dummy_text(mm_counts)
730 - + "<video>" * num_videos
731 - + AUDIO_CONTEXT * num_audios
652 + "<image>" * num_images + "<video>" * num_videos + AUDIO_CONTEXT * num_audios
653 )
654
655 def _get_dummy_videos(
@@ -740,25 +661,27 @@ class NanoNemotronVLDummyInputsBuilder(
661 num_videos: int,
662 overrides: VideoDummyOptions | None = None,
663 ) -> list[VideoItem]:
743 - video = super()._get_dummy_videos(
664 + videos = super()._get_dummy_videos(
665 width=width,
666 height=height,
667 num_frames=num_frames,
747 - num_videos=1,
668 + num_videos=num_videos,
669 overrides=overrides,
749 - )[0]
670 + )
671 + videos = [v.copy() for v in videos]
672 +
673 video_items = []
751 - for _ in range(num_videos):
674 + for video in videos:
675 + video_num_frames = video.shape[0]
676 video_metadata = {
753 - "total_num_frames": num_frames,
677 "fps": 2,
755 - "duration": num_frames / 2.0,
678 + "duration": video_num_frames / 2.0,
679 + "total_num_frames": video_num_frames,
680 + "frames_indices": list(range(video_num_frames)),
681 "video_backend": "opencv_dynamic",
757 - "frames_indices": [i for i in range(num_frames)],
682 "do_sample_frames": False,
683 }
760 - video_item = (video.copy(), video_metadata)
761 - video_items.append(video_item)
684 + video_items.append((video, video_metadata))
685
686 return video_items
687
@@ -768,11 +691,33 @@ class NanoNemotronVLDummyInputsBuilder(
691 mm_counts: Mapping[str, int],
692 mm_options: Mapping[str, BaseDummyOptions],
693 ) -> MultiModalDataDict:
771 - dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)
694 + num_images = mm_counts.get("image", 0)
695 + processor = self.info.get_hf_processor()
696 + if tiler := processor.dynamic_tiler:
697 + budget = tiler.max_num_tokens_available(text_prompt_length=num_images)
698 + target_width, target_height = (
699 + tiler.width_and_height_for_max_num_tokens_available(budget)
700 + )
701 + else:
702 + max_num_tiles = 12
703 + target_width, target_height = self.info.get_image_size_with_most_features(
704 + max_num_tiles
705 + )
706 +
707 + image_overrides = mm_options.get("image")
708 +
709 + dummy_image = {
710 + "image": self._get_dummy_images(
711 + width=target_width,
712 + height=target_height,
713 + num_images=num_images,
714 + overrides=image_overrides,
715 + )
716 + }
717 +
718 if self.info.supports_video:
719 config = self.info.get_hf_config()
720 image_size: int = config.force_image_size
775 - processor = self.info.get_hf_processor()
721
722 # When video_target_num_patches is set the per-frame pixel
723 # resolution can exceed image_size. Use the actual target
vllm/model_executor/models/nvlm_d.py
+14 -17
@@ -7,7 +7,7 @@
7 # Copyright (c) 2024 NVIDIA
8 # Licensed under Apache 2.0 License [see LICENSE for details]
9 # --------------------------------------------------------
10 -from collections.abc import Mapping, Sequence
10 +from collections.abc import Mapping
11
12 import torch
13 import torch.nn as nn
@@ -16,7 +16,10 @@ from transformers import PretrainedConfig
16 from vllm.config.multimodal import BaseDummyOptions
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 -from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems
19 +from vllm.multimodal.inputs import (
20 + BatchedTensorInputs,
21 + MultiModalDataDict,
22 +)
23 from vllm.multimodal.parse import (
24 ImageEmbeddingItems,
25 ImageProcessorItems,
@@ -24,7 +27,6 @@ from vllm.multimodal.parse import (
27 )
28 from vllm.multimodal.processing import (
29 PromptReplacement,
27 - PromptUpdate,
30 PromptUpdateDetails,
31 )
32 from vllm.transformers_utils.processors.internvl import InternVLImageProcessor
@@ -100,15 +102,12 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo])
102
103
104 class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo]):
103 - def _get_prompt_updates(
105 + def _get_prompt_repl_image(
106 self,
107 mm_items: MultiModalDataItems,
106 - hf_processor_mm_kwargs: Mapping[str, object],
107 - out_mm_kwargs: MultiModalKwargsItems,
108 - ) -> Sequence[PromptUpdate]:
109 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
110 -
111 - out_mm_data = out_mm_kwargs.get_data()
108 + hf_processor: NVLMProcessor,
109 + out_mm_data: BatchedTensorInputs,
110 + ):
111 if "image_num_patches" in out_mm_data:
112 image_num_patches = out_mm_data["image_num_patches"]
113 assert isinstance(image_num_patches, torch.Tensor)
@@ -146,13 +145,11 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
145 )
146
147 # See note in dummy data regarding why we have the extra newline
149 - return [
150 - PromptReplacement(
151 - modality="image",
152 - target="<image>\n",
153 - replacement=get_replacement_nvlm,
154 - )
155 - ]
148 + return PromptReplacement(
149 + modality="image",
150 + target="<image>\n",
151 + replacement=get_replacement_nvlm,
152 + )
153
154
155 @MULTIMODAL_REGISTRY.register_processor(
vllm/model_executor/models/qwen3_vl.py
+17 -7
@@ -931,20 +931,30 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
931 height: int,
932 num_frames: int,
933 num_videos: int,
934 + overrides: VideoDummyOptions | None = None,
935 ) -> list[VideoItem]:
935 - video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)
936 + videos = super()._get_dummy_videos(
937 + width=width,
938 + height=height,
939 + num_frames=num_frames,
940 + num_videos=num_videos,
941 + overrides=overrides,
942 + )
943 + videos = [v.copy() for v in videos]
944 +
945 video_items = []
937 - for i in range(num_videos):
946 + for video in videos:
947 + video_num_frames = video.shape[0]
948 video_metadata = {
949 "fps": 2.0,
940 - "duration": num_frames / 2.0,
941 - "total_num_frames": num_frames,
942 - "frames_indices": [i for i in range(num_frames)],
950 + "duration": video_num_frames / 2.0,
951 + "total_num_frames": video_num_frames,
952 + "frames_indices": list(range(video_num_frames)),
953 "video_backend": "opencv",
954 "do_sample_frames": False,
955 }
946 - video_item = (video.copy(), video_metadata)
947 - video_items.append(video_item)
956 + video_items.append((video, video_metadata))
957 +
958 return video_items
959
960
vllm/model_executor/models/skyworkr1v.py
+12 -155
@@ -7,12 +7,12 @@
7 # Copyright (c) 2025 Skywork
8 # Licensed under The MIT License [see LICENSE for details]
9 # --------------------------------------------------------
10 -from collections.abc import Iterable, Mapping, Sequence
10 +from collections.abc import Iterable, Mapping
11 from typing import Annotated, Literal, TypeAlias
12
13 import torch
14 import torch.nn as nn
15 -from transformers import BatchFeature, PretrainedConfig
15 +from transformers import PretrainedConfig
16
17 from vllm.config import VllmConfig
18 from vllm.config.multimodal import BaseDummyOptions
@@ -24,24 +24,8 @@ from vllm.model_executor.models.intern_vit import (
24 InternVisionPatchModel,
25 )
26 from vllm.multimodal import MULTIMODAL_REGISTRY
27 -from vllm.multimodal.inputs import (
28 - MultiModalDataDict,
29 - MultiModalFieldConfig,
30 - MultiModalKwargsItems,
31 -)
32 -from vllm.multimodal.parse import (
33 - ImageEmbeddingItems,
34 - ImageProcessorItems,
35 - ImageSize,
36 - MultiModalDataItems,
37 -)
38 -from vllm.multimodal.processing import (
39 - BaseDummyInputsBuilder,
40 - BaseMultiModalProcessor,
41 - BaseProcessingInfo,
42 - PromptReplacement,
43 - PromptUpdate,
44 -)
27 +from vllm.multimodal.inputs import MultiModalDataDict
28 +from vllm.multimodal.processing import BaseDummyInputsBuilder
29 from vllm.sequence import IntermediateTensors
30 from vllm.transformers_utils.processors.internvl import (
31 InternVLImageProcessor,
@@ -50,6 +34,11 @@ from vllm.transformers_utils.processors.internvl import (
34 from vllm.utils.tensor_schema import TensorSchema, TensorShape
35
36 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
37 +from .internvl import (
38 + BaseInternVLDummyInputsBuilder,
39 + BaseInternVLMultiModalProcessor,
40 + BaseInternVLProcessingInfo,
41 +)
42 from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
43
44
@@ -98,7 +87,7 @@ SkyworkR1VImageInputs: TypeAlias = (
87 )
88
89
101 -class SkyworkR1VProcessingInfo(BaseProcessingInfo):
90 +class SkyworkR1VProcessingInfo(BaseInternVLProcessingInfo):
91 def get_image_processor(self, **kwargs):
92 config = self.get_hf_config()
93 vision_config = config.vision_config
@@ -128,46 +117,6 @@ class SkyworkR1VProcessingInfo(BaseProcessingInfo):
117 image_seq_length=image_seq_length,
118 )
119
131 - def get_supported_mm_limits(self) -> Mapping[str, int | None]:
132 - return {"image": None}
133 -
134 - def get_num_image_tokens(
135 - self,
136 - *,
137 - image_width: int,
138 - image_height: int,
139 - processor: InternVLProcessor,
140 - ) -> int:
141 - return processor.get_num_image_tokens(
142 - image_width=image_width,
143 - image_height=image_height,
144 - )
145 -
146 - def get_image_size_with_most_features(self) -> ImageSize:
147 - processor = self.get_hf_processor()
148 - image_processor = processor.image_processor
149 -
150 - base_size = image_processor.image_size
151 - target_ratios = processor.resolve_target_ratios()
152 -
153 - largest_feature_size, largest_feature_pinpoint = 0, None
154 - for wr, hr in target_ratios:
155 - width, height = base_size * wr, base_size * hr
156 -
157 - feat_size = self.get_num_image_tokens(
158 - image_width=width,
159 - image_height=height,
160 - processor=processor,
161 - )
162 - if feat_size > largest_feature_size:
163 - largest_feature_size = feat_size
164 - largest_feature_pinpoint = ImageSize(width=width, height=height)
165 -
166 - if largest_feature_size == 0 or largest_feature_pinpoint is None:
167 - raise ValueError("Cannot have a largest feature size of 0!")
168 -
169 - return largest_feature_pinpoint
170 -
120
121 class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[SkyworkR1VProcessingInfo]):
122 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
@@ -196,102 +145,10 @@ class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[SkyworkR1VProcessingIn
145 }
146
147
199 -class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[SkyworkR1VProcessingInfo]):
200 - def _call_hf_processor(
201 - self,
202 - prompt: str,
203 - mm_data: Mapping[str, object],
204 - mm_kwargs: Mapping[str, object],
205 - tok_kwargs: Mapping[str, object],
206 - ) -> BatchFeature:
207 - processed_outputs = super()._call_hf_processor(
208 - prompt=prompt,
209 - mm_data=mm_data,
210 - mm_kwargs=mm_kwargs,
211 - tok_kwargs=tok_kwargs,
212 - )
213 -
214 - hf_processor = self.info.get_hf_processor(**mm_kwargs)
215 - image_token_id = hf_processor.ctx_image_token_id
216 -
217 - # Since there may be extra tokens in the feature placeholders,
218 - # we need to pass the image token ID to the model to select the
219 - # tokens to merge from the vision encoder outputs
220 - processed_outputs["image_token_id"] = torch.tensor(image_token_id)
221 -
222 - return processed_outputs
223 -
224 - def _get_mm_fields_config(
225 - self,
226 - hf_inputs: BatchFeature,
227 - hf_processor_mm_kwargs: Mapping[str, object],
228 - ) -> Mapping[str, MultiModalFieldConfig]:
229 - image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
230 - num_images = len(image_num_patches)
231 -
232 - return dict(
233 - pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
234 - "image", image_num_patches
235 - ),
236 - image_num_patches=MultiModalFieldConfig.batched("image"),
237 - image_embeds=MultiModalFieldConfig.batched("image"),
238 - image_token_id=MultiModalFieldConfig.shared("image", num_images),
239 - )
240 -
241 - def _get_prompt_updates(
242 - self,
243 - mm_items: MultiModalDataItems,
244 - hf_processor_mm_kwargs: Mapping[str, object],
245 - out_mm_kwargs: MultiModalKwargsItems,
246 - ) -> Sequence[PromptUpdate]:
247 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
248 -
249 - out_mm_data = out_mm_kwargs.get_data()
250 - if "image_num_patches" in out_mm_data:
251 - image_num_patches = out_mm_data["image_num_patches"]
252 - assert isinstance(image_num_patches, torch.Tensor)
253 - image_num_patches = image_num_patches.tolist()
254 - elif "image_embeds" in out_mm_data:
255 - # TODO: Use image size information in dictionary embedding inputs
256 - # to compute num_patches (similar to Qwen2-VL)
257 - image_num_patches = [None] * len(out_mm_data["image_embeds"])
258 - else:
259 - image_num_patches = []
260 -
261 - def get_replacement_skyworkr1v(item_idx: int):
262 - images = mm_items.get_items(
263 - "image", (ImageEmbeddingItems, ImageProcessorItems)
264 - )
265 -
266 - if isinstance(images, ImageEmbeddingItems):
267 - feature_size = images.get_feature_size(item_idx)
268 - else:
269 - image_size = images.get_image_size(item_idx)
270 - feature_size = self.info.get_num_image_tokens(
271 - image_width=image_size.width,
272 - image_height=image_size.height,
273 - processor=hf_processor,
274 - )
275 -
276 - num_patches = image_num_patches[item_idx]
277 - if num_patches is not None:
278 - assert isinstance(num_patches, int)
279 -
280 - return hf_processor.get_image_repl(num_patches, num_features=feature_size)
281 -
282 - return [
283 - PromptReplacement(
284 - modality="image",
285 - target="<image>",
286 - replacement=get_replacement_skyworkr1v,
287 - )
288 - ]
289 -
290 -
148 @MULTIMODAL_REGISTRY.register_processor(
292 - SkyworkR1VMultiModalProcessor,
149 + BaseInternVLMultiModalProcessor,
150 info=SkyworkR1VProcessingInfo,
294 - dummy_inputs=SkyworkR1VDummyInputsBuilder,
151 + dummy_inputs=BaseInternVLDummyInputsBuilder,
152 )
153 class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
154 @classmethod