@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e93ff6c8b9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Nemotron Nano V2 VL + EVS Video Support (#27107)

Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com> Signed-off-by: Natan Bagrov <nbagrov@nvidia.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Natan Bagrov <nbagrov@nvidia.com> Co-authored-by: Roger Wang <hey@rogerw.io>

Eugene Khvedchenya committed Oct 20, 2025 at 17:19 UTC e93ff6c8b92b7e7d067f535b52ac7c6304e6b316
5 files changed +316 -105
vllm/model_executor/models/nano_nemotron_vl.py
+266 -49
@@ -14,6 +14,7 @@ from collections.abc import Iterable, Mapping, Sequence
14 from typing import Annotated, Any, Literal, TypeAlias, TypeVar
15
16 import numpy.typing as npt
17 +import regex as re
18 import torch
19 import torch.nn as nn
20 import torchvision.transforms as T
@@ -21,7 +22,7 @@ from PIL import Image
22 from transformers import BatchFeature, PretrainedConfig, TensorType
23
24 from vllm.config import VllmConfig
24 -from vllm.config.multimodal import BaseDummyOptions
25 +from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
26 from vllm.model_executor.layers.activation import ReLUSquaredActivation
27 from vllm.model_executor.layers.layernorm import RMSNorm
28 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -53,12 +54,14 @@ from vllm.multimodal.inputs import (
54 MultiModalFieldConfig,
55 MultiModalKwargs,
56 MultiModalKwargsItems,
57 + VideoItem,
58 )
59 from vllm.multimodal.parse import (
60 ImageEmbeddingItems,
61 ImageProcessorItems,
62 ImageSize,
63 MultiModalDataItems,
64 + MultiModalDataParser,
65 )
66 from vllm.multimodal.processing import (
67 BaseMultiModalProcessor,
@@ -91,7 +94,7 @@ IMG_END = "</img>"
94 IMG_CONTEXT = "<image>"
95
96 # Profiling
94 -MAX_FRAMES = 16
97 +# MAX_FRAMES = 16
98 DEFAULT_NUM_TILES = 12
99
100
@@ -131,7 +134,8 @@ class NanoNemotronVLVideoPixelInputs(TensorSchema):
134 """
135 Dimensions:
136 - bvf: Batch size * number of videos * num_frames
134 - - bn: Batch size * number of images
137 + - bn: Batch size * number of videos
138 + - f: Number of frames
139 - c: Number of channels (3)
140 - h: Height of each video frame
141 - w: Width of each video frame
@@ -140,6 +144,8 @@ class NanoNemotronVLVideoPixelInputs(TensorSchema):
144 type: Literal["pixel_values_videos"]
145 pixel_values_flat: Annotated[torch.Tensor, TensorShape("bvf", 3, "h", "w")]
146 num_patches: Annotated[torch.Tensor, TensorShape("bn")]
147 + frames_indices: Annotated[torch.Tensor, TensorShape("bvf")]
148 + frame_duration_ms: Annotated[torch.Tensor, TensorShape("bn")]
149
150
151 class NanoNemotronVLVideoEmbeddingInputs(TensorSchema):
@@ -251,6 +257,21 @@ def video_to_pixel_values(
257 return torch.stack(frames_tensors)
258
259
260 +def input_conditioner(x, norm_mean, norm_std):
261 + return (x - norm_mean) / norm_std
262 +
263 +
264 +def calculate_timestamps(
265 + indices: list[int] | torch.Tensor,
266 + frame_duration_ms: int,
267 +):
268 + if not isinstance(indices, list):
269 + indices = indices.tolist()
270 +
271 + timestamps = [int(i) * frame_duration_ms / 1000.0 for i in indices]
272 + return timestamps
273 +
274 +
275 class BaseNanoNemotronVLProcessor(ABC):
276 """
277 This model doesn't define its own HF processor,
@@ -344,17 +365,30 @@ class BaseNanoNemotronVLProcessor(ABC):
365 else:
366 pixel_values_lst = self._images_to_pixel_values_lst(images, max_num_tiles)
367 image_inputs = {
347 - "pixel_values_flat": torch.cat(pixel_values_lst),
368 + "pixel_values_flat": input_conditioner(
369 + torch.cat(pixel_values_lst), self.norm_mean, self.norm_std
370 + ),
371 "image_num_patches": torch.tensor(
372 [len(item) for item in pixel_values_lst]
373 ),
374 }
375
353 - for pixel_values in pixel_values_lst:
376 + assert len(text) == 1, (
377 + "hf_processor is called on the output of get_dummy_text, "
378 + "which should be a single string"
379 + )
380 + parts = [x for x in re.split(r"(<image>)", text[0]) if x]
381 + assert parts.count("<image>") == len(pixel_values_lst), (
382 + "the number of <image> tokens in the text should be the "
383 + "same as the number of images"
384 + )
385 +
386 + for i, pixel_values in enumerate(pixel_values_lst):
387 num_patches = pixel_values.shape[0]
388 feature_size = num_patches * self.num_image_token
389 image_repl = self.get_image_repl(feature_size, num_patches)
357 - text = [t.replace("<image>", image_repl.full, 1) for t in text]
390 + parts[i] = parts[i].replace("<image>", image_repl.full)
391 + text = ["".join(parts)]
392 return text, image_inputs
393
394 def _make_batch_input(self, input_item: Any | list[Any] | None = None):
@@ -421,6 +455,18 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
455 self.video_token = video_token
456 self.video_pruning_rate = video_pruning_rate
457
458 + # Pre-tokenize special tokens for video processing
459 + # to avoid repeated tokenization
460 + self._img_start_token_ids = encode_tokens(
461 + tokenizer, IMG_START, add_special_tokens=False
462 + )
463 + self._img_end_token_ids = encode_tokens(
464 + tokenizer, IMG_END, add_special_tokens=False
465 + )
466 + self._img_context_token_ids = encode_tokens(
467 + tokenizer, IMG_CONTEXT, add_special_tokens=False
468 + )
469 +
470 @property
471 def supports_video(self) -> bool:
472 return self.video_token_id is not None
@@ -454,24 +500,43 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
500 def _preprocess_video(
501 self,
502 text: list[str],
457 - videos: list[npt.NDArray],
503 + videos: list[tuple[npt.NDArray, dict[str, Any]]],
504 max_num_tiles: int,
505 dynamic_image_size: bool | None = None,
506 ):
507 if len(videos) == 0 or not self.supports_video:
508 video_inputs = {}
509 else:
510 + videos_lst = [v[0] for v in videos]
511 + video_metadata_lst = [v[1] for v in videos]
512 pixel_values_lst_video = self._videos_to_pixel_values_lst(
465 - videos,
513 + videos_lst,
514 max_num_tiles=max_num_tiles,
515 dynamic_image_size=dynamic_image_size,
516 )
517
518 + # We use frame duration in milliseconds (as integer) to ensure
519 + # we have consistent timestamps calculation. At preprocessing
520 + # fps parameter is given in fp32, while at inference it is bf16
521 + # which leads to inaccurate timestamp calculation and causes
522 + # timestamp values to differ.In rare cases this causes
523 + # mismatching number of output tokens for tokenized frame prefixes
524 + frame_duration_ms_lst = [
525 + int(1000.0 / metadata["fps"]) for metadata in video_metadata_lst
526 + ]
527 + frames_indices_lst = [
528 + metadata["frames_indices"] for metadata in video_metadata_lst
529 + ]
530 +
531 video_inputs = {
471 - "pixel_values_flat_video": torch.cat(pixel_values_lst_video),
532 + "pixel_values_flat_video": input_conditioner(
533 + torch.cat(pixel_values_lst_video), self.norm_mean, self.norm_std
534 + ),
535 "video_num_patches": torch.tensor(
536 [len(item) for item in pixel_values_lst_video]
537 ),
538 + "frames_indices": frames_indices_lst,
539 + "frame_duration_ms": torch.tensor(frame_duration_ms_lst),
540 }
541
542 image_size: int = self.config.force_image_size
@@ -481,7 +546,12 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
546 (image_size * image_size // patch_size**2) * (downsample_ratio**2)
547 )
548
484 - for pixel_values in pixel_values_lst_video:
549 + for pixel_values, video_metadata, frames_indices, frame_duration_ms in zip(
550 + pixel_values_lst_video,
551 + video_metadata_lst,
552 + frames_indices_lst,
553 + frame_duration_ms_lst,
554 + ):
555 num_frames = pixel_values.shape[0]
556
557 if (
@@ -504,16 +574,29 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
574 else:
575 tokens_per_frame = [tokens_in_single_frame] * num_frames
576
507 - video_repl = self.get_video_repl(tokens_per_frame, self.video_token)
577 + video_repl = self.get_video_repl(
578 + tokens_per_frame=tokens_per_frame,
579 + frames_indices=frames_indices,
580 + frame_duration_ms=frame_duration_ms,
581 + tokenizer=self.tokenizer,
582 + img_start_token_ids=self._img_start_token_ids,
583 + img_end_token_ids=self._img_end_token_ids,
584 + img_context_token_ids=self._img_context_token_ids,
585 + )
586
509 - text = [t.replace("<video>", video_repl.full, 1) for t in text]
587 + # video_repl.full is a list of token IDs
588 + # Convert token IDs back to text for the HF processor flow
589 + video_repl_text = self.tokenizer.decode(
590 + video_repl.full, skip_special_tokens=False
591 + )
592 + text = [t.replace("<video>", video_repl_text, 1) for t in text]
593 return text, video_inputs
594
595 def __call__(
596 self,
597 text: str | list[str] | None = None,
598 images: Image.Image | list[Image.Image] | None = None,
516 - videos: npt.NDArray | list[npt.NDArray] | None = None,
599 + videos: list[tuple[npt.NDArray, dict[str, Any]]] | None = None,
600 return_tensors: str | TensorType | None = None,
601 max_num_tiles: int | None = None,
602 dynamic_image_size: bool | None = None,
@@ -558,9 +641,15 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
641 @classmethod
642 def get_video_repl(
643 cls,
644 + *,
645 tokens_per_frame: list[int],
562 - video_context_token: str = IMG_CONTEXT,
563 - ) -> PromptUpdateDetails[str]:
646 + frames_indices: list[int],
647 + frame_duration_ms: int,
648 + tokenizer: AnyTokenizer,
649 + img_start_token_ids: list[int],
650 + img_end_token_ids: list[int],
651 + img_context_token_ids: list[int],
652 + ) -> PromptUpdateDetails[list[int]]:
653 """
654 Build prompt replacement for a video.
655 The replacement returned is not actually used to replace the placeholder
@@ -579,16 +668,52 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
668 - EVS real (called from get_real_video_repl_for_evs) - different value per frame
669 Args:
670 tokens_per_frame (list[int]): number of tokens per frame
582 - video_context_token (str): the token to use for the video context
671 + frames_indices (list[int]): frame indices
672 + frame_duration_ms (int): duration of each frame in milliseconds
673 + tokenizer (AnyTokenizer): tokenizer to use for tokenizing frame separators
674 + img_start_token_ids (list[int]): pre-tokenized IMG_START tokens
675 + img_end_token_ids (list[int]): pre-tokenized IMG_END tokens
676 + img_context_token_ids (list[int]): pre-tokenized IMG_CONTEXT tokens
677 """
584 - repl_full = "".join(
585 - [
586 - f"Frame{i + 1}: {IMG_START}{video_context_token * num_tokens}{IMG_END}"
587 - for i, num_tokens in enumerate(tokens_per_frame)
678 + # TODO: Add support of frame_duration_ms to be None
679 + # At preprocessing step we should allow absent / metadata without
680 + # frames_indices field.
681 + timestamps_enabled = frame_duration_ms is not None
682 +
683 + if timestamps_enabled:
684 + timestamps = calculate_timestamps(frames_indices, frame_duration_ms)
685 +
686 + assert len(timestamps) == len(tokens_per_frame), (
687 + "timestamps and tokens_per_frame must have the same length"
688 + )
689 + frame_separators = [
690 + f"Frame {i + 1} sampled at {timestamp:.2f} seconds: "
691 + for i, timestamp in enumerate(timestamps)
692 ]
589 - )
693 + else:
694 + frame_separators = [
695 + f"Frame {i + 1}: " for i, _ in enumerate(tokens_per_frame)
696 + ]
697 +
698 + # Tokenize frame separator independently
699 + frame_separators_tokenized = [
700 + _seq2tokens(tokenizer, sep) for sep in frame_separators
701 + ]
702
591 - return PromptUpdateDetails.from_seq(repl_full)
703 + # Tokenize each component independently to avoid tokenizer merging tokens
704 + # across boundaries. This ensures consistent tokenization regardless of
705 + # num_tokens_per_frame values.
706 + all_token_ids = []
707 + for i, num_tokens in enumerate(tokens_per_frame):
708 + frame_sep_token_ids = frame_separators_tokenized[i]
709 + all_token_ids.extend(frame_sep_token_ids)
710 +
711 + # Add pre-tokenized special tokens
712 + all_token_ids.extend(img_start_token_ids)
713 + all_token_ids.extend(img_context_token_ids * num_tokens)
714 + all_token_ids.extend(img_end_token_ids)
715 +
716 + return PromptUpdateDetails.from_seq(all_token_ids)
717
718
719 class BaseNanoNemotronVLProcessingInfo(BaseProcessingInfo):
@@ -695,8 +820,6 @@ class NanoNemotronVLProcessingInfo(BaseNanoNemotronVLProcessingInfo):
820 max_image_tokens = self.get_max_image_tokens() * max_images
821 max_total_frames = (seq_len - max_image_tokens) // processor.num_image_token
822 max_frames_per_video = max_total_frames // max(max_videos, 1)
698 -
699 - max_frames_per_video = min(max_frames_per_video, MAX_FRAMES)
823 return max(max_frames_per_video, 1)
824
825 def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
@@ -791,6 +914,9 @@ class NanoNemotronVLMultiModalProcessor(
914 ):
915 """MultiModalProcessor extended for video support"""
916
917 + def _get_data_parser(self) -> MultiModalDataParser:
918 + return MultiModalDataParser(video_needs_metadata=True)
919 +
920 def _get_mm_fields_config(
921 self,
922 hf_inputs: BatchFeature,
@@ -805,6 +931,8 @@ class NanoNemotronVLMultiModalProcessor(
931 "video", video_num_patches
932 ),
933 video_num_patches=MultiModalFieldConfig.batched("video"),
934 + frames_indices=MultiModalFieldConfig.batched("video"),
935 + frame_duration_ms=MultiModalFieldConfig.batched("video"),
936 )
937 else:
938 video_fields = {}
@@ -835,6 +963,7 @@ class NanoNemotronVLMultiModalProcessor(
963
964 def get_video_replacement_internvl(item_idx: int):
965 feature_size = hf_processor.num_image_token
966 + video, metadata = mm_items["video"][item_idx]
967 num_patches = video_num_patches[item_idx]
968 if num_patches is not None:
969 assert isinstance(num_patches, int)
@@ -856,9 +985,15 @@ class NanoNemotronVLMultiModalProcessor(
985 else:
986 tokens_per_frame = [feature_size] * num_patches
987
988 + frame_duration_ms = int(1000 / metadata["fps"])
989 return hf_processor.get_video_repl(
860 - tokens_per_frame,
861 - video_context_token=hf_processor.video_token,
990 + tokens_per_frame=tokens_per_frame,
991 + frames_indices=metadata["frames_indices"],
992 + frame_duration_ms=frame_duration_ms,
993 + tokenizer=hf_processor.tokenizer,
994 + img_start_token_ids=hf_processor._img_start_token_ids,
995 + img_end_token_ids=hf_processor._img_end_token_ids,
996 + img_context_token_ids=hf_processor._img_context_token_ids,
997 )
998
999 if self.info.supports_video:
@@ -917,6 +1052,37 @@ class NanoNemotronVLDummyInputsBuilder(
1052
1053 return super().get_dummy_text(mm_counts) + "<video>" * num_videos
1054
1055 + def _get_dummy_videos(
1056 + self,
1057 + *,
1058 + width: int,
1059 + height: int,
1060 + num_frames: int,
1061 + num_videos: int,
1062 + overrides: VideoDummyOptions | None = None,
1063 + ) -> list[VideoItem]:
1064 + video = super()._get_dummy_videos(
1065 + width=width,
1066 + height=height,
1067 + num_frames=num_frames,
1068 + num_videos=1,
1069 + overrides=overrides,
1070 + )[0]
1071 + video_items = []
1072 + for _ in range(num_videos):
1073 + video_metadata = {
1074 + "total_num_frames": num_frames,
1075 + "fps": 2,
1076 + "duration": num_frames / 2.0,
1077 + "video_backend": "opencv_dynamic",
1078 + "frames_indices": [i for i in range(num_frames)],
1079 + "do_sample_frames": False,
1080 + }
1081 + video_item = (video.copy(), video_metadata)
1082 + video_items.append(video_item)
1083 +
1084 + return video_items
1085 +
1086 def get_dummy_mm_data(
1087 self,
1088 seq_len: int,
@@ -1013,6 +1179,19 @@ class NemotronH_Nano_VL_V2(
1179 self.config = config
1180 self.model_config = vllm_config.model_config
1181
1182 + # Pre-tokenize special tokens for video processing
1183 + # to avoid repeated tokenization
1184 + tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
1185 + self._img_start_token_ids = encode_tokens(
1186 + tokenizer, IMG_START, add_special_tokens=False
1187 + )
1188 + self._img_end_token_ids = encode_tokens(
1189 + tokenizer, IMG_END, add_special_tokens=False
1190 + )
1191 + self._img_context_token_ids = encode_tokens(
1192 + tokenizer, IMG_CONTEXT, add_special_tokens=False
1193 + )
1194 +
1195 def pixel_shuffle(self, x, scale_factor=0.5):
1196 n, w, h, c = x.size()
1197 # N, W, H, C --> N, W, H * scale, C // scale
@@ -1043,13 +1222,28 @@ class NemotronH_Nano_VL_V2(
1222 return x
1223
1224 def extract_feature(self, pixel_values):
1046 - vit_embeds = self.vision_model(pixel_values)
1047 - vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
1048 - h = w = int(vit_embeds.shape[1] ** 0.5)
1049 - vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
1050 - vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
1051 - vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
1052 - vit_embeds = self.mlp1(vit_embeds)
1225 + # Process images in a micro-batch of at most 128 frames per call
1226 + # This is done on purpose to ensure peak GPU ram usage of huge batch
1227 + # (namely for really long videos with EVS ON) won't cause any problems
1228 + # as we don't support chunked prefill for video media
1229 + micro_batch_size = 128
1230 + n = pixel_values.shape[0]
1231 + vit_embeds_list = []
1232 + for i in range(0, n, micro_batch_size):
1233 + vit_embeds = self.vision_model(pixel_values[i : i + micro_batch_size])
1234 + vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
1235 + h = w = int(vit_embeds.shape[1] ** 0.5)
1236 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
1237 + vit_embeds = self.pixel_shuffle(
1238 + vit_embeds, scale_factor=self.downsample_ratio
1239 + )
1240 + vit_embeds = vit_embeds.reshape(
1241 + vit_embeds.shape[0], -1, vit_embeds.shape[-1]
1242 + )
1243 + vit_embeds = self.mlp1(vit_embeds)
1244 + vit_embeds_list.append(vit_embeds)
1245 +
1246 + vit_embeds = torch.cat(vit_embeds_list, dim=0)
1247 return vit_embeds
1248
1249 def _parse_and_validate_image_input(
@@ -1117,12 +1311,15 @@ class NemotronH_Nano_VL_V2(
1311 rows = int(image_rows * downsample_ratio // patch_size)
1312 cols = int(image_cols * downsample_ratio // patch_size)
1313 video_pruning_rate = self.video_pruning_rate
1120 -
1314 + video_num_frames = video_input["num_patches"].tolist()
1315 + video_frames_indices = video_input["frames_indices"].split(video_num_frames)
1316 # Calculate video feature dimensions (number of frames and
1317 # their feature size (AKA tokens per frame))
1318 # TODO: Maybe this can be optimized to avoid the loop?
1319 for i, single_video_embeddings in enumerate(video_embeddings):
1125 - num_frames = video_input["num_patches"][i].item()
1320 + num_frames = video_num_frames[i]
1321 + frames_indices = video_frames_indices[i].tolist()
1322 + frame_duration_ms = video_input["frame_duration_ms"][i].item()
1323 assert single_video_embeddings.shape[0] % num_frames == 0
1324
1325 if video_pruning_rate is not None and video_pruning_rate > 0.0:
@@ -1151,6 +1348,8 @@ class NemotronH_Nano_VL_V2(
1348 self._create_final_video_embeddings(
1349 single_video_embeddings,
1350 num_tokens_per_frame,
1351 + frames_indices,
1352 + frame_duration_ms,
1353 ),
1354 )
1355
@@ -1160,6 +1359,8 @@ class NemotronH_Nano_VL_V2(
1359 self,
1360 video_embeddings: torch.Tensor,
1361 num_tokens_per_frame: list[int],
1362 + frames_indices: list[int],
1363 + frame_duration_ms: int,
1364 ) -> torch.Tensor:
1365 """Create final embeddings that combine video embeddings with
1366 text embeddings of indicator tokens.
@@ -1173,23 +1374,28 @@ class NemotronH_Nano_VL_V2(
1374 input_embeds for the LLM.
1375 """
1376 device = video_embeddings.device
1176 -
1177 - # Generate video replacement text and convert to token IDs
1178 - video_repl_text = NanoNemotronVLProcessor.get_video_repl(
1179 - num_tokens_per_frame,
1180 - IMG_CONTEXT,
1181 - ).full
1182 -
1377 tokenizer = cached_tokenizer_from_config(self.model_config)
1184 - repl_token_ids = torch.tensor(
1185 - _seq2tokens(tokenizer, video_repl_text), device=device
1186 - )
1378
1188 - # Get embedding token IDs for image context
1189 - embed_token_ids = torch.tensor(
1190 - encode_tokens(tokenizer, IMG_CONTEXT), device=device
1379 + # Generate video replacement token IDs using get_video_repl
1380 + # This tokenizes each frame separator independently, then uses pre-tokenized
1381 + # special tokens to ensure consistent tokenization regardless of
1382 + # num_tokens_per_frame values.
1383 + video_repl = NanoNemotronVLProcessor.get_video_repl(
1384 + tokens_per_frame=num_tokens_per_frame,
1385 + frames_indices=frames_indices,
1386 + frame_duration_ms=frame_duration_ms,
1387 + tokenizer=tokenizer,
1388 + img_start_token_ids=self._img_start_token_ids,
1389 + img_end_token_ids=self._img_end_token_ids,
1390 + img_context_token_ids=self._img_context_token_ids,
1391 )
1392
1393 + # video_repl.full is a list of token IDs
1394 + repl_token_ids = torch.tensor(video_repl.full, device=device)
1395 +
1396 + # Get embedding token IDs for image context (use pre-tokenized version)
1397 + embed_token_ids = torch.tensor(self._img_context_token_ids, device=device)
1398 +
1399 # Create mask for video embedding positions
1400 is_video_embed = torch.isin(repl_token_ids, embed_token_ids)
1401
@@ -1210,6 +1416,8 @@ class NemotronH_Nano_VL_V2(
1416 pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
1417 video_num_patches = kwargs.pop("video_num_patches", None)
1418 video_embeds = kwargs.pop("video_embeds", None)
1419 + frames_indices = kwargs.pop("frames_indices", None)
1420 + frame_duration_ms = kwargs.pop("frame_duration_ms", None)
1421
1422 if pixel_values_flat_video is None and video_embeds is None:
1423 return None
@@ -1221,13 +1429,22 @@ class NemotronH_Nano_VL_V2(
1429 )
1430
1431 if pixel_values_flat_video is not None:
1432 + if torch.is_tensor(frames_indices):
1433 + frames_indices = frames_indices.flatten()
1434 + else:
1435 + frames_indices = torch.cat([f.flatten() for f in frames_indices], dim=0)
1436 +
1437 + frame_duration_ms = frame_duration_ms.flatten()
1438 expected_h = expected_w = self.config.force_image_size
1225 - resolve_bindings = {"h": expected_h, "w": expected_w}
1439 + num_frames = video_num_patches[0].item()
1440 + resolve_bindings = {"h": expected_h, "w": expected_w, "f": num_frames}
1441
1442 return NanoNemotronVLVideoPixelInputs(
1443 type="pixel_values_videos",
1444 pixel_values_flat=pixel_values_flat_video,
1445 num_patches=video_num_patches,
1446 + frames_indices=frames_indices,
1447 + frame_duration_ms=frame_duration_ms,
1448 resolve_bindings=resolve_bindings,
1449 )
1450
vllm/model_executor/models/radio.py
+5 -33
@@ -43,32 +43,6 @@ to_4tuple = _ntuple(4)
43 to_ntuple = _ntuple
44
45
46 -class InputConditioner(nn.Module):
47 - def __init__(
48 - self,
49 - input_scale: float,
50 - norm_mean: norm_t,
51 - norm_std: norm_t,
52 - dtype: torch.dtype = None,
53 - ):
54 - super().__init__()
55 -
56 - self.dtype = dtype
57 -
58 - self.register_buffer("norm_mean", _to_tensor(norm_mean) / input_scale)
59 - self.register_buffer("norm_std", _to_tensor(norm_std) / input_scale)
60 -
61 - def forward(self, x: torch.Tensor):
62 - y = (x - self.norm_mean) / self.norm_std
63 - if self.dtype is not None:
64 - y = y.to(self.dtype)
65 - return y
66 -
67 -
68 -def _to_tensor(v: norm_t):
69 - return torch.as_tensor(v, dtype=torch.float32).view(-1, 1, 1)
70 -
71 -
46 class ClsToken(nn.Module):
47 def __init__(
48 self,
@@ -507,11 +481,6 @@ class RadioModel(nn.Module):
481 super().__init__()
482
483 self.config = config
510 - self.input_conditioner = InputConditioner(
511 - input_scale=1.0,
512 - norm_mean=config.norm_mean,
513 - norm_std=config.norm_std,
514 - )
484 self.model = RadioInternVisionModel(
485 config=config,
486 quant_config=quant_config,
@@ -525,8 +494,7 @@ class RadioModel(nn.Module):
494 pixel_values: torch.Tensor | None = None,
495 pixel_embeds: torch.Tensor | None = None,
496 ) -> torch.FloatTensor:
528 - x = self.input_conditioner(pixel_values)
529 - y = self.model(x)
497 + y = self.model(pixel_values)
498 return self._extract_final(y)
499
500 def load_weights(self, weights) -> set[str]:
@@ -548,6 +516,10 @@ class RadioModel(nn.Module):
516 # Skip buffers not used in vLLM
517 if sub in {"summary_idxs"}:
518 continue
519 + if sub.startswith("input_conditioner."):
520 + # we normalize in the input processor,
521 + # based on norm and std values from the config
522 + continue
523
524 vllm_key = None
525 if sub.startswith("model.patch_generator."):
vllm/multimodal/profiling.py
+1 -1
@@ -223,7 +223,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
223 height,
224 )
225 height = min(height, overrides.height)
226 - video = np.full((num_frames, width, height, 3), 255)
226 + video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)
227 return [video] * num_videos
228
229
vllm/multimodal/video.py
+23 -13
@@ -13,10 +13,13 @@ import numpy.typing as npt
13 from PIL import Image
14
15 from vllm import envs
16 +from vllm.logger import init_logger
17
18 from .base import MediaIO
19 from .image import ImageMediaIO
20
21 +logger = init_logger(__name__)
22 +
23
24 def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
25 num_frames, _, _, channels = frames.shape
@@ -103,6 +106,7 @@ class OpenCVVideoBackend(VideoLoader):
106 cls,
107 data: bytes,
108 num_frames: int = -1,
109 + fps: int = -1,
110 **kwargs,
111 ) -> tuple[npt.NDArray, dict[str, Any]]:
112 import cv2
@@ -116,14 +120,20 @@ class OpenCVVideoBackend(VideoLoader):
120 original_fps = cap.get(cv2.CAP_PROP_FPS)
121 duration = total_frames_num / original_fps if original_fps > 0 else 0
122
119 - # resample video to target num_frames
120 - full_read = num_frames == -1 or total_frames_num < num_frames
121 - if full_read:
122 - num_frames = total_frames_num
123 - frame_idx = list(range(0, num_frames))
123 + # resample video to target num_frames and fps
124 + # - the minimum of the two will be used
125 + num_frames_to_sample = total_frames_num
126 + if num_frames > 0:
127 + num_frames_to_sample = min(num_frames, total_frames_num)
128 + if fps > 0:
129 + num_frames_to_sample = min(num_frames_to_sample, math.floor(duration * fps))
130 + num_frames_to_sample = max(1, num_frames_to_sample) # at least one sample
131 +
132 + if num_frames_to_sample == total_frames_num:
133 + frame_idx = list(range(0, num_frames_to_sample))
134 else:
135 uniform_sampled_frames = np.linspace(
126 - 0, total_frames_num - 1, num_frames, dtype=int
136 + 0, total_frames_num - 1, num_frames_to_sample, dtype=int
137 )
138 frame_idx = uniform_sampled_frames.tolist()
139
@@ -132,7 +142,7 @@ class OpenCVVideoBackend(VideoLoader):
142 frames = np.empty((len(frame_idx), height, width, 3), dtype=np.uint8)
143
144 i = 0
135 - for idx in range(total_frames_num):
145 + for idx in range(max(frame_idx) + 1):
146 ok = cap.grab()
147 if not ok:
148 break
@@ -142,8 +152,8 @@ class OpenCVVideoBackend(VideoLoader):
152 frames[i] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
153 i += 1
154
145 - assert i == num_frames, (
146 - f"Expected reading {num_frames} frames, "
155 + assert i == num_frames_to_sample, (
156 + f"Expected reading {num_frames_to_sample} frames, "
157 f"but only loaded {i} frames from video."
158 )
159
@@ -151,14 +161,14 @@ class OpenCVVideoBackend(VideoLoader):
161 # NOTE(Isotr0py): For models like Qwen3-VL/GLM4.5V, this metadata
162 # can cause incorrect timestamp calculation without num_frames=-1.
163 metadata = {
154 - "total_num_frames": num_frames,
155 - "fps": num_frames / duration,
164 + "total_num_frames": total_frames_num,
165 + "fps": original_fps,
166 "duration": duration,
167 "video_backend": "opencv",
158 - "frames_indices": list(range(num_frames)),
168 + "frames_indices": list(frame_idx),
169 # extra field used to control hf processor's video
170 # sampling behavior
161 - "do_sample_frames": num_frames == total_frames_num,
171 + "do_sample_frames": num_frames_to_sample == total_frames_num,
172 }
173
174 return frames, metadata
vllm/v1/worker/gpu_model_runner.py
+21 -9
@@ -1735,20 +1735,32 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1735 pin_memory=self.pin_memory,
1736 merge_by_field_config=model.merge_by_field_config,
1737 ):
1738 + curr_group_outputs = []
1739 +
1740 + # EVS-related change.
1741 # (ekhvedchenia): Temporary hack to limit peak memory usage when
1739 - # processing multimodal data.This solves the issue with scheduler
1742 + # processing multimodal data. This solves the issue with scheduler
1743 # putting too many video samples into a single batch. Scheduler
1744 # uses pruned vision tokens count to compare it versus compute
1745 # budget which is incorrect (Either input media size or non-pruned
1746 # output vision tokens count should be considered)
1744 - curr_group_outputs = []
1745 -
1746 - if self.is_multimodal_pruning_enabled and modality == "video":
1747 - micro_batch_size = 1
1748 - for i in range(0, num_items, micro_batch_size):
1749 - micro_batch_mm_inputs = dict(
1750 - (k, v[i : i + micro_batch_size])
1751 - for k, v in mm_kwargs_group.items()
1747 + # TODO(ywang96): Fix memory profiling to take EVS into account and
1748 + # remove this hack.
1749 + if (
1750 + self.is_multimodal_pruning_enabled
1751 + and modality == "video"
1752 + and num_items > 1
1753 + ):
1754 + for video_mm_kwargs_item in filter(
1755 + lambda item: item.modality == "video", mm_kwargs
1756 + ):
1757 + _, _, micro_batch_mm_inputs = next(
1758 + group_mm_kwargs_by_modality(
1759 + [video_mm_kwargs_item],
1760 + device=self.device,
1761 + pin_memory=self.pin_memory,
1762 + merge_by_field_config=model.merge_by_field_config,
1763 + )
1764 )
1765
1766 micro_batch_outputs = model.get_multimodal_embeddings(