A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 12:15 UTC
nano-nemotron-vl: get_mm_max_tokens_per_item for audio, video, image == seq_len (#38727)
Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
Netanel Haber committed
Apr 7, 2026 at 10:23 UTC
a9a0e0551f038d7306cf8887069264d917c643a5
1 file changed
+52
-10
vllm/model_executor/models/nano_nemotron_vl.py
+52
-10
@@ -288,6 +288,35 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
288
max_num_tiles=max_num_tiles,
289
)
290
291
+ def get_dummy_image_size_and_max_tokens(
292
+ self, mm_counts: Mapping[str, int]
293
+ ) -> tuple[tuple[int, int], int]:
294
+ processor = self.get_hf_processor()
295
+ num_images = mm_counts.get("image", 0)
296
+
297
+ if tiler := processor.dynamic_tiler:
298
+ budget = tiler.max_num_tokens_available(text_prompt_length=num_images)
299
+ target_width, target_height = (
300
+ tiler.width_and_height_for_max_num_tokens_available(budget)
301
+ )
302
+ return (
303
+ (target_width, target_height),
304
+ tiler._get_num_embeddings(target_width, target_height),
305
+ )
306
+
307
+ max_num_tiles = processor.max_num_tiles
308
+ target_width, target_height = self.get_image_size_with_most_features(
309
+ max_num_tiles
310
+ )
311
+ return (
312
+ (target_width, target_height),
313
+ processor.get_num_image_tokens(
314
+ image_width=target_width,
315
+ image_height=target_height,
316
+ max_num_tiles=max_num_tiles,
317
+ ),
318
+ )
319
+
320
def get_num_frames_with_most_features(
321
self,
322
seq_len: int,
@@ -306,6 +335,26 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
335
max_frames_per_video = max_tubelets_per_video * T
336
return max(max_frames_per_video, 1)
337
338
+ def get_mm_max_tokens_per_item(
339
+ self, seq_len: int, mm_counts: Mapping[str, int]
340
+ ) -> Mapping[str, int]:
341
+ mm_max_tokens: dict[str, int] = {}
342
+
343
+ if mm_counts.get("image", 0) > 0:
344
+ _, mm_max_tokens["image"] = self.get_dummy_image_size_and_max_tokens(
345
+ mm_counts
346
+ )
347
+
348
+ if mm_counts.get("video", 0) > 0:
349
+ assert self.supports_video
350
+ mm_max_tokens["video"] = seq_len
351
+
352
+ if mm_counts.get("audio", 0) > 0:
353
+ assert self.supports_audio
354
+ mm_max_tokens["audio"] = seq_len
355
+
356
+ return mm_max_tokens
357
+
358
359
class NanoNemotronVLMultiModalProcessor(
360
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
@@ -708,17 +757,10 @@ class NanoNemotronVLDummyInputsBuilder(
757
mm_options: Mapping[str, BaseDummyOptions],
758
) -> MultiModalDataDict:
759
num_images = mm_counts.get("image", 0)
760
+ (target_width, target_height), _ = (
761
+ self.info.get_dummy_image_size_and_max_tokens(mm_counts)
762
+ )
763
processor = self.info.get_hf_processor()
712
- if tiler := processor.dynamic_tiler:
713
- budget = tiler.max_num_tokens_available(text_prompt_length=num_images)
714
- target_width, target_height = (
715
- tiler.width_and_height_for_max_num_tokens_available(budget)
716
- )
717
- else:
718
- max_num_tiles = 12
719
- target_width, target_height = self.info.get_image_size_with_most_features(
720
- max_num_tiles
721
- )
764
765
image_overrides = mm_options.get("image")
766