@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ef00b5cac
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 01:00 UTC

[VLM] Add Nemotron-Nano-VL-8B-V1 support (#20349)

Signed-off-by: Kyle Huang <kylhuang@nvidia.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

kYLe committed Jul 17, 2025 at 05:07 UTC 4ef00b5caca4274ef53629ad069eefe770cb156d
11 files changed +701 -3
docker/Dockerfile.cpu
+1 -1
@@ -95,7 +95,7 @@ WORKDIR /workspace/vllm
95 RUN --mount=type=bind,src=requirements/test.in,target=requirements/test.in \
96 cp requirements/test.in requirements/cpu-test.in && \
97 sed -i '/mamba_ssm/d' requirements/cpu-test.in && \
98 - sed -i 's/torch==.*/torch==2.6.0/g' requirements/cpu-test.in && \
98 + sed -i 's/^torch==.*/torch==2.6.0/g' requirements/cpu-test.in && \
99 sed -i 's/torchaudio.*/torchaudio/g' requirements/cpu-test.in && \
100 sed -i 's/torchvision.*/torchvision/g' requirements/cpu-test.in && \
101 uv pip compile requirements/cpu-test.in -o requirements/cpu-test.txt --index-strategy unsafe-best-match --torch-backend cpu
docs/models/supported_models.md
+1
@@ -581,6 +581,7 @@ Specified using `--task generate`.
581 | `KeyeForConditionalGeneration` | Keye-VL-8B-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `Kwai-Keye/Keye-VL-8B-Preview` | | | ✅︎ |
582 | `KimiVLForConditionalGeneration` | Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking | T + I<sup>+</sup> | `moonshotai/Kimi-VL-A3B-Instruct`, `moonshotai/Kimi-VL-A3B-Thinking` | | | ✅︎ |
583 | `Llama4ForConditionalGeneration` | Llama 4 | T + I<sup>+</sup> | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | | ✅︎ | ✅︎ |
584 +| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ | ✅︎ |
585 | `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `TIGER-Lab/Mantis-8B-siglip-llama3` (see note), `mistral-community/pixtral-12b`, etc. | | ✅︎ | ✅︎ |
586 | `LlavaNextForConditionalGeneration` | LLaVA-NeXT | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, etc. | | ✅︎ | ✅︎ |
587 | `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ | ✅︎ |
examples/offline_inference/vision_language.py
+39
@@ -429,6 +429,44 @@ def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
429 )
430
431
432 +# Nemontron_VL
433 +def run_nemotron_vl(questions: list[str], modality: str) -> ModelRequestData:
434 + model_name = "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"
435 +
436 + engine_args = EngineArgs(
437 + model=model_name,
438 + trust_remote_code=True,
439 + max_model_len=8192,
440 + limit_mm_per_prompt={modality: 1},
441 + )
442 +
443 + assert modality == "image"
444 + placeholder = "<image>"
445 +
446 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
447 + messages = [
448 + [{"role": "user", "content": f"{placeholder}\n{question}"}]
449 + for question in questions
450 + ]
451 + prompts = tokenizer.apply_chat_template(
452 + messages, tokenize=False, add_generation_prompt=True
453 + )
454 +
455 + # Stop tokens for InternVL
456 + # models variants may have different stop tokens
457 + # please refer to the model card for the correct "stop words":
458 + # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
459 + stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
460 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
461 + stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
462 +
463 + return ModelRequestData(
464 + engine_args=engine_args,
465 + prompts=prompts,
466 + stop_token_ids=stop_token_ids,
467 + )
468 +
469 +
470 # Keye-VL
471 def run_keye_vl(questions: list[str], modality: str) -> ModelRequestData:
472 model_name = "Kwai-Keye/Keye-VL-8B-Preview"
@@ -1186,6 +1224,7 @@ model_example_map = {
1224 "h2ovl_chat": run_h2ovl,
1225 "idefics3": run_idefics3,
1226 "internvl_chat": run_internvl,
1227 + "nemotron_vl": run_nemotron_vl,
1228 "keye_vl": run_keye_vl,
1229 "kimi_vl": run_kimi_vl,
1230 "llava": run_llava,
requirements/test.in
+1
@@ -30,6 +30,7 @@ mamba_ssm # required for plamo2 test
30 matplotlib # required for qwen-vl test
31 mistral_common[opencv] >= 1.8.0 # required for voxtral test
32 num2words # required for smolvlm test
33 +open_clip_torch==2.32.0 # Required for nemotron_vl test
34 opencv-python-headless >= 4.11.0 # required for video test
35 datamodel_code_generator # required for minicpm3 test
36 lm-eval[api]==0.4.8 # required for model evaluation test
requirements/test.txt
+15 -1
@@ -174,6 +174,8 @@ fsspec==2024.9.0
174 # fastparquet
175 # huggingface-hub
176 # torch
177 +ftfy==6.3.1
178 + # via open-clip-torch
179 genai-perf==0.0.8
180 # via -r requirements/test.in
181 genson==1.3.0
@@ -208,6 +210,7 @@ huggingface-hub==0.33.0
210 # accelerate
211 # datasets
212 # evaluate
213 + # open-clip-torch
214 # peft
215 # sentence-transformers
216 # timm
@@ -414,6 +417,8 @@ nvidia-nvjitlink-cu12==12.8.61
417 # torch
418 nvidia-nvtx-cu12==12.8.55
419 # via torch
420 +open-clip-torch==2.32.0
421 + # via -r requirements/test.in
422 opencensus==0.11.4
423 # via ray
424 opencensus-context==0.1.3
@@ -615,6 +620,7 @@ referencing==0.35.1
620 regex==2024.9.11
621 # via
622 # nltk
623 + # open-clip-torch
624 # sacrebleu
625 # tiktoken
626 # transformers
@@ -665,6 +671,7 @@ sacrebleu==2.4.3
671 safetensors==0.4.5
672 # via
673 # accelerate
674 + # open-clip-torch
675 # peft
676 # timm
677 # transformers
@@ -753,7 +760,9 @@ tiktoken==0.7.0
760 # lm-eval
761 # mistral-common
762 timm==1.0.11
756 - # via -r requirements/test.in
763 + # via
764 + # -r requirements/test.in
765 + # open-clip-torch
766 tokenizers==0.21.1
767 # via
768 # -r requirements/test.in
@@ -772,6 +781,7 @@ torch==2.7.1+cu128
781 # lm-eval
782 # mamba-ssm
783 # mteb
784 + # open-clip-torch
785 # peft
786 # runai-model-streamer
787 # sentence-transformers
@@ -789,6 +799,7 @@ torchaudio==2.7.1+cu128
799 torchvision==0.22.1+cu128
800 # via
801 # -r requirements/test.in
802 + # open-clip-torch
803 # timm
804 tqdm==4.66.6
805 # via
@@ -798,6 +809,7 @@ tqdm==4.66.6
809 # lm-eval
810 # mteb
811 # nltk
812 + # open-clip-torch
813 # peft
814 # pqdm
815 # sentence-transformers
@@ -863,6 +875,8 @@ virtualenv==20.31.2
875 # via ray
876 vocos==0.1.0
877 # via -r requirements/test.in
878 +wcwidth==0.2.13
879 + # via ftfy
880 webcolors==24.11.1
881 # via jsonschema
882 werkzeug==3.1.3
tests/models/multimodal/processing/test_common.py
+1
@@ -291,6 +291,7 @@ def _test_processing_correctness_one(
291 "allenai/Molmo-7B-D-0924",
292 "allenai/Molmo-7B-O-0924",
293 "nvidia/NVLM-D-72B",
294 + "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
295 "AIDC-AI/Ovis1.6-Gemma2-9B",
296 "AIDC-AI/Ovis1.6-Llama3.2-3B",
297 "AIDC-AI/Ovis2-1B",
tests/models/multimodal/processing/test_nemotron_vl.py new
+134
@@ -0,0 +1,134 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for Nemotron-Nano-VL's multimodal preprocessing kwargs."""
4 +from collections.abc import Mapping
5 +from typing import Optional
6 +
7 +import pytest
8 +from PIL import Image
9 +from transformers import PretrainedConfig
10 +
11 +from vllm.multimodal import MULTIMODAL_REGISTRY
12 +from vllm.multimodal.image import rescale_image_size
13 +from vllm.multimodal.processing import BaseMultiModalProcessor
14 +
15 +from ....conftest import ImageTestAssets
16 +from ...utils import build_model_context
17 +
18 +
19 +def _get_expected_num_patches(
20 + config: PretrainedConfig,
21 + image: Image.Image,
22 + num_imgs: int,
23 + min_num: int,
24 + max_num: int,
25 +):
26 + from vllm.model_executor.models.internvl import (
27 + calculate_internvl_targets, get_internvl_target_ratios)
28 +
29 + width, height = image.size
30 +
31 + blocks, _, _ = calculate_internvl_targets(
32 + orig_width=width,
33 + orig_height=height,
34 + target_ratios=get_internvl_target_ratios(
35 + min_num,
36 + max_num,
37 + ),
38 + image_size=config.force_image_size,
39 + use_thumbnail=False,
40 + )
41 + expected_num_patches = blocks
42 +
43 + if config.use_thumbnail and expected_num_patches > 1:
44 + expected_num_patches += 1
45 +
46 + return expected_num_patches
47 +
48 +
49 +def _run_check(
50 + processor: BaseMultiModalProcessor,
51 + images: list[Image.Image],
52 + min_num: int,
53 + max_num: int,
54 + mm_processor_kwargs: Mapping[str, object],
55 +):
56 + tokenizer = processor.info.get_tokenizer()
57 + config = processor.info.get_hf_config()
58 + image_processor = processor.info.get_image_processor()
59 +
60 + config.use_thumbnail = image_processor.use_thumbnail
61 + prompt = "<image>" * len(images)
62 + mm_data = {"image": images}
63 +
64 + total_expected_num_patches = sum(
65 + _get_expected_num_patches(config, image, len(images), min_num, max_num)
66 + for image in images)
67 + print(total_expected_num_patches)
68 + processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
69 +
70 + # Ensure we have the right number of placeholders per num_crops size
71 + image_token_id = tokenizer.convert_tokens_to_ids("<image>")
72 + img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
73 + pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
74 + print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape)
75 + assert img_tok_count == 256 * total_expected_num_patches
76 + assert pixel_shape[0] == total_expected_num_patches
77 +
78 +
79 +@pytest.mark.parametrize("model_id",
80 + ["nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"])
81 +@pytest.mark.parametrize(
82 + "size_factors",
83 + [
84 + # Single-scale
85 + [1.0],
86 + # Single-scale, batched
87 + [1.0, 1.0, 1.0],
88 + # Multi-scale
89 + [0.25, 0.5, 1.0],
90 + [4.0, 2.0, 1.0],
91 + ],
92 +)
93 +@pytest.mark.parametrize(
94 + ("min_dynamic_patch", "max_dynamic_patch"),
95 + [(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
96 +)
97 +@pytest.mark.parametrize("dynamic_image_size", [True, False])
98 +@pytest.mark.parametrize("kwargs_on_init", [True, False])
99 +def test_processor_override(
100 + model_id: str,
101 + image_assets: ImageTestAssets,
102 + size_factors: list[int],
103 + min_dynamic_patch: int,
104 + max_dynamic_patch: int,
105 + dynamic_image_size: Optional[bool],
106 + kwargs_on_init: bool,
107 +):
108 + mm_processor_kwargs = {
109 + "min_dynamic_patch": min_dynamic_patch,
110 + "max_dynamic_patch": max_dynamic_patch,
111 + "dynamic_image_size": dynamic_image_size,
112 + }
113 +
114 + ctx = build_model_context(
115 + model_id,
116 + mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
117 + limit_mm_per_prompt={"image": len(size_factors)},
118 + )
119 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
120 + hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
121 +
122 + min_num = min_dynamic_patch if dynamic_image_size else 1
123 + max_num = max_dynamic_patch if dynamic_image_size else 1
124 +
125 + _run_check(
126 + processor,
127 + [
128 + rescale_image_size(image_assets[0].pil_image, f)
129 + for f in size_factors
130 + ],
131 + min_num,
132 + max_num,
133 + hf_processor_mm_kwargs,
134 + )
tests/models/registry.py
+2
@@ -401,6 +401,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
401 trust_remote_code=True),
402 "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B",
403 trust_remote_code=True),
404 + "Llama_Nemotron_Nano_VL" : _HfExamplesInfo("nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1", # noqa: E501
405 + trust_remote_code=True),
406 "PaliGemmaForConditionalGeneration": _HfExamplesInfo("google/paligemma-3b-mix-224", # noqa: E501
407 extras={"v2": "google/paligemma2-3b-ft-docci-448"}), # noqa: E501
408 "Phi3VForCausalLM": _HfExamplesInfo("microsoft/Phi-3-vision-128k-instruct",
vllm/model_executor/models/nemotron_vl.py new
+505
@@ -0,0 +1,505 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py
5 +# --------------------------------------------------------
6 +# InternVL
7 +# Copyright (c) 2023 OpenGVLab
8 +# Licensed under The MIT License [see LICENSE for details]
9 +# --------------------------------------------------------
10 +from abc import ABC
11 +from collections.abc import Iterable
12 +from typing import Optional
13 +
14 +import torch
15 +import torch.nn as nn
16 +from PIL import Image
17 +from transformers import AutoModel, PretrainedConfig
18 +from transformers.image_processing_utils_fast import BaseImageProcessorFast
19 +
20 +from vllm.config import VllmConfig
21 +from vllm.model_executor.layers.quantization import QuantizationConfig
22 +from vllm.model_executor.layers.quantization.awq import AWQConfig
23 +from vllm.model_executor.models.internvl import (
24 + BaseInternVLDummyInputsBuilder, BaseInternVLMultiModalProcessor,
25 + BaseInternVLProcessingInfo, InternVLImageEmbeddingInputs,
26 + InternVLImageInputs, InternVLImagePixelInputs, InternVLProcessor)
27 +from vllm.model_executor.models.module_mapping import MultiModelKeys
28 +from vllm.model_executor.sampling_metadata import SamplingMetadata
29 +from vllm.multimodal import MULTIMODAL_REGISTRY
30 +from vllm.multimodal.inputs import NestedTensors
31 +from vllm.multimodal.processing import PromptUpdateDetails
32 +from vllm.sequence import IntermediateTensors
33 +from vllm.transformers_utils.processor import (
34 + cached_image_processor_from_config)
35 +from vllm.transformers_utils.tokenizer import AnyTokenizer
36 +
37 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
38 + SupportsMultiModal, SupportsPP)
39 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
40 + maybe_prefix, merge_multimodal_embeddings)
41 +
42 +IMG_START = '<img>'
43 +IMG_END = '</img>'
44 +IMG_CONTEXT = '<image>'
45 +
46 +
47 +class NemotronVLProcessor(InternVLProcessor):
48 +
49 + def __init__(
50 + self,
51 + config: PretrainedConfig,
52 + tokenizer: AnyTokenizer,
53 + image_processor: BaseImageProcessorFast,
54 + *,
55 + min_dynamic_patch: Optional[int] = None,
56 + max_dynamic_patch: Optional[int] = None,
57 + dynamic_image_size: Optional[bool] = None,
58 + ) -> None:
59 + ABC.__init__(self)
60 + self.config = config
61 + self.tokenizer = tokenizer
62 + self.image_processor = image_processor
63 + image_size: int = config.force_image_size
64 + patch_size: int = config.patch_size
65 +
66 + if min_dynamic_patch is None:
67 + min_dynamic_patch = 1
68 + assert isinstance(min_dynamic_patch, int)
69 +
70 + if max_dynamic_patch is None:
71 + max_dynamic_patch = self.image_processor.max_num_tiles
72 + assert isinstance(max_dynamic_patch, int)
73 +
74 + if dynamic_image_size is None:
75 + dynamic_image_size = True
76 + assert isinstance(dynamic_image_size, bool)
77 +
78 + self.num_image_token = int(
79 + (image_size // patch_size)**2 * (config.downsample_ratio**2))
80 + self.image_size = image_size
81 + self.min_dynamic_patch = min_dynamic_patch
82 + self.max_dynamic_patch = max_dynamic_patch
83 + self.dynamic_image_size = dynamic_image_size
84 + self.use_thumbnail: bool = self.image_processor.use_thumbnail
85 +
86 + @property
87 + def image_token_id(self) -> int:
88 + return self.tokenizer.get_vocab()[IMG_CONTEXT]
89 +
90 + def _preprocess_image(
91 + self,
92 + text: list[str],
93 + images: list[Image.Image],
94 + min_dynamic_patch: Optional[int] = None,
95 + max_dynamic_patch: Optional[int] = None,
96 + dynamic_image_size: Optional[bool] = None,
97 + ) -> tuple[list[str], dict[str, torch.Tensor]]:
98 + if len(images) == 0:
99 + image_inputs = {}
100 + else:
101 + pixel_values_lst = self._images_to_pixel_values_lst(
102 + images,
103 + min_dynamic_patch=min_dynamic_patch,
104 + max_dynamic_patch=max_dynamic_patch,
105 + dynamic_image_size=dynamic_image_size,
106 + )
107 + image_inputs: dict[str, NestedTensors] = {
108 + "pixel_values_flat":
109 + torch.cat(pixel_values_lst),
110 + "image_num_patches":
111 + torch.tensor([len(item) for item in pixel_values_lst]),
112 + }
113 +
114 + for pixel_values in pixel_values_lst:
115 + num_patches = pixel_values.shape[0]
116 + feature_size = num_patches * self.num_image_token
117 + image_repl = self.get_image_repl(feature_size, num_patches)
118 + NVL_IMAGE_CONTEXT = image_repl.full.replace(
119 + "<image>", "<NVL_IMG_CONTEXT>")
120 + text = [
121 + t.replace('<image>', NVL_IMAGE_CONTEXT, 1) for t in text
122 + ]
123 + text = [t.replace("<NVL_IMG_CONTEXT>", IMG_CONTEXT) for t in text]
124 + return text, image_inputs
125 +
126 + def get_image_repl(
127 + self,
128 + feature_size: int,
129 + num_patches: Optional[int],
130 + ) -> PromptUpdateDetails[str]:
131 + repl_features = IMG_CONTEXT * feature_size
132 + repl_full = IMG_START + repl_features + IMG_END
133 +
134 + return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
135 +
136 +
137 +class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
138 + """Processing info for Nemotron VL models."""
139 +
140 + def get_hf_processor(
141 + self,
142 + *,
143 + min_dynamic_patch: Optional[int] = None,
144 + max_dynamic_patch: Optional[int] = None,
145 + dynamic_image_size: Optional[bool] = None,
146 + **kwargs: object,
147 + ) -> NemotronVLProcessor:
148 + if min_dynamic_patch is not None:
149 + kwargs["min_dynamic_patch"] = min_dynamic_patch
150 + if max_dynamic_patch is not None:
151 + kwargs["max_dynamic_patch"] = max_dynamic_patch
152 + if dynamic_image_size is not None:
153 + kwargs["dynamic_image_size"] = dynamic_image_size
154 +
155 + image_processor = self.get_image_processor()
156 + return self.ctx.init_processor(
157 + NemotronVLProcessor,
158 + config=self.get_hf_config(),
159 + tokenizer=self.get_tokenizer(),
160 + image_processor=image_processor,
161 + **kwargs,
162 + )
163 +
164 + def get_image_processor(
165 + self,
166 + **kwargs: object,
167 + ):
168 + return cached_image_processor_from_config(
169 + self.ctx.model_config,
170 + **kwargs,
171 + )
172 +
173 +
174 +@MULTIMODAL_REGISTRY.register_processor(
175 + BaseInternVLMultiModalProcessor[NemotronVLProcessingInfo],
176 + info=NemotronVLProcessingInfo,
177 + dummy_inputs=BaseInternVLDummyInputsBuilder[NemotronVLProcessingInfo])
178 +class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
179 + SupportsLoRA):
180 +
181 + @classmethod
182 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
183 + if modality.startswith("image"):
184 + return "<image>"
185 +
186 + raise ValueError("Only image modality is supported")
187 +
188 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
189 + super().__init__()
190 +
191 + config = vllm_config.model_config.hf_config
192 + quant_config = vllm_config.quant_config
193 + multimodal_config = vllm_config.model_config.multimodal_config
194 +
195 + self.config = config
196 + self.multimodal_config = multimodal_config
197 + self._patch_quant_config(config, quant_config)
198 +
199 + image_size = config.force_image_size or config.vision_config.image_size
200 + patch_size = config.vision_config.patch_size
201 + self.patch_size = patch_size
202 + self.num_image_token = int(
203 + (image_size // patch_size)**2 * (config.downsample_ratio**2))
204 + self.downsample_ratio = config.downsample_ratio
205 + self.ps_version = config.ps_version
206 +
207 + self.llm_arch_name = config.text_config.architectures[0]
208 + self.vision_model = self._init_vision_model(
209 + config,
210 + quant_config=quant_config,
211 + prefix=maybe_prefix(prefix, "vision_model"),
212 + )
213 +
214 + self.language_model = init_vllm_registered_model(
215 + vllm_config=vllm_config,
216 + hf_config=config.text_config,
217 + prefix=maybe_prefix(prefix, "language_model"),
218 + )
219 +
220 + self.mlp1 = self._init_mlp1(config)
221 +
222 + self.img_context_token_id = None
223 +
224 + self.visual_token_mask = None
225 + self.make_empty_intermediate_tensors = (
226 + self.language_model.make_empty_intermediate_tensors)
227 +
228 + def _patch_quant_config(self, config: PretrainedConfig,
229 + quant_config: QuantizationConfig):
230 + # the awq models from OpenGVLab missing `modules_to_not_convert`
231 + # patch the quant_config to add `modules_to_not_convert` back
232 + if isinstance(quant_config, AWQConfig):
233 + text_config = config.text_config
234 + llm_quant_config = getattr(text_config, "quantization_config",
235 + None)
236 + if (not quant_config.modules_to_not_convert) and \
237 + (llm_quant_config is not None):
238 + quant_config.modules_to_not_convert.append("vision_model")
239 +
240 + def _init_vision_model(
241 + self,
242 + config: PretrainedConfig,
243 + quant_config: Optional[QuantizationConfig],
244 + *,
245 + prefix: str,
246 + ):
247 + return AutoModel.from_config(config.vision_config,
248 + trust_remote_code=True)
249 +
250 + def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
251 + vit_hidden_size = config.vit_hidden_size
252 + vision_projection_hidden_size = config.projector_hidden_size
253 + llm_hidden_size = config.text_config.hidden_size
254 +
255 + return nn.Sequential(
256 + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2,
257 + bias=True),
258 + nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
259 + vision_projection_hidden_size,
260 + bias=True),
261 + nn.GELU(),
262 + nn.Linear(vision_projection_hidden_size, llm_hidden_size),
263 + )
264 +
265 + def pixel_shuffle(self, x, scale_factor=0.5):
266 + n, w, h, c = x.size()
267 + # N, W, H, C --> N, W, H * scale, C // scale
268 + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
269 + # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
270 + x = x.permute(0, 2, 1, 3).contiguous()
271 + x = x.view(n, int(h * scale_factor), int(w * scale_factor),
272 + int(c / (scale_factor * scale_factor)))
273 + if self.ps_version == 'v1':
274 + pass
275 + else:
276 + x = x.permute(0, 2, 1, 3).contiguous()
277 + return x
278 +
279 + def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
280 + # https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1/blob/main/modeling.py#L177
281 + vit_embeds = self.vision_model(x=pixel_values).features
282 + vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
283 +
284 + h = w = int(vit_embeds.shape[1]**0.5)
285 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
286 + vit_embeds = self.pixel_shuffle(vit_embeds,
287 + scale_factor=self.downsample_ratio)
288 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1,
289 + vit_embeds.shape[-1])
290 + vit_embeds = self.mlp1(vit_embeds)
291 + return vit_embeds
292 +
293 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
294 +
295 + #use force_image_size to get image_size
296 + h = w = self.config.force_image_size
297 + expected_dims = (3, h, w)
298 +
299 + def _validate_shape(d: torch.Tensor):
300 + actual_dims = tuple(d.shape)
301 +
302 + if actual_dims != expected_dims:
303 + expected_expr = str(expected_dims)
304 + raise ValueError(
305 + "The expected shape of pixel values per image per batch "
306 + f" per patch is {expected_expr}. "
307 + f"You supplied {tuple(d.shape)}.")
308 +
309 + for d in data:
310 + _validate_shape(d)
311 +
312 + return data
313 +
314 + def _parse_and_validate_image_input(
315 + self, **kwargs: object) -> Optional[InternVLImageInputs]:
316 + pixel_values_flat = kwargs.pop("pixel_values_flat", None)
317 + image_num_patches = kwargs.pop("image_num_patches", None)
318 + image_embeds = kwargs.pop("image_embeds", None)
319 +
320 + if pixel_values_flat is None and image_embeds is None:
321 + return None
322 +
323 + if image_embeds is not None:
324 + if not isinstance(image_embeds, (torch.Tensor, list)):
325 + raise ValueError("Incorrect type of image embeddings. "
326 + f"Got type: {type(image_embeds)}")
327 +
328 + return InternVLImageEmbeddingInputs(
329 + type="image_embeds",
330 + data=flatten_bn(image_embeds),
331 + )
332 +
333 + image_token_id = kwargs["image_token_id"]
334 + assert isinstance(image_token_id, torch.Tensor)
335 + self.img_context_token_id = image_token_id.flatten().unique().item()
336 +
337 + if pixel_values_flat is not None:
338 + if not isinstance(pixel_values_flat, (torch.Tensor, list)):
339 + raise ValueError("Incorrect type of pixel values. "
340 + f"Got type: {type(pixel_values_flat)}")
341 +
342 + if not isinstance(image_num_patches, (torch.Tensor, list)):
343 + raise ValueError("Incorrect type of image_num_patches. "
344 + f"Got type: {type(image_num_patches)}")
345 +
346 + pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
347 + image_num_patches = flatten_bn(image_num_patches, concat=True)
348 +
349 + return InternVLImagePixelInputs(
350 + type="pixel_values",
351 + pixel_values_flat=self._validate_pixel_values(
352 + pixel_values_flat),
353 + num_patches=image_num_patches,
354 + )
355 +
356 + raise AssertionError("This line should be unreachable.")
357 +
358 + def _process_image_input(
359 + self,
360 + image_input: InternVLImageInputs,
361 + ) -> tuple[torch.Tensor, ...]:
362 + if image_input["type"] == "image_embeds":
363 + return image_input["data"]
364 +
365 + assert self.vision_model is not None
366 +
367 + image_embeds = self.extract_feature(image_input["pixel_values_flat"])
368 +
369 + num_patches = image_input["num_patches"]
370 +
371 + # Only one image in the current batch
372 + if len(num_patches) == 1:
373 + return (image_embeds.view(-1,
374 + self.config.text_config.hidden_size), )
375 +
376 + # NOTE: Image embeddings are split into separate tensors for each image
377 + # by the size of each embedding.
378 + feature_size = image_embeds.shape[1]
379 + image_embeds = image_embeds.view(-1,
380 + self.config.text_config.hidden_size)
381 + image_feature_sizes = [
382 + num_patches * feature_size for num_patches in num_patches
383 + ]
384 + return image_embeds.split(image_feature_sizes)
385 +
386 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
387 + modalities = {}
388 +
389 + # Preserve the order of modalities if there are multiple of them
390 + # from the order of kwargs.
391 + for input_key in kwargs:
392 + if input_key in ("pixel_values_flat",
393 + "image_embeds") and "images" not in modalities:
394 + modalities["images"] = self._parse_and_validate_image_input(
395 + **kwargs)
396 +
397 + return modalities
398 +
399 + def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
400 + self.visual_token_mask = None
401 +
402 + def get_language_model(self) -> torch.nn.Module:
403 + return self.language_model
404 +
405 + def get_multimodal_embeddings(self,
406 + **kwargs: object) -> MultiModalEmbeddings:
407 +
408 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
409 + if not modalities:
410 + return []
411 +
412 + # The result multimodal_embeddings is tuple of tensors, with each
413 + # tensor correspoending to a multimodal data item (image).
414 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
415 +
416 + # NOTE: It is important to iterate over the keys in this dictionary
417 + # to preserve the order of the modalities.
418 + for modality in modalities:
419 + if modality == "images":
420 + image_input = modalities["images"]
421 + vision_embeddings = self._process_image_input(image_input)
422 + multimodal_embeddings += vision_embeddings
423 +
424 + return multimodal_embeddings
425 +
426 + def get_input_embeddings(
427 + self,
428 + input_ids: torch.Tensor,
429 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
430 + ) -> torch.Tensor:
431 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
432 + if multimodal_embeddings is not None \
433 + and len(multimodal_embeddings) != 0:
434 + context_token_ids = [self.img_context_token_id]
435 + assert len(context_token_ids) >= 1
436 + self._set_visual_token_mask(input_ids)
437 + inputs_embeds = merge_multimodal_embeddings(
438 + input_ids,
439 + inputs_embeds,
440 + multimodal_embeddings,
441 + context_token_ids,
442 + )
443 + return inputs_embeds
444 +
445 + def forward(
446 + self,
447 + input_ids: torch.Tensor,
448 + positions: torch.Tensor,
449 + intermediate_tensors: Optional[IntermediateTensors] = None,
450 + inputs_embeds: Optional[torch.Tensor] = None,
451 + **kwargs: object,
452 + ) -> IntermediateTensors:
453 +
454 + if intermediate_tensors is not None:
455 + input_ids = None
456 + inputs_embeds = None
457 +
458 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
459 + # condition is for v0 compatibility.
460 + elif inputs_embeds is None:
461 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
462 + inputs_embeds = self.get_input_embeddings(input_ids,
463 + vision_embeddings)
464 + input_ids = None
465 +
466 + forward_kwargs = {
467 + "input_ids": input_ids,
468 + "positions": positions,
469 + "intermediate_tensors": intermediate_tensors,
470 + "inputs_embeds": inputs_embeds,
471 + }
472 +
473 + # Only required if the model is mono-architecture
474 + if self.visual_token_mask is not None:
475 + forward_kwargs.update(
476 + {"visual_token_mask": self.visual_token_mask})
477 + self.visual_token_mask = None
478 +
479 + hidden_states = self.language_model.model(**forward_kwargs)
480 + return hidden_states
481 +
482 + def compute_logits(
483 + self,
484 + hidden_states: torch.Tensor,
485 + sampling_metadata: SamplingMetadata,
486 + ) -> Optional[torch.Tensor]:
487 + return self.language_model.compute_logits(hidden_states,
488 + sampling_metadata)
489 +
490 + def load_weights(self, weights: Iterable[tuple[str,
491 + torch.Tensor]]) -> set[str]:
492 + ## Ignore registered_buffers
493 + ## see https://huggingface.co/nvidia/C-RADIOv2-H/blob/main/input_conditioner.py#L28 # noqa: E501
494 + skip_substrs = ["norm_mean", "norm_std"]
495 + loader = AutoWeightsLoader(self, skip_substrs=skip_substrs)
496 + return loader.load_weights(weights)
497 +
498 + def get_mm_mapping(self) -> MultiModelKeys:
499 + """
500 + Get the module prefix in multimodal models
501 + """
502 + return MultiModelKeys.from_string_field(
503 + language_model="language_model",
504 + connector="mlp1",
505 + tower_model="vision_model")
vllm/model_executor/models/registry.py
+1
@@ -206,6 +206,7 @@ _MULTIMODAL_MODELS = {
206 "SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"), # noqa: E501
207 "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
208 "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"), # noqa: E501
209 + "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
210 "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
211 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
212 "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
vllm/transformers_utils/configs/nemotron.py
+1 -1
@@ -202,4 +202,4 @@ class NemotronConfig(PretrainedConfig):
202 rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
203 raise ValueError(
204 "`rope_scaling`'s factor field must be a float > 1, got "
205 - f"{rope_scaling_factor}")
205 + f"{rope_scaling_factor}")
\ No newline at end of file