A high-throughput and memory-efficient inference and serving engine for LLMs
[Model] Remove Mono-InternVL (InternLM2VEForCausalLM) (#45129)
Signed-off-by: Xianbao QIAN <xianbao.qian@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Tiezhen WANG committed
Jun 12, 2026 at 18:41 UTC
f1e13f7df9ad360df756ffeced301df97b209414
9 files changed
+53
-262
docs/models/supported_models.md
+1
-1
@@ -577,7 +577,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
577
| `InternS1ForConditionalGeneration` | Intern-S1 | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S1`, `internlm/Intern-S1-mini`, etc. | ✅︎ | ✅︎ |
578
| `InternS1ProForConditionalGeneration` | Intern-S1-Pro | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S1-Pro`, etc. | ✅︎ | ✅︎ |
579
| `InternS2PreviewForConditionalGeneration` | Intern-S2-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S2-Preview`, etc. | ✅︎ | ✅︎ |
580
-| `InternVLChatModel` | InternVL 3.5, InternVL 3.0, InternVideo 2.5, InternVL 2.5, Mono-InternVL, InternVL 2.0 | T + I<sup>E+</sup> + (V<sup>E+</sup>) | `OpenGVLab/InternVL3_5-14B`, `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/Mono-InternVL-2B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ |
580
+| `InternVLChatModel` | InternVL 3.5, InternVL 3.0, InternVideo 2.5, InternVL 2.5, InternVL 2.0 | T + I<sup>E+</sup> + (V<sup>E+</sup>) | `OpenGVLab/InternVL3_5-14B`, `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ |
581
| `InternVLForConditionalGeneration` | InternVL 3.0 (HF format) | T + I<sup>E+</sup> + V<sup>E+</sup> | `OpenGVLab/InternVL3-1B-hf`, etc. | ✅︎ | ✅︎ |
582
| `KananaVForConditionalGeneration` | Kanana-V | T + I<sup>+</sup> | `kakaocorp/kanana-1.5-v-3b-instruct`, etc. | | ✅︎ |
583
| `KeyeForConditionalGeneration` | Keye-VL-8B-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `Kwai-Keye/Keye-VL-8B-Preview` | ✅︎ | ✅︎ |
tests/models/multimodal/generation/test_common.py
-2
@@ -604,8 +604,6 @@ VLM_TEST_SETTINGS = {
604
models=[
605
"OpenGVLab/InternVL2-1B",
606
"OpenGVLab/InternVL2-2B",
607
- # FIXME: Config cannot be loaded in transformers 4.52
608
- # "OpenGVLab/Mono-InternVL-2B",
607
],
608
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
609
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
tests/models/registry.py
-11
@@ -341,17 +341,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
341
"InternLM2ForCausalLM": _HfExamplesInfo(
342
"internlm/internlm2-chat-7b", trust_remote_code=True
343
),
344
- "InternLM2VEForCausalLM": _HfExamplesInfo(
345
- "OpenGVLab/Mono-InternVL-2B",
346
- trust_remote_code=True,
347
- max_transformers_version="4.57",
348
- transformers_version_reason={
349
- "vllm": (
350
- "Custom config cannot be loaded with Transformers "
351
- "v5 because `vision_config` is not always set"
352
- )
353
- },
354
- ),
344
"InternLM3ForCausalLM": _HfExamplesInfo(
345
"internlm/internlm3-8b-instruct", trust_remote_code=True
346
),
vllm/model_executor/models/h2ovl.py
+12
-17
@@ -157,27 +157,22 @@ class H2OVLChatModel(InternVLChatModel):
157
config: PretrainedConfig,
158
quant_config: QuantizationConfig | None,
159
*,
160
- is_mono: bool,
160
prefix: str,
161
):
163
- if not is_mono:
164
- vision_feature_layer = config.select_layer
165
- if vision_feature_layer < 0:
166
- num_hidden_layers = (
167
- config.vision_config.num_hidden_layers + vision_feature_layer + 1
168
- )
169
- else:
170
- num_hidden_layers = vision_feature_layer + 1
171
-
172
- return InternVisionModel(
173
- config.vision_config,
174
- quant_config=quant_config,
175
- num_hidden_layers_override=num_hidden_layers,
176
- prefix=prefix,
162
+ vision_feature_layer = config.select_layer
163
+ if vision_feature_layer < 0:
164
+ num_hidden_layers = (
165
+ config.vision_config.num_hidden_layers + vision_feature_layer + 1
166
)
167
else:
179
- msg = "Monolith mode is not applicable to H2OVL"
180
- raise NotImplementedError(msg)
168
+ num_hidden_layers = vision_feature_layer + 1
169
+
170
+ return InternVisionModel(
171
+ config.vision_config,
172
+ quant_config=quant_config,
173
+ num_hidden_layers_override=num_hidden_layers,
174
+ prefix=prefix,
175
+ )
176
177
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
178
if num_image_tokens <= 0 or self.num_image_token <= 0:
vllm/model_executor/models/internlm2_ve.py
deleted
-139
@@ -1,139 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from itertools import islice
5
-
6
-import torch
7
-from torch import nn
8
-from transformers import PretrainedConfig
9
-
10
-from vllm.config import CacheConfig, VllmConfig
11
-from vllm.distributed import get_pp_group
12
-from vllm.model_executor.layers.layernorm import RMSNorm
13
-from vllm.model_executor.layers.quantization import QuantizationConfig
14
-from vllm.model_executor.models.internlm2 import (
15
- InternLM2Attention,
16
- InternLM2ForCausalLM,
17
- InternLM2MLP,
18
- InternLM2Model,
19
-)
20
-from vllm.sequence import IntermediateTensors
21
-
22
-
23
-class InternLM2VEDecoderLayer(nn.Module):
24
- def __init__(
25
- self,
26
- config: PretrainedConfig,
27
- cache_config: CacheConfig | None = None,
28
- quant_config: QuantizationConfig | None = None,
29
- prefix: str = "",
30
- ) -> None:
31
- super().__init__()
32
- self.hidden_size = config.hidden_size
33
- max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
34
- self.attention = InternLM2Attention(
35
- hidden_size=self.hidden_size,
36
- num_heads=config.num_attention_heads,
37
- num_kv_heads=config.num_key_value_heads,
38
- rope_parameters=config.rope_parameters,
39
- max_position_embeddings=max_position_embeddings,
40
- cache_config=cache_config,
41
- quant_config=quant_config,
42
- prefix=f"{prefix}.attention",
43
- )
44
- self.feed_forward = InternLM2MLP(
45
- hidden_size=self.hidden_size,
46
- intermediate_size=config.intermediate_size,
47
- hidden_act=config.hidden_act,
48
- quant_config=quant_config,
49
- prefix=f"{prefix}.feed_forward",
50
- )
51
- self.feed_forward_ve = InternLM2MLP(
52
- hidden_size=self.hidden_size,
53
- intermediate_size=config.intermediate_size,
54
- hidden_act=config.hidden_act,
55
- quant_config=quant_config,
56
- prefix=f"{prefix}.feed_forward_ve",
57
- )
58
- self.attention_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
59
- self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
60
-
61
- def forward(
62
- self,
63
- positions: torch.Tensor,
64
- hidden_states: torch.Tensor,
65
- residual: torch.Tensor | None,
66
- visual_token_mask: torch.Tensor | None = None,
67
- ) -> tuple[torch.Tensor, torch.Tensor]:
68
- # Self Attention
69
- if residual is None:
70
- residual = hidden_states
71
- hidden_states = self.attention_norm(hidden_states)
72
- else:
73
- hidden_states, residual = self.attention_norm(hidden_states, residual)
74
- hidden_states = self.attention(
75
- positions=positions,
76
- hidden_states=hidden_states,
77
- )
78
-
79
- # Fully Connected
80
- hidden_states, residual = self.ffn_norm(hidden_states, residual)
81
- if visual_token_mask is not None and visual_token_mask.any():
82
- visual_token_mask = visual_token_mask.repeat(1, self.hidden_size).bool()
83
- text_token_mask = ~visual_token_mask
84
- hidden_states[visual_token_mask] = self.feed_forward_ve(
85
- hidden_states[visual_token_mask].reshape(-1, self.hidden_size)
86
- ).flatten()
87
- if text_token_mask.any():
88
- hidden_states[text_token_mask] = self.feed_forward(
89
- hidden_states[text_token_mask].reshape(-1, self.hidden_size)
90
- ).flatten()
91
- else:
92
- hidden_states = self.feed_forward(hidden_states)
93
- return hidden_states, residual
94
-
95
-
96
-class InternLM2VEModel(InternLM2Model):
97
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
98
- super().__init__(
99
- vllm_config=vllm_config, prefix=prefix, layer_type=InternLM2VEDecoderLayer
100
- )
101
-
102
- def forward(
103
- self,
104
- input_ids: torch.Tensor | None,
105
- positions: torch.Tensor,
106
- intermediate_tensors: IntermediateTensors | None = None,
107
- inputs_embeds: torch.Tensor | None = None,
108
- visual_token_mask: torch.Tensor | None = None,
109
- ) -> torch.Tensor | IntermediateTensors:
110
- if get_pp_group().is_first_rank:
111
- if inputs_embeds is not None:
112
- hidden_states = inputs_embeds
113
- else:
114
- hidden_states = self.tok_embeddings(input_ids)
115
- residual = None
116
- else:
117
- assert intermediate_tensors is not None
118
- hidden_states = intermediate_tensors["hidden_states"]
119
- residual = intermediate_tensors["residual"]
120
- for layer in islice(self.layers, self.start_layer, self.end_layer):
121
- hidden_states, residual = layer(
122
- positions,
123
- hidden_states,
124
- residual,
125
- visual_token_mask=visual_token_mask,
126
- )
127
- if not get_pp_group().is_last_rank:
128
- return IntermediateTensors(
129
- {"hidden_states": hidden_states, "residual": residual}
130
- )
131
- hidden_states, _ = self.norm(hidden_states, residual)
132
- return hidden_states
133
-
134
-
135
-class InternLM2VEForCausalLM(InternLM2ForCausalLM):
136
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
137
- super().__init__(
138
- vllm_config=vllm_config, prefix=prefix, model_type=InternLM2VEModel
139
- )
vllm/model_executor/models/internvl.py
+12
-39
@@ -23,7 +23,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.quantization.awq import AWQConfig
24
from vllm.model_executor.models.intern_vit import (
25
InternVisionModel,
26
- InternVisionPatchModel,
26
)
27
from vllm.model_executor.models.module_mapping import MultiModelKeys
28
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -582,14 +581,10 @@ class InternVLChatModel(
581
self.downsample_ratio = config.downsample_ratio
582
self.ps_version = config.ps_version
583
585
- llm_arch_name = config.text_config.architectures[0]
586
- self.is_mono = llm_arch_name == "InternLM2VEForCausalLM"
587
-
584
with self._mark_tower_model(vllm_config, {"image", "video"}):
585
self.vision_model = self._init_vision_model(
586
config,
587
quant_config=quant_config,
592
- is_mono=self.is_mono,
588
prefix=maybe_prefix(prefix, "vision_model"),
589
)
590
self.mlp1 = self._init_mlp1(config)
@@ -604,7 +599,6 @@ class InternVLChatModel(
599
self.img_context_token_id = None
600
self.video_context_token_id = None
601
607
- self.visual_token_mask = None
602
self.make_empty_intermediate_tensors = (
603
self.language_model.make_empty_intermediate_tensors
604
)
@@ -627,26 +621,22 @@ class InternVLChatModel(
621
config: PretrainedConfig,
622
quant_config: QuantizationConfig | None,
623
*,
630
- is_mono: bool,
624
prefix: str,
625
):
633
- if not is_mono:
634
- vision_feature_layer = config.select_layer
635
- if vision_feature_layer < 0:
636
- num_hidden_layers = (
637
- config.vision_config.num_hidden_layers + vision_feature_layer + 1
638
- )
639
- else:
640
- num_hidden_layers = vision_feature_layer + 1
641
-
642
- return InternVisionModel(
643
- config.vision_config,
644
- quant_config=quant_config,
645
- num_hidden_layers_override=num_hidden_layers,
646
- prefix=prefix,
626
+ vision_feature_layer = config.select_layer
627
+ if vision_feature_layer < 0:
628
+ num_hidden_layers = (
629
+ config.vision_config.num_hidden_layers + vision_feature_layer + 1
630
)
631
else:
649
- return InternVisionPatchModel(config.vision_config)
632
+ num_hidden_layers = vision_feature_layer + 1
633
+
634
+ return InternVisionModel(
635
+ config.vision_config,
636
+ quant_config=quant_config,
637
+ num_hidden_layers_override=num_hidden_layers,
638
+ prefix=prefix,
639
+ )
640
641
def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
642
vit_hidden_size = config.vision_config.hidden_size
@@ -805,15 +795,6 @@ class InternVLChatModel(
795
796
return modalities
797
808
- def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
809
- if self.is_mono:
810
- assert self.img_context_token_id is not None
811
- self.visual_token_mask = (input_ids == self.img_context_token_id).reshape(
812
- -1, 1
813
- )
814
- else:
815
- self.visual_token_mask = None
816
-
798
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
799
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
800
if not modalities:
@@ -844,9 +825,6 @@ class InternVLChatModel(
825
*,
826
is_multimodal: torch.Tensor | None = None,
827
) -> torch.Tensor:
847
- if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
848
- self._set_visual_token_mask(input_ids)
849
-
828
# This is to satisfy the type checker for each overload
829
if multimodal_embeddings is None or is_multimodal is None:
830
return super().embed_input_ids(input_ids)
@@ -875,11 +853,6 @@ class InternVLChatModel(
853
"inputs_embeds": inputs_embeds,
854
}
855
878
- # Only required if the model is mono-architecture
879
- if self.visual_token_mask is not None:
880
- forward_kwargs.update({"visual_token_mask": self.visual_token_mask})
881
- self.visual_token_mask = None
882
-
856
hidden_states = self.language_model.model(**forward_kwargs)
857
return hidden_states
858
vllm/model_executor/models/nvlm_d.py
+15
-20
@@ -177,27 +177,22 @@ class NVLM_D_Model(InternVLChatModel):
177
config: PretrainedConfig,
178
quant_config: QuantizationConfig | None,
179
*,
180
- is_mono: bool,
180
prefix: str,
181
):
183
- if not is_mono:
184
- vision_feature_layer = config.select_layer
185
- if vision_feature_layer < 0:
186
- num_hidden_layers = (
187
- config.vision_config.num_hidden_layers + vision_feature_layer + 1
188
- )
189
- else:
190
- num_hidden_layers = vision_feature_layer + 1
191
-
192
- # We added additional dummy heads to the original num of heads to
193
- # make the number of heads divisible by 8.
194
- return InternVisionModel(
195
- config.vision_config,
196
- quant_config=quant_config,
197
- num_hidden_layers_override=num_hidden_layers,
198
- num_dummy_heads=7,
199
- prefix=prefix,
182
+ vision_feature_layer = config.select_layer
183
+ if vision_feature_layer < 0:
184
+ num_hidden_layers = (
185
+ config.vision_config.num_hidden_layers + vision_feature_layer + 1
186
)
187
else:
202
- msg = "Monolith mode is not applicable to NVLM_D"
203
- raise NotImplementedError(msg)
188
+ num_hidden_layers = vision_feature_layer + 1
189
+
190
+ # We added additional dummy heads to the original num of heads to
191
+ # make the number of heads divisible by 8.
192
+ return InternVisionModel(
193
+ config.vision_config,
194
+ quant_config=quant_config,
195
+ num_hidden_layers_override=num_hidden_layers,
196
+ num_dummy_heads=7,
197
+ prefix=prefix,
198
+ )
vllm/model_executor/models/registry.py
+1
-1
@@ -141,7 +141,6 @@ _TEXT_GENERATION_MODELS = {
141
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
142
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
143
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
144
- "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
144
"InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
145
"IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
146
"IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
@@ -716,6 +715,7 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
715
"ErnieModel": "0.23.0",
716
"ErnieForSequenceClassification": "0.23.0",
717
"ErnieForTokenClassification": "0.23.0",
718
+ "InternLM2VEForCausalLM": "0.23.0",
719
"QWenLMHeadModel": "0.23.0",
720
"QwenVLForConditionalGeneration": "0.23.0",
721
"InternLMForCausalLM": "0.23.0",
vllm/model_executor/models/skyworkr1v.py
+12
-32
@@ -22,7 +22,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.layers.quantization.awq import AWQConfig
23
from vllm.model_executor.models.intern_vit import (
24
InternVisionModel,
25
- InternVisionPatchModel,
25
)
26
from vllm.multimodal import MULTIMODAL_REGISTRY
27
from vllm.multimodal.processing import BaseDummyInputsBuilder
@@ -178,14 +177,10 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
177
self.downsample_ratio = config.downsample_ratio
178
self.ps_version = config.ps_version
179
181
- llm_arch_name = config.text_config.architectures[0]
182
- self.is_mono = llm_arch_name == "SkyworkLM2VEForCausalLM"
183
-
180
with self._mark_tower_model(vllm_config, "image"):
181
self.vision_model = self._init_vision_model(
182
config,
183
quant_config=quant_config,
188
- is_mono=self.is_mono,
184
prefix=maybe_prefix(prefix, "vision_model"),
185
)
186
self.mlp1 = self._init_mlp1(
@@ -223,26 +218,22 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
218
config: PretrainedConfig,
219
quant_config: QuantizationConfig | None,
220
*,
226
- is_mono: bool,
221
prefix: str,
222
):
229
- if not is_mono:
230
- vision_feature_layer = config.select_layer
231
- if vision_feature_layer < 0:
232
- num_hidden_layers = (
233
- config.vision_config.num_hidden_layers + vision_feature_layer + 1
234
- )
235
- else:
236
- num_hidden_layers = vision_feature_layer + 1
237
-
238
- return InternVisionModel(
239
- config.vision_config,
240
- quant_config=quant_config,
241
- num_hidden_layers_override=num_hidden_layers,
242
- prefix=prefix,
223
+ vision_feature_layer = config.select_layer
224
+ if vision_feature_layer < 0:
225
+ num_hidden_layers = (
226
+ config.vision_config.num_hidden_layers + vision_feature_layer + 1
227
)
228
else:
245
- return InternVisionPatchModel(config.vision_config)
229
+ num_hidden_layers = vision_feature_layer + 1
230
+
231
+ return InternVisionModel(
232
+ config.vision_config,
233
+ quant_config=quant_config,
234
+ num_hidden_layers_override=num_hidden_layers,
235
+ prefix=prefix,
236
+ )
237
238
def _init_mlp1(
239
self,
@@ -363,14 +354,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
354
]
355
return image_embeds.split(image_feature_sizes)
356
366
- def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
367
- if self.is_mono:
368
- self.visual_token_mask = (input_ids == self.img_context_token_id).reshape(
369
- -1, 1
370
- )
371
- else:
372
- self.visual_token_mask = None
373
-
357
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
358
image_input = self._parse_and_validate_image_input(**kwargs)
359
if image_input is None:
@@ -385,9 +368,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
368
*,
369
is_multimodal: torch.Tensor | None = None,
370
) -> torch.Tensor:
388
- if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
389
- self._set_visual_token_mask(input_ids)
390
-
371
# This is to satisfy the type checker for each overload
372
if multimodal_embeddings is None or is_multimodal is None:
373
return super().embed_input_ids(input_ids)