A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model][VLM] Initialize support for Mono-InternVL model (#9528)
Isotr0py committed
Oct 23, 2024 at 00:01 UTC
bb392ea2d2bfde4ce101ff8c87774b85100469c9
6 files changed
+254
-28
docs/source/models/supported_models.rst
+1
-1
@@ -376,7 +376,7 @@ Text Generation
376
* - :code:`InternVLChatModel`
377
- InternVL2
378
- T + I\ :sup:`E+`
379
- - :code:`OpenGVLab/InternVL2-4B`, :code:`OpenGVLab/InternVL2-8B`, etc.
379
+ - :code:`OpenGVLab/Mono-InternVL-2B`, :code:`OpenGVLab/InternVL2-4B`, :code:`OpenGVLab/InternVL2-8B`, etc.
380
-
381
- ✅︎
382
* - :code:`LlavaForConditionalGeneration`
tests/models/decoder_only/vision_language/test_internvl.py
+13
-8
@@ -7,7 +7,6 @@ from PIL.Image import Image
7
from transformers import AutoConfig
8
9
from vllm.multimodal.utils import rescale_image_size
10
-from vllm.platforms import current_platform
10
11
from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12
_ImageAssets)
@@ -19,15 +18,20 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18
"cherry_blossom":
19
"<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
20
})
22
-HF_MULTIIMAGE_IMAGE_PROMPT = "<|im_start|>User\nImage-1: <image>\nImage-2: <image>\nDescribe the two images in detail.<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
21
+HF_MULTIIMAGE_IMAGE_PROMPT = "<|im_start|>User\nImage-1: <image>\nImage-2: <image>\nDescribe the two images in short.<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
22
23
models = [
24
"OpenGVLab/InternVL2-1B",
25
"OpenGVLab/InternVL2-2B",
26
+ # NOTE: Mono-InternVL-2B doesn't work with fp16,
27
+ # it will result NaN during inference.
28
+ # See: https://huggingface.co/OpenGVLab/Mono-InternVL-2B/discussions/9
29
+ "OpenGVLab/Mono-InternVL-2B",
30
# Broken due to outdated implementation of Phi-3
31
# See: https://huggingface.co/OpenGVLab/InternVL2-4B/discussions/3
32
# "OpenGVLab/InternVL2-4B",
33
]
34
+target_dtype = "bfloat16"
35
36
37
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py
@@ -52,9 +56,15 @@ def generate(
56
57
input_embeds = input_embeds.reshape(B, N, C)
58
55
- outputs = self.language_model.generate(
59
+ forward_kwargs = dict(
60
inputs_embeds=input_embeds,
61
attention_mask=attention_mask,
62
+ )
63
+ if getattr(self, "use_visual_token_mask", False):
64
+ visual_token_mask = selected.reshape(B, N, 1).to(input_embeds.dtype)
65
+ forward_kwargs["visual_token_mask"] = visual_token_mask
66
+ outputs = self.language_model.generate(
67
+ **forward_kwargs,
68
**generate_kwargs,
69
)
70
@@ -243,11 +253,6 @@ def run_awq_test(
253
)
254
255
246
-target_dtype = "half"
247
-if current_platform.is_cpu():
248
- target_dtype = "bfloat16"
249
-
250
-
256
@pytest.mark.parametrize("model", models)
257
@pytest.mark.parametrize(
258
"size_factors",
vllm/model_executor/models/intern_vit.py
+31
@@ -97,6 +97,37 @@ class InternVisionEmbeddings(nn.Module):
97
return embeddings
98
99
100
+class InternVisionPatchModel(nn.Module):
101
+
102
+ def __init__(self, config: PretrainedConfig):
103
+ super().__init__()
104
+ self.config = config
105
+ self.embeddings = InternVisionEmbeddings(config)
106
+
107
+ def get_input_embeddings(self):
108
+ return self.embeddings
109
+
110
+ def forward(
111
+ self,
112
+ pixel_values: Optional[torch.Tensor] = None,
113
+ pixel_embeds: Optional[torch.Tensor] = None,
114
+ ) -> torch.FloatTensor:
115
+ if pixel_values is None and pixel_embeds is None:
116
+ raise ValueError(
117
+ 'You have to specify pixel_values or pixel_embeds')
118
+
119
+ if pixel_embeds is not None:
120
+ hidden_states = pixel_embeds
121
+ elif pixel_values is not None:
122
+ if pixel_values.ndim == 4:
123
+ hidden_states = self.embeddings(pixel_values)
124
+ else:
125
+ raise ValueError(
126
+ f'wrong pixel_values size: {pixel_values.shape}')
127
+
128
+ return hidden_states
129
+
130
+
131
class InternParallelAttention(nn.Module):
132
"""Multi-headed attention from 'Attention Is All You Need' paper"""
133
vllm/model_executor/models/internlm2_ve.py
new
+166
@@ -0,0 +1,166 @@
1
+# -*- coding: utf-8 -*-
2
+from typing import List, Optional, Tuple, Union
3
+
4
+import torch
5
+from torch import nn
6
+from transformers import PretrainedConfig
7
+
8
+from vllm.attention import AttentionMetadata
9
+from vllm.config import CacheConfig
10
+from vllm.distributed import get_pp_group
11
+from vllm.model_executor.layers.layernorm import RMSNorm
12
+from vllm.model_executor.layers.quantization import QuantizationConfig
13
+from vllm.model_executor.models.internlm2 import (InternLM2Attention,
14
+ InternLM2ForCausalLM,
15
+ InternLM2MLP, InternLM2Model)
16
+from vllm.sequence import IntermediateTensors
17
+
18
+from .utils import make_layers
19
+
20
+
21
+class InternLM2VEDecoderLayer(nn.Module):
22
+
23
+ def __init__(
24
+ self,
25
+ config: PretrainedConfig,
26
+ cache_config: Optional[CacheConfig] = None,
27
+ quant_config: Optional[QuantizationConfig] = None,
28
+ ) -> None:
29
+ super().__init__()
30
+ self.hidden_size = config.hidden_size
31
+ rope_theta = getattr(config, "rope_theta", 10000)
32
+ rope_scaling = getattr(config, "rope_scaling", None)
33
+ max_position_embeddings = getattr(config, "max_position_embeddings",
34
+ 8192)
35
+ self.attention = InternLM2Attention(
36
+ hidden_size=self.hidden_size,
37
+ num_heads=config.num_attention_heads,
38
+ num_kv_heads=config.num_key_value_heads,
39
+ rope_theta=rope_theta,
40
+ rope_scaling=rope_scaling,
41
+ max_position_embeddings=max_position_embeddings,
42
+ cache_config=cache_config,
43
+ quant_config=quant_config,
44
+ )
45
+ self.feed_forward = InternLM2MLP(
46
+ hidden_size=self.hidden_size,
47
+ intermediate_size=config.intermediate_size,
48
+ hidden_act=config.hidden_act,
49
+ quant_config=quant_config,
50
+ )
51
+ self.feed_forward_ve = InternLM2MLP(
52
+ hidden_size=self.hidden_size,
53
+ intermediate_size=config.intermediate_size,
54
+ hidden_act=config.hidden_act,
55
+ quant_config=quant_config,
56
+ )
57
+ self.attention_norm = RMSNorm(config.hidden_size,
58
+ eps=config.rms_norm_eps)
59
+ self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
60
+
61
+ def forward(
62
+ self,
63
+ positions: torch.Tensor,
64
+ hidden_states: torch.Tensor,
65
+ kv_cache: torch.Tensor,
66
+ attn_metadata: AttentionMetadata,
67
+ residual: Optional[torch.Tensor],
68
+ visual_token_mask: Optional[torch.Tensor] = None,
69
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
70
+ # Self Attention
71
+ if residual is None:
72
+ residual = hidden_states
73
+ hidden_states = self.attention_norm(hidden_states)
74
+ else:
75
+ hidden_states, residual = self.attention_norm(
76
+ hidden_states, residual)
77
+ hidden_states = self.attention(
78
+ positions=positions,
79
+ hidden_states=hidden_states,
80
+ kv_cache=kv_cache,
81
+ attn_metadata=attn_metadata,
82
+ )
83
+
84
+ # Fully Connected
85
+ hidden_states, residual = self.ffn_norm(hidden_states, residual)
86
+ if visual_token_mask is not None and visual_token_mask.any():
87
+ visual_token_mask = visual_token_mask.repeat(
88
+ 1, self.hidden_size).bool()
89
+ text_token_mask = ~visual_token_mask
90
+ hidden_states[visual_token_mask] = self.feed_forward_ve(
91
+ hidden_states[visual_token_mask].reshape(
92
+ -1, self.hidden_size)).flatten()
93
+ if text_token_mask.any():
94
+ hidden_states[text_token_mask] = self.feed_forward(
95
+ hidden_states[text_token_mask].reshape(
96
+ -1, self.hidden_size)).flatten()
97
+ else:
98
+ hidden_states = self.feed_forward(hidden_states)
99
+ return hidden_states, residual
100
+
101
+
102
+class InternLM2VEModel(InternLM2Model):
103
+
104
+ def __init__(
105
+ self,
106
+ config: PretrainedConfig,
107
+ cache_config: Optional[CacheConfig] = None,
108
+ quant_config: Optional[QuantizationConfig] = None,
109
+ prefix: str = "",
110
+ ) -> None:
111
+ super().__init__(config, cache_config, quant_config)
112
+ self.start_layer, self.end_layer, self.layers = make_layers(
113
+ config.num_hidden_layers,
114
+ lambda prefix: InternLM2VEDecoderLayer(config, cache_config,
115
+ quant_config),
116
+ prefix=f"{prefix}.layers")
117
+
118
+ def forward(
119
+ self,
120
+ input_ids: torch.Tensor,
121
+ positions: torch.Tensor,
122
+ kv_caches: List[torch.Tensor],
123
+ attn_metadata: AttentionMetadata,
124
+ intermediate_tensors: Optional[IntermediateTensors] = None,
125
+ inputs_embeds: Optional[torch.Tensor] = None,
126
+ visual_token_mask: Optional[torch.Tensor] = None,
127
+ ) -> Union[torch.Tensor, IntermediateTensors]:
128
+ if get_pp_group().is_first_rank:
129
+ if inputs_embeds is not None:
130
+ hidden_states = inputs_embeds
131
+ else:
132
+ hidden_states = self.tok_embeddings(input_ids)
133
+ residual = None
134
+ else:
135
+ assert intermediate_tensors is not None
136
+ hidden_states = intermediate_tensors["hidden_states"]
137
+ residual = intermediate_tensors["residual"]
138
+ for i in range(self.start_layer, self.end_layer):
139
+ layer = self.layers[i]
140
+ hidden_states, residual = layer(
141
+ positions,
142
+ hidden_states,
143
+ kv_caches[i - self.start_layer],
144
+ attn_metadata,
145
+ residual,
146
+ visual_token_mask=visual_token_mask,
147
+ )
148
+ if not get_pp_group().is_last_rank:
149
+ return IntermediateTensors({
150
+ "hidden_states": hidden_states,
151
+ "residual": residual
152
+ })
153
+ hidden_states, _ = self.norm(hidden_states, residual)
154
+ return hidden_states
155
+
156
+
157
+class InternLM2VEForCausalLM(InternLM2ForCausalLM):
158
+
159
+ def __init__(
160
+ self,
161
+ config: PretrainedConfig,
162
+ cache_config: Optional[CacheConfig] = None,
163
+ quant_config: Optional[QuantizationConfig] = None,
164
+ ) -> None:
165
+ super().__init__(config, cache_config, quant_config)
166
+ self.model = InternLM2VEModel(config, cache_config, quant_config)
vllm/model_executor/models/internvl.py
+42
-19
@@ -21,7 +21,8 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21
token_inputs)
22
from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24
-from vllm.model_executor.models.intern_vit import InternVisionModel
24
+from vllm.model_executor.models.intern_vit import (InternVisionModel,
25
+ InternVisionPatchModel)
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.multimodal import MULTIMODAL_REGISTRY
28
from vllm.multimodal.base import MultiModalInputs
@@ -427,13 +428,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
428
self.downsample_ratio = config.downsample_ratio
429
self.ps_version = config.ps_version
430
430
- vision_feature_layer = self.select_layer
431
- if vision_feature_layer < 0:
432
- num_hidden_layers = config.vision_config.num_hidden_layers \
433
- + vision_feature_layer + 1
434
- else:
435
- num_hidden_layers = vision_feature_layer + 1
436
- self.vision_model = self._init_vision_model(config, num_hidden_layers)
431
+ self.llm_arch_name = config.text_config.architectures[0]
432
+ self.is_mono = self.llm_arch_name == 'InternLM2VEForCausalLM'
433
+ self.vision_model = self._init_vision_model(config, self.is_mono)
434
435
self.language_model = init_vllm_registered_model(
436
config.text_config, cache_config, quant_config)
@@ -451,10 +448,19 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
448
449
return Sampler()
450
454
- def _init_vision_model(self, config: PretrainedConfig,
455
- num_hidden_layers: int):
456
- return InternVisionModel(config.vision_config,
457
- num_hidden_layers_override=num_hidden_layers)
451
+ def _init_vision_model(self, config: PretrainedConfig, is_mono: bool):
452
+ if not is_mono:
453
+ vision_feature_layer = self.select_layer
454
+ if vision_feature_layer < 0:
455
+ num_hidden_layers = config.vision_config.num_hidden_layers \
456
+ + vision_feature_layer + 1
457
+ else:
458
+ num_hidden_layers = vision_feature_layer + 1
459
+ return InternVisionModel(
460
+ config.vision_config,
461
+ num_hidden_layers_override=num_hidden_layers)
462
+ else:
463
+ return InternVisionPatchModel(config.vision_config)
464
465
def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
466
vit_hidden_size = config.vision_config.hidden_size
@@ -562,6 +568,14 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
568
569
return image_embeds
570
571
+ def _get_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
572
+ if self.is_mono:
573
+ visual_token_mask = (
574
+ input_ids == self.img_context_token_id).reshape(-1, 1)
575
+ else:
576
+ visual_token_mask = None
577
+ return visual_token_mask
578
+
579
def forward(
580
self,
581
input_ids: torch.Tensor,
@@ -574,6 +588,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
588
if intermediate_tensors is not None:
589
input_ids = None
590
inputs_embeds = None
591
+ visual_token_mask = None
592
else:
593
image_input = self._parse_and_validate_image_input(**kwargs)
594
if image_input is not None:
@@ -583,16 +598,24 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
598
inputs_embeds = merge_multimodal_embeddings(
599
input_ids, inputs_embeds, vision_embeddings,
600
self.img_context_token_id)
601
+ visual_token_mask = self._get_visual_token_mask(input_ids)
602
input_ids = None
603
else:
604
inputs_embeds = None
589
-
590
- hidden_states = self.language_model.model(input_ids,
591
- positions,
592
- kv_caches,
593
- attn_metadata,
594
- intermediate_tensors,
595
- inputs_embeds=inputs_embeds)
605
+ visual_token_mask = None
606
+
607
+ forward_kwargs = {
608
+ "input_ids": input_ids,
609
+ "positions": positions,
610
+ "kv_caches": kv_caches,
611
+ "attn_metadata": attn_metadata,
612
+ "intermediate_tensors": intermediate_tensors,
613
+ "inputs_embeds": inputs_embeds,
614
+ }
615
+ if self.is_mono:
616
+ forward_kwargs.update({"visual_token_mask": visual_token_mask})
617
+
618
+ hidden_states = self.language_model.model(**forward_kwargs)
619
return hidden_states
620
621
def compute_logits(
vllm/model_executor/models/registry.py
+1
@@ -47,6 +47,7 @@ _TEXT_GENERATION_MODELS = {
47
"GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
48
"InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
49
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
50
+ "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
51
"JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
52
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
53
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),