@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb392ea2d2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model][VLM] Initialize support for Mono-InternVL model (#9528)

Isotr0py committed Oct 23, 2024 at 00:01 UTC bb392ea2d2bfde4ce101ff8c87774b85100469c9
6 files changed +254 -28
docs/source/models/supported_models.rst
+1 -1
@@ -376,7 +376,7 @@ Text Generation
376 * - :code:`InternVLChatModel`
377 - InternVL2
378 - T + I\ :sup:`E+`
379 - - :code:`OpenGVLab/InternVL2-4B`, :code:`OpenGVLab/InternVL2-8B`, etc.
379 + - :code:`OpenGVLab/Mono-InternVL-2B`, :code:`OpenGVLab/InternVL2-4B`, :code:`OpenGVLab/InternVL2-8B`, etc.
380 -
381 - ✅︎
382 * - :code:`LlavaForConditionalGeneration`
tests/models/decoder_only/vision_language/test_internvl.py
+13 -8
@@ -7,7 +7,6 @@ from PIL.Image import Image
7 from transformers import AutoConfig
8
9 from vllm.multimodal.utils import rescale_image_size
10 -from vllm.platforms import current_platform
10
11 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12 _ImageAssets)
@@ -19,15 +18,20 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18 "cherry_blossom":
19 "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
20 })
22 -HF_MULTIIMAGE_IMAGE_PROMPT = "<|im_start|>User\nImage-1: <image>\nImage-2: <image>\nDescribe the two images in detail.<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
21 +HF_MULTIIMAGE_IMAGE_PROMPT = "<|im_start|>User\nImage-1: <image>\nImage-2: <image>\nDescribe the two images in short.<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
22
23 models = [
24 "OpenGVLab/InternVL2-1B",
25 "OpenGVLab/InternVL2-2B",
26 + # NOTE: Mono-InternVL-2B doesn't work with fp16,
27 + # it will result NaN during inference.
28 + # See: https://huggingface.co/OpenGVLab/Mono-InternVL-2B/discussions/9
29 + "OpenGVLab/Mono-InternVL-2B",
30 # Broken due to outdated implementation of Phi-3
31 # See: https://huggingface.co/OpenGVLab/InternVL2-4B/discussions/3
32 # "OpenGVLab/InternVL2-4B",
33 ]
34 +target_dtype = "bfloat16"
35
36
37 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py
@@ -52,9 +56,15 @@ def generate(
56
57 input_embeds = input_embeds.reshape(B, N, C)
58
55 - outputs = self.language_model.generate(
59 + forward_kwargs = dict(
60 inputs_embeds=input_embeds,
61 attention_mask=attention_mask,
62 + )
63 + if getattr(self, "use_visual_token_mask", False):
64 + visual_token_mask = selected.reshape(B, N, 1).to(input_embeds.dtype)
65 + forward_kwargs["visual_token_mask"] = visual_token_mask
66 + outputs = self.language_model.generate(
67 + **forward_kwargs,
68 **generate_kwargs,
69 )
70
@@ -243,11 +253,6 @@ def run_awq_test(
253 )
254
255
246 -target_dtype = "half"
247 -if current_platform.is_cpu():
248 - target_dtype = "bfloat16"
249 -
250 -
256 @pytest.mark.parametrize("model", models)
257 @pytest.mark.parametrize(
258 "size_factors",
vllm/model_executor/models/intern_vit.py
+31
@@ -97,6 +97,37 @@ class InternVisionEmbeddings(nn.Module):
97 return embeddings
98
99
100 +class InternVisionPatchModel(nn.Module):
101 +
102 + def __init__(self, config: PretrainedConfig):
103 + super().__init__()
104 + self.config = config
105 + self.embeddings = InternVisionEmbeddings(config)
106 +
107 + def get_input_embeddings(self):
108 + return self.embeddings
109 +
110 + def forward(
111 + self,
112 + pixel_values: Optional[torch.Tensor] = None,
113 + pixel_embeds: Optional[torch.Tensor] = None,
114 + ) -> torch.FloatTensor:
115 + if pixel_values is None and pixel_embeds is None:
116 + raise ValueError(
117 + 'You have to specify pixel_values or pixel_embeds')
118 +
119 + if pixel_embeds is not None:
120 + hidden_states = pixel_embeds
121 + elif pixel_values is not None:
122 + if pixel_values.ndim == 4:
123 + hidden_states = self.embeddings(pixel_values)
124 + else:
125 + raise ValueError(
126 + f'wrong pixel_values size: {pixel_values.shape}')
127 +
128 + return hidden_states
129 +
130 +
131 class InternParallelAttention(nn.Module):
132 """Multi-headed attention from 'Attention Is All You Need' paper"""
133
vllm/model_executor/models/internlm2_ve.py new
+166
@@ -0,0 +1,166 @@
1 +# -*- coding: utf-8 -*-
2 +from typing import List, Optional, Tuple, Union
3 +
4 +import torch
5 +from torch import nn
6 +from transformers import PretrainedConfig
7 +
8 +from vllm.attention import AttentionMetadata
9 +from vllm.config import CacheConfig
10 +from vllm.distributed import get_pp_group
11 +from vllm.model_executor.layers.layernorm import RMSNorm
12 +from vllm.model_executor.layers.quantization import QuantizationConfig
13 +from vllm.model_executor.models.internlm2 import (InternLM2Attention,
14 + InternLM2ForCausalLM,
15 + InternLM2MLP, InternLM2Model)
16 +from vllm.sequence import IntermediateTensors
17 +
18 +from .utils import make_layers
19 +
20 +
21 +class InternLM2VEDecoderLayer(nn.Module):
22 +
23 + def __init__(
24 + self,
25 + config: PretrainedConfig,
26 + cache_config: Optional[CacheConfig] = None,
27 + quant_config: Optional[QuantizationConfig] = None,
28 + ) -> None:
29 + super().__init__()
30 + self.hidden_size = config.hidden_size
31 + rope_theta = getattr(config, "rope_theta", 10000)
32 + rope_scaling = getattr(config, "rope_scaling", None)
33 + max_position_embeddings = getattr(config, "max_position_embeddings",
34 + 8192)
35 + self.attention = InternLM2Attention(
36 + hidden_size=self.hidden_size,
37 + num_heads=config.num_attention_heads,
38 + num_kv_heads=config.num_key_value_heads,
39 + rope_theta=rope_theta,
40 + rope_scaling=rope_scaling,
41 + max_position_embeddings=max_position_embeddings,
42 + cache_config=cache_config,
43 + quant_config=quant_config,
44 + )
45 + self.feed_forward = InternLM2MLP(
46 + hidden_size=self.hidden_size,
47 + intermediate_size=config.intermediate_size,
48 + hidden_act=config.hidden_act,
49 + quant_config=quant_config,
50 + )
51 + self.feed_forward_ve = InternLM2MLP(
52 + hidden_size=self.hidden_size,
53 + intermediate_size=config.intermediate_size,
54 + hidden_act=config.hidden_act,
55 + quant_config=quant_config,
56 + )
57 + self.attention_norm = RMSNorm(config.hidden_size,
58 + eps=config.rms_norm_eps)
59 + self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
60 +
61 + def forward(
62 + self,
63 + positions: torch.Tensor,
64 + hidden_states: torch.Tensor,
65 + kv_cache: torch.Tensor,
66 + attn_metadata: AttentionMetadata,
67 + residual: Optional[torch.Tensor],
68 + visual_token_mask: Optional[torch.Tensor] = None,
69 + ) -> Tuple[torch.Tensor, torch.Tensor]:
70 + # Self Attention
71 + if residual is None:
72 + residual = hidden_states
73 + hidden_states = self.attention_norm(hidden_states)
74 + else:
75 + hidden_states, residual = self.attention_norm(
76 + hidden_states, residual)
77 + hidden_states = self.attention(
78 + positions=positions,
79 + hidden_states=hidden_states,
80 + kv_cache=kv_cache,
81 + attn_metadata=attn_metadata,
82 + )
83 +
84 + # Fully Connected
85 + hidden_states, residual = self.ffn_norm(hidden_states, residual)
86 + if visual_token_mask is not None and visual_token_mask.any():
87 + visual_token_mask = visual_token_mask.repeat(
88 + 1, self.hidden_size).bool()
89 + text_token_mask = ~visual_token_mask
90 + hidden_states[visual_token_mask] = self.feed_forward_ve(
91 + hidden_states[visual_token_mask].reshape(
92 + -1, self.hidden_size)).flatten()
93 + if text_token_mask.any():
94 + hidden_states[text_token_mask] = self.feed_forward(
95 + hidden_states[text_token_mask].reshape(
96 + -1, self.hidden_size)).flatten()
97 + else:
98 + hidden_states = self.feed_forward(hidden_states)
99 + return hidden_states, residual
100 +
101 +
102 +class InternLM2VEModel(InternLM2Model):
103 +
104 + def __init__(
105 + self,
106 + config: PretrainedConfig,
107 + cache_config: Optional[CacheConfig] = None,
108 + quant_config: Optional[QuantizationConfig] = None,
109 + prefix: str = "",
110 + ) -> None:
111 + super().__init__(config, cache_config, quant_config)
112 + self.start_layer, self.end_layer, self.layers = make_layers(
113 + config.num_hidden_layers,
114 + lambda prefix: InternLM2VEDecoderLayer(config, cache_config,
115 + quant_config),
116 + prefix=f"{prefix}.layers")
117 +
118 + def forward(
119 + self,
120 + input_ids: torch.Tensor,
121 + positions: torch.Tensor,
122 + kv_caches: List[torch.Tensor],
123 + attn_metadata: AttentionMetadata,
124 + intermediate_tensors: Optional[IntermediateTensors] = None,
125 + inputs_embeds: Optional[torch.Tensor] = None,
126 + visual_token_mask: Optional[torch.Tensor] = None,
127 + ) -> Union[torch.Tensor, IntermediateTensors]:
128 + if get_pp_group().is_first_rank:
129 + if inputs_embeds is not None:
130 + hidden_states = inputs_embeds
131 + else:
132 + hidden_states = self.tok_embeddings(input_ids)
133 + residual = None
134 + else:
135 + assert intermediate_tensors is not None
136 + hidden_states = intermediate_tensors["hidden_states"]
137 + residual = intermediate_tensors["residual"]
138 + for i in range(self.start_layer, self.end_layer):
139 + layer = self.layers[i]
140 + hidden_states, residual = layer(
141 + positions,
142 + hidden_states,
143 + kv_caches[i - self.start_layer],
144 + attn_metadata,
145 + residual,
146 + visual_token_mask=visual_token_mask,
147 + )
148 + if not get_pp_group().is_last_rank:
149 + return IntermediateTensors({
150 + "hidden_states": hidden_states,
151 + "residual": residual
152 + })
153 + hidden_states, _ = self.norm(hidden_states, residual)
154 + return hidden_states
155 +
156 +
157 +class InternLM2VEForCausalLM(InternLM2ForCausalLM):
158 +
159 + def __init__(
160 + self,
161 + config: PretrainedConfig,
162 + cache_config: Optional[CacheConfig] = None,
163 + quant_config: Optional[QuantizationConfig] = None,
164 + ) -> None:
165 + super().__init__(config, cache_config, quant_config)
166 + self.model = InternLM2VEModel(config, cache_config, quant_config)
vllm/model_executor/models/internvl.py
+42 -19
@@ -21,7 +21,8 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21 token_inputs)
22 from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24 -from vllm.model_executor.models.intern_vit import InternVisionModel
24 +from vllm.model_executor.models.intern_vit import (InternVisionModel,
25 + InternVisionPatchModel)
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.multimodal import MULTIMODAL_REGISTRY
28 from vllm.multimodal.base import MultiModalInputs
@@ -427,13 +428,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
428 self.downsample_ratio = config.downsample_ratio
429 self.ps_version = config.ps_version
430
430 - vision_feature_layer = self.select_layer
431 - if vision_feature_layer < 0:
432 - num_hidden_layers = config.vision_config.num_hidden_layers \
433 - + vision_feature_layer + 1
434 - else:
435 - num_hidden_layers = vision_feature_layer + 1
436 - self.vision_model = self._init_vision_model(config, num_hidden_layers)
431 + self.llm_arch_name = config.text_config.architectures[0]
432 + self.is_mono = self.llm_arch_name == 'InternLM2VEForCausalLM'
433 + self.vision_model = self._init_vision_model(config, self.is_mono)
434
435 self.language_model = init_vllm_registered_model(
436 config.text_config, cache_config, quant_config)
@@ -451,10 +448,19 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
448
449 return Sampler()
450
454 - def _init_vision_model(self, config: PretrainedConfig,
455 - num_hidden_layers: int):
456 - return InternVisionModel(config.vision_config,
457 - num_hidden_layers_override=num_hidden_layers)
451 + def _init_vision_model(self, config: PretrainedConfig, is_mono: bool):
452 + if not is_mono:
453 + vision_feature_layer = self.select_layer
454 + if vision_feature_layer < 0:
455 + num_hidden_layers = config.vision_config.num_hidden_layers \
456 + + vision_feature_layer + 1
457 + else:
458 + num_hidden_layers = vision_feature_layer + 1
459 + return InternVisionModel(
460 + config.vision_config,
461 + num_hidden_layers_override=num_hidden_layers)
462 + else:
463 + return InternVisionPatchModel(config.vision_config)
464
465 def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
466 vit_hidden_size = config.vision_config.hidden_size
@@ -562,6 +568,14 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
568
569 return image_embeds
570
571 + def _get_visual_token_mask(self, input_ids: torch.Tensor) -> torch.Tensor:
572 + if self.is_mono:
573 + visual_token_mask = (
574 + input_ids == self.img_context_token_id).reshape(-1, 1)
575 + else:
576 + visual_token_mask = None
577 + return visual_token_mask
578 +
579 def forward(
580 self,
581 input_ids: torch.Tensor,
@@ -574,6 +588,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
588 if intermediate_tensors is not None:
589 input_ids = None
590 inputs_embeds = None
591 + visual_token_mask = None
592 else:
593 image_input = self._parse_and_validate_image_input(**kwargs)
594 if image_input is not None:
@@ -583,16 +598,24 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
598 inputs_embeds = merge_multimodal_embeddings(
599 input_ids, inputs_embeds, vision_embeddings,
600 self.img_context_token_id)
601 + visual_token_mask = self._get_visual_token_mask(input_ids)
602 input_ids = None
603 else:
604 inputs_embeds = None
589 -
590 - hidden_states = self.language_model.model(input_ids,
591 - positions,
592 - kv_caches,
593 - attn_metadata,
594 - intermediate_tensors,
595 - inputs_embeds=inputs_embeds)
605 + visual_token_mask = None
606 +
607 + forward_kwargs = {
608 + "input_ids": input_ids,
609 + "positions": positions,
610 + "kv_caches": kv_caches,
611 + "attn_metadata": attn_metadata,
612 + "intermediate_tensors": intermediate_tensors,
613 + "inputs_embeds": inputs_embeds,
614 + }
615 + if self.is_mono:
616 + forward_kwargs.update({"visual_token_mask": visual_token_mask})
617 +
618 + hidden_states = self.language_model.model(**forward_kwargs)
619 return hidden_states
620
621 def compute_logits(
vllm/model_executor/models/registry.py
+1
@@ -47,6 +47,7 @@ _TEXT_GENERATION_MODELS = {
47 "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
48 "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
49 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
50 + "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
51 "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
52 "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
53 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),