A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] Initialize support for InternVL2 series models (#6514)
Co-authored-by: Roger Wang <ywang@roblox.com>
Isotr0py committed
Jul 29, 2024 at 18:16 UTC
7cbd9ec7a9bfd4952ad522355b6bbb8e82b54fc9
14 files changed
+1042
-6
docs/source/models/supported_models.rst
+4
@@ -200,6 +200,10 @@ Vision Language Models
200
- Fuyu
201
- :code:`adept/fuyu-8b` etc.
202
-
203
+ * - :code:`InternVLChatModel`
204
+ - InternVL2
205
+ - :code:`OpenGVLab/InternVL2-4B`, :code:`OpenGVLab/InternVL2-8B`, etc.
206
+ -
207
* - :code:`LlavaForConditionalGeneration`
208
- LLaVA-1.5
209
- :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
examples/offline_inference_vision_language.py
+15
@@ -106,6 +106,20 @@ def run_minicpmv(question):
106
return llm, prompt
107
108
109
+# InternVL
110
+def run_internvl(question):
111
+ # Generally, InternVL can use chatml template for conversation
112
+ TEMPLATE = "<|im_start|>User\n{prompt}<|im_end|>\n<|im_start|>Assistant\n"
113
+ prompt = f"<image>\n{question}\n"
114
+ prompt = TEMPLATE.format(prompt=prompt)
115
+ llm = LLM(
116
+ model="OpenGVLab/InternVL2-4B",
117
+ trust_remote_code=True,
118
+ max_num_seqs=5,
119
+ )
120
+ return llm, prompt
121
+
122
+
123
# BLIP-2
124
def run_blip2(question):
125
@@ -125,6 +139,7 @@ model_example_map = {
139
"chameleon": run_chameleon,
140
"minicpmv": run_minicpmv,
141
"blip-2": run_blip2,
142
+ "internvl_chat": run_internvl,
143
}
144
145
examples/openai_vision_api_client.py
+2
@@ -42,6 +42,7 @@ chat_completion_from_url = client.chat.completions.create(
42
],
43
}],
44
model=model,
45
+ max_tokens=64,
46
)
47
48
result = chat_completion_from_url.choices[0].message.content
@@ -78,6 +79,7 @@ chat_completion_from_base64 = client.chat.completions.create(
79
],
80
}],
81
model=model,
82
+ max_tokens=64,
83
)
84
85
result = chat_completion_from_base64.choices[0].message.content
requirements-test.txt
+1
@@ -16,6 +16,7 @@ ray
16
sentence-transformers # required for embedding
17
sparseml==1.8.0 # required for compressed-tensors
18
compressed-tensors==0.4.0 # required for compressed-tensors
19
+timm # required for internvl test
20
21
# Benchmarking
22
aiohttp
tests/models/test_internvl.py
new
+201
@@ -0,0 +1,201 @@
1
+import types
2
+from typing import List, Optional, Type
3
+
4
+import pytest
5
+import torch
6
+from huggingface_hub import snapshot_download
7
+from PIL.Image import Image
8
+
9
+from vllm.model_executor.models.internvl import (IMG_CONTEXT, IMG_END,
10
+ IMG_START,
11
+ image_to_pixel_values)
12
+from vllm.multimodal.utils import rescale_image_size
13
+from vllm.utils import is_cpu
14
+
15
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
16
+from .utils import check_logprobs_close
17
+
18
+pytestmark = pytest.mark.vlm
19
+
20
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21
+ "stop_sign":
22
+ "<|im_start|>User\n<image>\nWhat's the content in the center of the image?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
23
+ "cherry_blossom":
24
+ "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
25
+})
26
+
27
+# we use snapshot_download to prevent conflicts between
28
+# dynamic_module and trust_remote_code for hf_runner
29
+models = [
30
+ snapshot_download("OpenGVLab/InternVL2-1B"),
31
+ snapshot_download("OpenGVLab/InternVL2-2B"),
32
+ # snapshot_download("OpenGVLab/InternVL2-4B"), # broken
33
+]
34
+
35
+
36
+class InternVLProcessor:
37
+ """A simple processor for InternVL2 HF model which misses a processor."""
38
+
39
+ def __init__(self, hf_runner: HfRunner):
40
+ self.num_image_token = hf_runner.model.num_image_token
41
+ self.tokenizer = hf_runner.tokenizer
42
+ self.dtype = hf_runner.model.dtype
43
+
44
+ def __call__(self, text: str, images: Image, **kwargs):
45
+ pixel_values = image_to_pixel_values(images).to(self.dtype)
46
+ num_patches_list = [pixel_values.shape[0]]
47
+ for num_patches in num_patches_list:
48
+ context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
49
+ image_tokens = IMG_START + context_tokens + IMG_END
50
+ text = text.replace('<image>', image_tokens, 1)
51
+ prompt = self.tokenizer(text, return_tensors="pt")
52
+ prompt.update({"pixel_values": pixel_values})
53
+ return prompt
54
+
55
+
56
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py
57
+def generate(
58
+ self,
59
+ pixel_values: torch.FloatTensor,
60
+ input_ids: torch.FloatTensor,
61
+ attention_mask: Optional[torch.LongTensor] = None,
62
+ **generate_kwargs,
63
+) -> torch.LongTensor:
64
+ """Generate method for InternVL2 model without fixed use_cache."""
65
+ assert self.img_context_token_id is not None
66
+ vit_embeds = self.extract_feature(pixel_values)
67
+ input_embeds = self.language_model.get_input_embeddings()(input_ids)
68
+ B, N, C = input_embeds.shape
69
+ input_embeds = input_embeds.reshape(B * N, C)
70
+
71
+ input_ids = input_ids.reshape(B * N)
72
+ selected = (input_ids == self.img_context_token_id)
73
+ assert selected.sum() != 0
74
+ input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
75
+
76
+ input_embeds = input_embeds.reshape(B, N, C)
77
+
78
+ outputs = self.language_model.generate(
79
+ inputs_embeds=input_embeds,
80
+ attention_mask=attention_mask,
81
+ **generate_kwargs,
82
+ )
83
+
84
+ return outputs
85
+
86
+
87
+def run_test(
88
+ hf_runner: Type[HfRunner],
89
+ vllm_runner: Type[VllmRunner],
90
+ image_assets: _ImageAssets,
91
+ model: str,
92
+ *,
93
+ size_factors: List[float],
94
+ dtype: str,
95
+ max_tokens: int,
96
+ num_logprobs: int,
97
+ tensor_parallel_size: int,
98
+ distributed_executor_backend: Optional[str] = None,
99
+):
100
+ """Inference result should be the same between hf and vllm.
101
+
102
+ All the image fixtures for the test is under tests/images.
103
+ For huggingface runner, we provide the PIL images as input.
104
+ For vllm runner, we provide MultiModalDataDict objects
105
+ and corresponding vision language config as input.
106
+ Note, the text input is also adjusted to abide by vllm contract.
107
+ The text output is sanitized to be able to compare with hf.
108
+ """
109
+ images = [asset.pil_image for asset in image_assets]
110
+
111
+ inputs_per_image = [(
112
+ [prompt for _ in size_factors],
113
+ [rescale_image_size(image, factor) for factor in size_factors],
114
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
115
+
116
+ # NOTE: take care of the order. run vLLM first, and then run HF.
117
+ # vLLM needs a fresh new process without cuda initialization.
118
+ # if we run HF first, the cuda initialization will be done and it
119
+ # will hurt multiprocessing backend with fork method (the default method).
120
+
121
+ # max_model_len should be greater than image_feature_size
122
+ with vllm_runner(model,
123
+ max_model_len=4096,
124
+ dtype=dtype,
125
+ tensor_parallel_size=tensor_parallel_size,
126
+ distributed_executor_backend=distributed_executor_backend,
127
+ enforce_eager=True) as vllm_model:
128
+ vllm_outputs_per_image = [
129
+ vllm_model.generate_greedy_logprobs(prompts,
130
+ max_tokens,
131
+ num_logprobs=num_logprobs,
132
+ images=images)
133
+ for prompts, images in inputs_per_image
134
+ ]
135
+
136
+ with hf_runner(model, dtype=dtype) as hf_model:
137
+ img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids(
138
+ "<IMG_CONTEXT>")
139
+ hf_model.model.img_context_token_id = img_context_token_id
140
+ hf_model.processor = InternVLProcessor(hf_model)
141
+ hf_model.model.get_output_embeddings = lambda: \
142
+ hf_model.model.language_model.get_output_embeddings()
143
+ hf_model.model.generate = types.MethodType(generate, hf_model.model)
144
+ eos_token_id = hf_model.tokenizer.eos_token_id
145
+ hf_outputs_per_image = [
146
+ hf_model.generate_greedy_logprobs_limit(prompts,
147
+ max_tokens,
148
+ num_logprobs=num_logprobs,
149
+ images=hf_images,
150
+ eos_token_id=eos_token_id)
151
+ for prompts, hf_images in inputs_per_image
152
+ ]
153
+
154
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
155
+ vllm_outputs_per_image):
156
+ # TODO: Check whether using original CLIPVisionModel can improve
157
+ # consistency against HF
158
+ check_logprobs_close(
159
+ outputs_0_lst=hf_outputs,
160
+ outputs_1_lst=vllm_outputs,
161
+ name_0="hf",
162
+ name_1="vllm",
163
+ )
164
+
165
+
166
+target_dtype = "half"
167
+if is_cpu():
168
+ target_dtype = "bfloat16"
169
+
170
+
171
+@pytest.mark.parametrize("model", models)
172
+@pytest.mark.parametrize(
173
+ "size_factors",
174
+ [
175
+ # No image
176
+ [],
177
+ # Single-scale
178
+ [1.0],
179
+ # Single-scale, batched
180
+ [1.0, 1.0, 1.0],
181
+ # Multi-scale
182
+ [0.25, 0.5, 1.0],
183
+ ],
184
+)
185
+@pytest.mark.parametrize("dtype", [target_dtype])
186
+@pytest.mark.parametrize("max_tokens", [128])
187
+@pytest.mark.parametrize("num_logprobs", [5])
188
+@torch.inference_mode()
189
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
190
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
191
+ run_test(
192
+ hf_runner,
193
+ vllm_runner,
194
+ image_assets,
195
+ model,
196
+ size_factors=size_factors,
197
+ dtype=dtype,
198
+ max_tokens=max_tokens,
199
+ num_logprobs=num_logprobs,
200
+ tensor_parallel_size=1,
201
+ )
vllm/entrypoints/chat_utils.py
+1
-1
@@ -107,7 +107,7 @@ def _image_token_str(model_config: ModelConfig,
107
return None
108
if model_type.startswith("llava"):
109
return tokenizer.decode(model_config.hf_config.image_token_index)
110
- if model_type == "chameleon":
110
+ if model_type in ("chameleon", "internvl_chat"):
111
return "<image>"
112
raise TypeError(f"Unknown model type: {model_type}")
113
vllm/model_executor/models/__init__.py
+1
@@ -37,6 +37,7 @@ _GENERATION_MODELS = {
37
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
38
"InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
39
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
40
+ "InternVLChatModel": ("internvl", "InternVLChatModel"),
41
"JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
42
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
43
"LlavaForConditionalGeneration":
vllm/model_executor/models/intern_vit.py
new
+270
@@ -0,0 +1,270 @@
1
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py
2
+# --------------------------------------------------------
3
+# InternVL
4
+# Copyright (c) 2023 OpenGVLab
5
+# Licensed under The MIT License [see LICENSE for details]
6
+# --------------------------------------------------------
7
+from typing import Optional
8
+
9
+import torch
10
+import torch.nn as nn
11
+import torch.nn.functional as F
12
+from transformers import PretrainedConfig
13
+
14
+from vllm.model_executor.layers.activation import get_act_fn
15
+from vllm.model_executor.layers.layernorm import RMSNorm
16
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
+ RowParallelLinear)
18
+from vllm.model_executor.layers.quantization import QuantizationConfig
19
+
20
+NORM2FN = {
21
+ 'rms_norm': RMSNorm,
22
+ 'layer_norm': nn.LayerNorm,
23
+}
24
+
25
+
26
+class InternVisionEmbeddings(nn.Module):
27
+
28
+ def __init__(self, config: PretrainedConfig):
29
+ super().__init__()
30
+ self.config = config
31
+ self.embed_dim = config.hidden_size
32
+ self.image_size = config.image_size
33
+ self.patch_size = config.patch_size
34
+
35
+ self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim))
36
+
37
+ self.patch_embedding = nn.Conv2d(in_channels=3,
38
+ out_channels=self.embed_dim,
39
+ kernel_size=self.patch_size,
40
+ stride=self.patch_size)
41
+
42
+ self.num_patches = (self.image_size // self.patch_size)**2
43
+ self.num_positions = self.num_patches + 1
44
+
45
+ self.position_embedding = nn.Parameter(
46
+ torch.randn(1, self.num_positions, self.embed_dim))
47
+
48
+ def _get_pos_embed(self, pos_embed, H, W):
49
+ target_dtype = pos_embed.dtype
50
+ pos_embed = pos_embed.float().reshape(
51
+ 1, self.image_size // self.patch_size,
52
+ self.image_size // self.patch_size, -1).permute(0, 3, 1, 2)
53
+ pos_embed = F.interpolate(pos_embed,
54
+ size=(H, W),
55
+ mode='bicubic',
56
+ align_corners=False)
57
+ pos_embed = pos_embed.reshape(1, -1, H * W).permute(0, 2,
58
+ 1).to(target_dtype)
59
+ return pos_embed
60
+
61
+ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
62
+ target_dtype = self.patch_embedding.weight.dtype
63
+ patch_embeds = self.patch_embedding(pixel_values.to(
64
+ target_dtype)) # shape = [*, channel, width, height]
65
+ batch_size, _, height, width = patch_embeds.shape
66
+ patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
67
+ class_embeds = self.class_embedding.expand(batch_size, 1,
68
+ -1).to(target_dtype)
69
+ embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
70
+ position_embedding = torch.cat([
71
+ self.position_embedding[:, :1, :],
72
+ self._get_pos_embed(self.position_embedding[:, 1:, :], height,
73
+ width)
74
+ ],
75
+ dim=1)
76
+ embeddings = embeddings + position_embedding.to(target_dtype)
77
+ return embeddings
78
+
79
+
80
+class InternAttention(nn.Module):
81
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
82
+
83
+ def __init__(self, config: PretrainedConfig):
84
+ super().__init__()
85
+ self.config = config
86
+ self.embed_dim = config.hidden_size
87
+ self.num_heads = config.num_attention_heads
88
+ self.head_dim = self.embed_dim // self.num_heads
89
+ if self.head_dim * self.num_heads != self.embed_dim:
90
+ raise ValueError(
91
+ f'embed_dim must be divisible by num_heads '
92
+ f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
93
+ f' {self.num_heads}).')
94
+
95
+ self.scale = self.head_dim**-0.5
96
+ self.qkv = nn.Linear(self.embed_dim,
97
+ 3 * self.embed_dim,
98
+ bias=config.qkv_bias)
99
+
100
+ self.qk_normalization = config.qk_normalization
101
+
102
+ if self.qk_normalization:
103
+ self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
104
+ self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
105
+
106
+ self.proj = nn.Linear(self.embed_dim, self.embed_dim)
107
+
108
+ def forward(self, x):
109
+ B, N, C = x.shape
110
+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads,
111
+ C // self.num_heads).permute(2, 0, 3, 1, 4)
112
+ q, k, v = qkv.unbind(0)
113
+
114
+ if self.qk_normalization:
115
+ B_, H_, N_, D_ = q.shape
116
+ q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(
117
+ B_, N_, H_, D_).transpose(1, 2)
118
+ k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(
119
+ B_, N_, H_, D_).transpose(1, 2)
120
+
121
+ x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
122
+ x = x.transpose(1, 2).reshape(B, N, C)
123
+
124
+ x = self.proj(x)
125
+ return x
126
+
127
+
128
+class InternMLP(nn.Module):
129
+
130
+ def __init__(self,
131
+ config: PretrainedConfig,
132
+ quant_config: Optional[QuantizationConfig] = None):
133
+ super().__init__()
134
+ self.config = config
135
+ self.activation_fn = get_act_fn(config.hidden_act)
136
+ self.fc1 = ColumnParallelLinear(config.hidden_size,
137
+ config.intermediate_size,
138
+ bias=True,
139
+ quant_config=quant_config)
140
+ self.fc2 = RowParallelLinear(config.intermediate_size,
141
+ config.hidden_size,
142
+ bias=True,
143
+ quant_config=quant_config)
144
+
145
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
146
+ hidden_states, _ = self.fc1(hidden_states)
147
+ hidden_states = self.activation_fn(hidden_states)
148
+ hidden_states, _ = self.fc2(hidden_states)
149
+
150
+ return hidden_states
151
+
152
+
153
+class InternVisionEncoderLayer(nn.Module):
154
+
155
+ def __init__(self,
156
+ config: PretrainedConfig,
157
+ quant_config: Optional[QuantizationConfig] = None):
158
+ super().__init__()
159
+ self.embed_dim = config.hidden_size
160
+ self.intermediate_size = config.intermediate_size
161
+ self.norm_type = config.norm_type
162
+
163
+ self.attn = InternAttention(config)
164
+ self.mlp = InternMLP(config, quant_config=quant_config)
165
+ self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
166
+ eps=config.layer_norm_eps)
167
+ self.norm2 = NORM2FN[self.norm_type](self.embed_dim,
168
+ eps=config.layer_norm_eps)
169
+
170
+ self.ls1 = nn.Parameter(config.initializer_factor *
171
+ torch.ones(self.embed_dim))
172
+ self.ls2 = nn.Parameter(config.initializer_factor *
173
+ torch.ones(self.embed_dim))
174
+
175
+ def forward(
176
+ self,
177
+ hidden_states: torch.Tensor,
178
+ ):
179
+ hidden_states = hidden_states + self.attn(
180
+ self.norm1(hidden_states)) * self.ls1
181
+
182
+ hidden_states = hidden_states + self.mlp(
183
+ self.norm2(hidden_states)) * self.ls2
184
+
185
+ return hidden_states
186
+
187
+
188
+class InternVisionEncoder(nn.Module):
189
+
190
+ def __init__(self,
191
+ config: PretrainedConfig,
192
+ quant_config: Optional[QuantizationConfig] = None,
193
+ num_hidden_layers_override: Optional[int] = None):
194
+ super().__init__()
195
+ self.config = config
196
+
197
+ if num_hidden_layers_override is None:
198
+ num_hidden_layers = config.num_hidden_layers
199
+ else:
200
+ num_hidden_layers = num_hidden_layers_override
201
+ self.layers = nn.ModuleList([
202
+ InternVisionEncoderLayer(config=config, quant_config=quant_config)
203
+ for _ in range(num_hidden_layers)
204
+ ])
205
+
206
+ def forward(self, inputs_embeds: torch.Tensor):
207
+
208
+ hidden_states = inputs_embeds
209
+ for encoder_layer in self.layers:
210
+ hidden_states = encoder_layer(hidden_states)
211
+
212
+ return hidden_states
213
+
214
+
215
+class InternVisionModel(nn.Module):
216
+
217
+ def __init__(self,
218
+ config: PretrainedConfig,
219
+ quant_config: Optional[QuantizationConfig] = None,
220
+ num_hidden_layers_override: Optional[int] = None):
221
+ super().__init__()
222
+ self.config = config
223
+
224
+ self.embeddings = InternVisionEmbeddings(config)
225
+ self.encoder = InternVisionEncoder(
226
+ config=config,
227
+ quant_config=quant_config,
228
+ num_hidden_layers_override=num_hidden_layers_override)
229
+
230
+ def resize_pos_embeddings(self, old_size, new_size, patch_size):
231
+ pos_emb = self.embeddings.position_embedding
232
+ _, num_positions, embed_dim = pos_emb.shape
233
+ cls_emb = pos_emb[:, :1, :]
234
+ pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size,
235
+ old_size // patch_size,
236
+ -1).permute(0, 3, 1, 2)
237
+ pos_emb = F.interpolate(pos_emb.float(),
238
+ size=new_size // patch_size,
239
+ mode='bicubic',
240
+ align_corners=False)
241
+ pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim,
242
+ -1).permute(0, 2, 1)
243
+ pos_emb = torch.cat([cls_emb, pos_emb], dim=1)
244
+ self.embeddings.position_embedding = nn.Parameter(pos_emb)
245
+ self.embeddings.image_size = new_size
246
+
247
+ def get_input_embeddings(self):
248
+ return self.embeddings
249
+
250
+ def forward(
251
+ self,
252
+ pixel_values: Optional[torch.Tensor] = None,
253
+ pixel_embeds: Optional[torch.Tensor] = None,
254
+ ) -> torch.FloatTensor:
255
+ if pixel_values is None and pixel_embeds is None:
256
+ raise ValueError(
257
+ 'You have to specify pixel_values or pixel_embeds')
258
+
259
+ if pixel_embeds is not None:
260
+ hidden_states = pixel_embeds
261
+ elif pixel_values is not None:
262
+ if pixel_values.ndim == 4:
263
+ hidden_states = self.embeddings(pixel_values)
264
+ else:
265
+ raise ValueError(
266
+ f'wrong pixel_values size: {pixel_values.shape}')
267
+
268
+ encoder_outputs = self.encoder(inputs_embeds=hidden_states)
269
+
270
+ return encoder_outputs
vllm/model_executor/models/internlm2.py
+9
-1
@@ -219,14 +219,22 @@ class InternLM2Model(nn.Module):
219
])
220
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
221
222
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
223
+ return self.tok_embeddings(input_ids)
224
+
225
def forward(
226
self,
227
input_ids: torch.Tensor,
228
positions: torch.Tensor,
229
kv_caches: List[torch.Tensor],
230
attn_metadata: AttentionMetadata,
231
+ intermediate_tensors: IntermediateTensors = None,
232
+ inputs_embeds: Optional[torch.Tensor] = None,
233
) -> torch.Tensor:
229
- hidden_states = self.tok_embeddings(input_ids)
234
+ if inputs_embeds is not None:
235
+ hidden_states = inputs_embeds
236
+ else:
237
+ hidden_states = self.tok_embeddings(input_ids)
238
residual = None
239
for i in range(len(self.layers)):
240
layer = self.layers[i]
vllm/model_executor/models/internvl.py
new
+471
@@ -0,0 +1,471 @@
1
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py
2
+# --------------------------------------------------------
3
+# InternVL
4
+# Copyright (c) 2023 OpenGVLab
5
+# Licensed under The MIT License [see LICENSE for details]
6
+# --------------------------------------------------------
7
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
8
+
9
+import torch
10
+import torch.nn as nn
11
+import torchvision.transforms as T
12
+from PIL import Image
13
+from transformers import PretrainedConfig
14
+
15
+from vllm.attention import AttentionMetadata
16
+from vllm.config import CacheConfig, MultiModalConfig
17
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
18
+from vllm.model_executor.layers.quantization import QuantizationConfig
19
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
+from vllm.model_executor.models import ModelRegistry
21
+from vllm.model_executor.models.intern_vit import InternVisionModel
22
+from vllm.model_executor.sampling_metadata import SamplingMetadata
23
+from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensors
24
+from vllm.multimodal.base import MultiModalInputs
25
+from vllm.multimodal.image import cached_get_tokenizer
26
+from vllm.sequence import IntermediateTensors, SamplerOutput
27
+
28
+from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
29
+ get_clip_num_patches)
30
+from .interfaces import SupportsVision
31
+from .utils import merge_vision_embeddings
32
+
33
+IMG_START = '<img>'
34
+IMG_END = '</img>'
35
+IMG_CONTEXT = '<IMG_CONTEXT>'
36
+
37
+IMAGENET_MEAN = (0.485, 0.456, 0.406)
38
+IMAGENET_STD = (0.229, 0.224, 0.225)
39
+
40
+MAX_IMAGE_FEATURE_SIZE_WIDTH = 3000
41
+MAX_IMAGE_FEATURE_SIZE_HEIGHT = 500
42
+
43
+
44
+class InternVLImagePixelInputs(TypedDict):
45
+ type: Literal["pixel_values"]
46
+ data: BatchedTensors
47
+ """
48
+ Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
49
+
50
+ Note that `num_patches` may be different for each batch, in which case
51
+ the data is passed as a list instead of a batched tensor.
52
+ """
53
+
54
+
55
+# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
56
+def build_transform(input_size):
57
+ MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
58
+ transform = T.Compose([
59
+ T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
60
+ T.Resize((input_size, input_size),
61
+ interpolation=T.InterpolationMode.BICUBIC),
62
+ T.ToTensor(),
63
+ T.Normalize(mean=MEAN, std=STD)
64
+ ])
65
+ return transform
66
+
67
+
68
+# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
69
+def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
70
+ image_size):
71
+ best_ratio_diff = float('inf')
72
+ best_ratio = (1, 1)
73
+ area = width * height
74
+ for ratio in target_ratios:
75
+ target_aspect_ratio = ratio[0] / ratio[1]
76
+ ratio_diff = abs(aspect_ratio - target_aspect_ratio)
77
+ if ratio_diff < best_ratio_diff:
78
+ best_ratio_diff = ratio_diff
79
+ best_ratio = ratio
80
+ elif ratio_diff == best_ratio_diff:
81
+ if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
82
+ best_ratio = ratio
83
+ return best_ratio
84
+
85
+
86
+def calculate_num_blocks(orig_width: int,
87
+ orig_height: int,
88
+ min_num=1,
89
+ max_num=6,
90
+ image_size=448):
91
+ aspect_ratio = orig_width / orig_height
92
+
93
+ # calculate the existing image aspect ratio
94
+ target_ratios = set((i, j) for n in range(min_num, max_num + 1)
95
+ for i in range(1, n + 1) for j in range(1, n + 1)
96
+ if i * j <= max_num and i * j >= min_num)
97
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
98
+
99
+ # find the closest aspect ratio to the target
100
+ target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio,
101
+ target_ratios, orig_width,
102
+ orig_height, image_size)
103
+
104
+ # calculate the target width and height
105
+ target_width = image_size * target_aspect_ratio[0]
106
+ target_height = image_size * target_aspect_ratio[1]
107
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
108
+ return blocks, target_width, target_height
109
+
110
+
111
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
112
+def dynamic_preprocess(image,
113
+ min_num=1,
114
+ max_num=6,
115
+ image_size=448,
116
+ use_thumbnail=False):
117
+ orig_width, orig_height = image.size
118
+
119
+ blocks, target_width, target_height = calculate_num_blocks(
120
+ orig_width, orig_height, min_num, max_num, image_size)
121
+ # resize the image
122
+ resized_img = image.resize((target_width, target_height))
123
+ processed_images = []
124
+ for i in range(blocks):
125
+ box = ((i % (target_width // image_size)) * image_size,
126
+ (i // (target_width // image_size)) * image_size,
127
+ ((i % (target_width // image_size)) + 1) * image_size,
128
+ ((i // (target_width // image_size)) + 1) * image_size)
129
+ # split the image
130
+ split_img = resized_img.crop(box)
131
+ processed_images.append(split_img)
132
+ assert len(processed_images) == blocks
133
+ if use_thumbnail and len(processed_images) != 1:
134
+ thumbnail_img = image.resize((image_size, image_size))
135
+ processed_images.append(thumbnail_img)
136
+ return processed_images
137
+
138
+
139
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
140
+def image_to_pixel_values(image: Image.Image, input_size=448, max_num=6):
141
+ transform = build_transform(input_size=input_size)
142
+ images = dynamic_preprocess(image,
143
+ image_size=input_size,
144
+ use_thumbnail=True,
145
+ max_num=max_num)
146
+ pixel_values = [transform(image) for image in images]
147
+ pixel_values = torch.stack(pixel_values)
148
+ return pixel_values
149
+
150
+
151
+def get_internvl_num_patches(image_size: int, patch_size: int,
152
+ downsample_ratio: float):
153
+ return int(
154
+ get_clip_num_patches(image_size=image_size, patch_size=patch_size) *
155
+ (downsample_ratio**2))
156
+
157
+
158
+def get_max_internvl_image_tokens(ctx: InputContext):
159
+ hf_config = ctx.get_hf_config(PretrainedConfig)
160
+ vision_config = hf_config.vision_config
161
+ image_size = vision_config.image_size
162
+ patch_size = vision_config.patch_size
163
+ downsample_ratio = hf_config.downsample_ratio
164
+ num_patches = get_internvl_num_patches(image_size, patch_size,
165
+ downsample_ratio)
166
+ return num_patches * 7
167
+
168
+
169
+def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
170
+ multi_modal_data = llm_inputs.get("multi_modal_data")
171
+ if multi_modal_data is None or "image" not in multi_modal_data:
172
+ return llm_inputs
173
+
174
+ model_config = ctx.model_config
175
+ hf_config = ctx.get_hf_config(PretrainedConfig)
176
+ vision_config = hf_config.vision_config
177
+
178
+ image_data = multi_modal_data["image"]
179
+ if isinstance(image_data, Image.Image):
180
+ width, height = image_data.size
181
+ num_blocks, _, _ = calculate_num_blocks(width, height)
182
+ elif isinstance(image_data, torch.Tensor):
183
+ raise NotImplementedError("Embeddings input is not supported yet")
184
+ else:
185
+ raise TypeError(f"Invalid image type: {type(image_data)}")
186
+
187
+ image_size = vision_config.image_size
188
+ patch_size = vision_config.patch_size
189
+ downsample_ratio = hf_config.downsample_ratio
190
+ num_patches = get_internvl_num_patches(image_size, patch_size,
191
+ downsample_ratio)
192
+
193
+ tokenizer = cached_get_tokenizer(model_config.tokenizer,
194
+ trust_remote_code=True)
195
+
196
+ prompt = llm_inputs["prompt"]
197
+ prompt_token_ids = llm_inputs["prompt_token_ids"]
198
+ if prompt is None:
199
+ prompt = tokenizer.decode(prompt_token_ids)
200
+ image_prompt = IMG_START + IMG_CONTEXT * (num_blocks +
201
+ 1) * num_patches + IMG_END
202
+ new_prompt = prompt.replace('<image>', image_prompt, 1)
203
+ new_prompt_token_ids = tokenizer.encode(new_prompt)
204
+
205
+ return LLMInputs(prompt=prompt,
206
+ prompt_token_ids=new_prompt_token_ids,
207
+ multi_modal_data=multi_modal_data)
208
+
209
+
210
+def input_mapper_for_internvl(ctx: InputContext, data: object):
211
+ if isinstance(data, Image.Image):
212
+ data = image_to_pixel_values(data)
213
+ model_config = ctx.model_config
214
+ tokenizer = cached_get_tokenizer(model_config.tokenizer,
215
+ trust_remote_code=True)
216
+ image_token_id = tokenizer.encode(IMG_CONTEXT,
217
+ add_special_tokens=False,
218
+ return_tensors="pt")[0]
219
+
220
+ return MultiModalInputs({
221
+ "pixel_values": data,
222
+ "image_token_id": image_token_id
223
+ })
224
+
225
+
226
+def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
227
+
228
+ image_feature_size = get_max_internvl_image_tokens(ctx)
229
+ model_config = ctx.model_config
230
+ hf_config = ctx.get_hf_config(PretrainedConfig)
231
+ vision_config = hf_config.vision_config
232
+ tokenizer = cached_get_tokenizer(model_config.tokenizer,
233
+ trust_remote_code=True)
234
+
235
+ seq_data = dummy_seq_data_for_clip(
236
+ vision_config,
237
+ seq_len,
238
+ image_token_id=tokenizer.encode(IMG_CONTEXT,
239
+ add_special_tokens=False)[0],
240
+ image_feature_size_override=image_feature_size,
241
+ )
242
+ mm_data = dummy_image_for_clip(
243
+ vision_config,
244
+ image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
245
+ image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
246
+ )
247
+
248
+ return seq_data, mm_data
249
+
250
+
251
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_internvl)
252
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
253
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_internvl)
254
+@INPUT_REGISTRY.register_input_processor(input_processor_for_internvl)
255
+class InternVLChatModel(nn.Module, SupportsVision):
256
+
257
+ def __init__(self,
258
+ config: PretrainedConfig,
259
+ multimodal_config: MultiModalConfig,
260
+ cache_config: Optional[CacheConfig] = None,
261
+ quant_config: Optional[QuantizationConfig] = None) -> None:
262
+ super().__init__()
263
+
264
+ self.config = config
265
+ self.multimodal_config = multimodal_config
266
+
267
+ image_size = config.force_image_size or config.vision_config.image_size
268
+ patch_size = config.vision_config.patch_size
269
+ self.patch_size = patch_size
270
+ self.select_layer = config.select_layer
271
+ self.num_image_token = int(
272
+ (image_size // patch_size)**2 * (config.downsample_ratio**2))
273
+ self.downsample_ratio = config.downsample_ratio
274
+ self.ps_version = config.ps_version
275
+
276
+ vision_feature_layer = self.select_layer
277
+ if vision_feature_layer < 0:
278
+ num_hidden_layers = config.vision_config.num_hidden_layers \
279
+ + vision_feature_layer + 1
280
+ else:
281
+ num_hidden_layers = vision_feature_layer + 1
282
+ self.vision_model = InternVisionModel(
283
+ config.vision_config, num_hidden_layers_override=num_hidden_layers)
284
+
285
+ llm_class = ModelRegistry.load_model_cls(
286
+ config.text_config.architectures[0])
287
+ self.language_model = llm_class(config.text_config, cache_config,
288
+ quant_config)
289
+
290
+ vit_hidden_size = config.vision_config.hidden_size
291
+ llm_hidden_size = config.text_config.hidden_size
292
+
293
+ self.mlp1 = nn.Sequential(
294
+ nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
295
+ nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
296
+ llm_hidden_size), nn.GELU(),
297
+ nn.Linear(llm_hidden_size, llm_hidden_size))
298
+
299
+ self.img_context_token_id = None
300
+
301
+ def pixel_shuffle(self, x, scale_factor=0.5):
302
+ n, w, h, c = x.size()
303
+ # N, W, H, C --> N, W, H * scale, C // scale
304
+ x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
305
+ # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
306
+ x = x.permute(0, 2, 1, 3).contiguous()
307
+ x = x.view(n, int(h * scale_factor), int(w * scale_factor),
308
+ int(c / (scale_factor * scale_factor)))
309
+ if self.ps_version == 'v1':
310
+ pass
311
+ else:
312
+ x = x.permute(0, 2, 1, 3).contiguous()
313
+ return x
314
+
315
+ def extract_feature(self, pixel_values):
316
+ vit_embeds = self.vision_model(pixel_values=pixel_values)
317
+ vit_embeds = vit_embeds[:, 1:, :]
318
+
319
+ h = w = int(vit_embeds.shape[1]**0.5)
320
+ vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
321
+ vit_embeds = self.pixel_shuffle(vit_embeds,
322
+ scale_factor=self.downsample_ratio)
323
+ vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1,
324
+ vit_embeds.shape[-1])
325
+ vit_embeds = self.mlp1(vit_embeds)
326
+ return vit_embeds
327
+
328
+ def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
329
+ if list(data.shape[1:]) != [2]:
330
+ raise ValueError(
331
+ f"The expected image sizes shape is batch dimension plus "
332
+ f"{[2]}. You supplied {data.shape}.")
333
+
334
+ return data
335
+
336
+ def _validate_pixel_values(
337
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
338
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
339
+
340
+ h = w = self.config.vision_config.image_size
341
+ expected_dims = (3, h, w)
342
+
343
+ def _validate_shape(d: torch.Tensor):
344
+ actual_dims = tuple(d.shape)
345
+
346
+ if actual_dims != expected_dims:
347
+ expected_expr = ("num_patches", *map(str, expected_dims))
348
+ raise ValueError(
349
+ "The expected shape of pixel values in each batch element "
350
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
351
+
352
+ for d in data:
353
+ _validate_shape(d)
354
+
355
+ return data
356
+
357
+ def _parse_and_validate_image_input(
358
+ self, **kwargs: object) -> Optional[InternVLImagePixelInputs]:
359
+ pixel_values = kwargs.pop("pixel_values", None)
360
+ image_token_id = kwargs.pop("image_token_id", None)
361
+
362
+ if pixel_values is None:
363
+ return None
364
+
365
+ self.img_context_token_id = image_token_id[0]
366
+
367
+ if not isinstance(pixel_values, (torch.Tensor, list)):
368
+ raise ValueError("Incorrect type of pixel values. "
369
+ f"Got type: {type(pixel_values)}")
370
+
371
+ return InternVLImagePixelInputs(
372
+ type="pixel_values",
373
+ data=self._validate_pixel_values(pixel_values),
374
+ )
375
+
376
+ def forward(
377
+ self,
378
+ input_ids: torch.Tensor,
379
+ positions: torch.Tensor,
380
+ kv_caches: List[torch.Tensor],
381
+ attn_metadata: AttentionMetadata,
382
+ intermediate_tensors: Optional[IntermediateTensors] = None,
383
+ **kwargs: object,
384
+ ) -> SamplerOutput:
385
+ image_input = self._parse_and_validate_image_input(**kwargs)
386
+ if image_input is not None:
387
+ inputs_embeds = self.language_model.model.get_input_embeddings(
388
+ input_ids)
389
+ vit_embeds = self.extract_feature(image_input["data"])
390
+ inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
391
+ vit_embeds,
392
+ self.img_context_token_id)
393
+ input_ids = None
394
+ else:
395
+ inputs_embeds = None
396
+
397
+ hidden_states = self.language_model.model(input_ids,
398
+ positions,
399
+ kv_caches,
400
+ attn_metadata,
401
+ None,
402
+ inputs_embeds=inputs_embeds)
403
+ return hidden_states
404
+
405
+ def compute_logits(self, hidden_states: torch.Tensor,
406
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
407
+ return self.language_model.compute_logits(hidden_states,
408
+ sampling_metadata)
409
+
410
+ def sample(
411
+ self,
412
+ logits: torch.Tensor,
413
+ sampling_metadata: SamplingMetadata,
414
+ ) -> Optional[SamplerOutput]:
415
+ return self.language_model.sample(logits, sampling_metadata)
416
+
417
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
418
+ stacked_params_mapping = [
419
+ # (param_name, shard_name, shard_id)
420
+ (".qkv_proj", ".q_proj", "q"),
421
+ (".qkv_proj", ".k_proj", "k"),
422
+ (".qkv_proj", ".v_proj", "v"),
423
+ (".gate_up_proj", ".gate_proj", 0),
424
+ (".gate_up_proj", ".up_proj", 1),
425
+ (".gate_up_proj", ".w1", 0),
426
+ (".gate_up_proj", ".w3", 1),
427
+ ]
428
+ params_dict = dict(self.named_parameters())
429
+ for name, loaded_weight in weights:
430
+ if "rotary_emb.inv_freq" in name:
431
+ continue
432
+ if self.config.text_config.tie_word_embeddings \
433
+ and "lm_head.weight" in name:
434
+ continue
435
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
436
+ # We only do sharding for language model
437
+ # and not vision model for now.
438
+ if "vision_embed_tokens" in name and self.vision_embed_tokens:
439
+ continue
440
+ if weight_name not in name:
441
+ continue
442
+ param = params_dict[name.replace(weight_name, param_name)]
443
+ weight_loader = param.weight_loader
444
+ weight_loader(param, loaded_weight, shard_id)
445
+ break
446
+ else:
447
+ # Skip loading extra bias for GPTQ models.
448
+ if name.endswith(".bias") and name not in params_dict:
449
+ continue
450
+ param = params_dict[name]
451
+ if "wqkv" in name:
452
+ config = self.config.text_config
453
+ kv_groups = (config.num_attention_heads //
454
+ config.num_key_value_heads)
455
+ head_dim = config.hidden_size // config.num_attention_heads
456
+ loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
457
+ head_dim,
458
+ loaded_weight.shape[-1])
459
+ wq, wk, wv = torch.split(loaded_weight, [kv_groups, 1, 1],
460
+ dim=1)
461
+ wq = wq.reshape(-1, wq.shape[-1])
462
+ wk = wk.reshape(-1, wk.shape[-1])
463
+ wv = wv.reshape(-1, wv.shape[-1])
464
+ weight_loader = param.weight_loader
465
+ weight_loader(param, wq, 'q')
466
+ weight_loader(param, wk, 'k')
467
+ weight_loader(param, wv, 'v')
468
+ continue
469
+ weight_loader = getattr(param, "weight_loader",
470
+ default_weight_loader)
471
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen2.py
+9
-1
@@ -243,14 +243,22 @@ class Qwen2Model(nn.Module):
243
])
244
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
245
246
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
247
+ return self.embed_tokens(input_ids)
248
+
249
def forward(
250
self,
251
input_ids: torch.Tensor,
252
positions: torch.Tensor,
253
kv_caches: List[torch.Tensor],
254
attn_metadata: AttentionMetadata,
255
+ intermediate_tensors: Optional[IntermediateTensors] = None,
256
+ inputs_embeds: Optional[torch.Tensor] = None,
257
) -> torch.Tensor:
253
- hidden_states = self.embed_tokens(input_ids)
258
+ if inputs_embeds is not None:
259
+ hidden_states = inputs_embeds
260
+ else:
261
+ hidden_states = self.embed_tokens(input_ids)
262
residual = None
263
for i in range(len(self.layers)):
264
layer = self.layers[i]
vllm/transformers_utils/config.py
+5
-3
@@ -6,9 +6,10 @@ from transformers import GenerationConfig, PretrainedConfig
6
from vllm.envs import VLLM_USE_MODELSCOPE
7
from vllm.logger import init_logger
8
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9
- JAISConfig, MedusaConfig,
10
- MLPSpeculatorConfig, MPTConfig,
11
- NemotronConfig, RWConfig)
9
+ InternVLChatConfig, JAISConfig,
10
+ MedusaConfig, MLPSpeculatorConfig,
11
+ MPTConfig, NemotronConfig,
12
+ RWConfig)
13
14
if VLLM_USE_MODELSCOPE:
15
from modelscope import AutoConfig
@@ -26,6 +27,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
27
"jais": JAISConfig,
28
"mlp_speculator": MLPSpeculatorConfig,
29
"medusa": MedusaConfig,
30
+ "internvl_chat": InternVLChatConfig,
31
"nemotron": NemotronConfig,
32
}
33
vllm/transformers_utils/configs/__init__.py
+2
@@ -4,6 +4,7 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
4
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5
# `FalconConfig` class from the official HuggingFace transformers library.
6
from vllm.transformers_utils.configs.falcon import RWConfig
7
+from vllm.transformers_utils.configs.internvl import InternVLChatConfig
8
from vllm.transformers_utils.configs.jais import JAISConfig
9
from vllm.transformers_utils.configs.medusa import MedusaConfig
10
from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
@@ -15,6 +16,7 @@ __all__ = [
16
"DbrxConfig",
17
"MPTConfig",
18
"RWConfig",
19
+ "InternVLChatConfig",
20
"JAISConfig",
21
"MedusaConfig",
22
"MLPSpeculatorConfig",
vllm/transformers_utils/configs/internvl.py
new
+51
@@ -0,0 +1,51 @@
1
+# Adapted from
2
+# https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/configuration_internvl_chat.py
3
+# --------------------------------------------------------
4
+# InternVL
5
+# Copyright (c) 2024 OpenGVLab
6
+# Licensed under The MIT License [see LICENSE for details]
7
+# --------------------------------------------------------
8
+from transformers.configuration_utils import PretrainedConfig
9
+
10
+
11
+class InternVLChatConfig(PretrainedConfig):
12
+ model_type = 'internvl_chat'
13
+ is_composition = True
14
+
15
+ def __init__(self,
16
+ vision_config=None,
17
+ llm_config=None,
18
+ use_backbone_lora=0,
19
+ use_llm_lora=0,
20
+ select_layer=-1,
21
+ force_image_size=None,
22
+ downsample_ratio=0.5,
23
+ template=None,
24
+ dynamic_image_size=False,
25
+ use_thumbnail=False,
26
+ ps_version='v1',
27
+ min_dynamic_patch=1,
28
+ max_dynamic_patch=6,
29
+ **kwargs):
30
+ super().__init__(**kwargs)
31
+
32
+ if vision_config is None:
33
+ vision_config = {}
34
+
35
+ if llm_config is None:
36
+ llm_config = {}
37
+
38
+ self.vision_config = PretrainedConfig(**vision_config)
39
+ self.text_config = PretrainedConfig(**llm_config)
40
+
41
+ self.use_backbone_lora = use_backbone_lora
42
+ self.use_llm_lora = use_llm_lora
43
+ self.select_layer = select_layer
44
+ self.force_image_size = force_image_size
45
+ self.downsample_ratio = downsample_ratio
46
+ self.template = template
47
+ self.dynamic_image_size = dynamic_image_size
48
+ self.use_thumbnail = use_thumbnail
49
+ self.ps_version = ps_version # pixel shuffle version
50
+ self.min_dynamic_patch = min_dynamic_patch
51
+ self.max_dynamic_patch = max_dynamic_patch