@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7cbd9ec7a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model] Initialize support for InternVL2 series models (#6514)

Co-authored-by: Roger Wang <ywang@roblox.com>

Isotr0py committed Jul 29, 2024 at 18:16 UTC 7cbd9ec7a9bfd4952ad522355b6bbb8e82b54fc9
14 files changed +1042 -6
docs/source/models/supported_models.rst
+4
@@ -200,6 +200,10 @@ Vision Language Models
200 - Fuyu
201 - :code:`adept/fuyu-8b` etc.
202 -
203 + * - :code:`InternVLChatModel`
204 + - InternVL2
205 + - :code:`OpenGVLab/InternVL2-4B`, :code:`OpenGVLab/InternVL2-8B`, etc.
206 + -
207 * - :code:`LlavaForConditionalGeneration`
208 - LLaVA-1.5
209 - :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
examples/offline_inference_vision_language.py
+15
@@ -106,6 +106,20 @@ def run_minicpmv(question):
106 return llm, prompt
107
108
109 +# InternVL
110 +def run_internvl(question):
111 + # Generally, InternVL can use chatml template for conversation
112 + TEMPLATE = "<|im_start|>User\n{prompt}<|im_end|>\n<|im_start|>Assistant\n"
113 + prompt = f"<image>\n{question}\n"
114 + prompt = TEMPLATE.format(prompt=prompt)
115 + llm = LLM(
116 + model="OpenGVLab/InternVL2-4B",
117 + trust_remote_code=True,
118 + max_num_seqs=5,
119 + )
120 + return llm, prompt
121 +
122 +
123 # BLIP-2
124 def run_blip2(question):
125
@@ -125,6 +139,7 @@ model_example_map = {
139 "chameleon": run_chameleon,
140 "minicpmv": run_minicpmv,
141 "blip-2": run_blip2,
142 + "internvl_chat": run_internvl,
143 }
144
145
examples/openai_vision_api_client.py
+2
@@ -42,6 +42,7 @@ chat_completion_from_url = client.chat.completions.create(
42 ],
43 }],
44 model=model,
45 + max_tokens=64,
46 )
47
48 result = chat_completion_from_url.choices[0].message.content
@@ -78,6 +79,7 @@ chat_completion_from_base64 = client.chat.completions.create(
79 ],
80 }],
81 model=model,
82 + max_tokens=64,
83 )
84
85 result = chat_completion_from_base64.choices[0].message.content
requirements-test.txt
+1
@@ -16,6 +16,7 @@ ray
16 sentence-transformers # required for embedding
17 sparseml==1.8.0 # required for compressed-tensors
18 compressed-tensors==0.4.0 # required for compressed-tensors
19 +timm # required for internvl test
20
21 # Benchmarking
22 aiohttp
tests/models/test_internvl.py new
+201
@@ -0,0 +1,201 @@
1 +import types
2 +from typing import List, Optional, Type
3 +
4 +import pytest
5 +import torch
6 +from huggingface_hub import snapshot_download
7 +from PIL.Image import Image
8 +
9 +from vllm.model_executor.models.internvl import (IMG_CONTEXT, IMG_END,
10 + IMG_START,
11 + image_to_pixel_values)
12 +from vllm.multimodal.utils import rescale_image_size
13 +from vllm.utils import is_cpu
14 +
15 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
16 +from .utils import check_logprobs_close
17 +
18 +pytestmark = pytest.mark.vlm
19 +
20 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21 + "stop_sign":
22 + "<|im_start|>User\n<image>\nWhat's the content in the center of the image?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
23 + "cherry_blossom":
24 + "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
25 +})
26 +
27 +# we use snapshot_download to prevent conflicts between
28 +# dynamic_module and trust_remote_code for hf_runner
29 +models = [
30 + snapshot_download("OpenGVLab/InternVL2-1B"),
31 + snapshot_download("OpenGVLab/InternVL2-2B"),
32 + # snapshot_download("OpenGVLab/InternVL2-4B"), # broken
33 +]
34 +
35 +
36 +class InternVLProcessor:
37 + """A simple processor for InternVL2 HF model which misses a processor."""
38 +
39 + def __init__(self, hf_runner: HfRunner):
40 + self.num_image_token = hf_runner.model.num_image_token
41 + self.tokenizer = hf_runner.tokenizer
42 + self.dtype = hf_runner.model.dtype
43 +
44 + def __call__(self, text: str, images: Image, **kwargs):
45 + pixel_values = image_to_pixel_values(images).to(self.dtype)
46 + num_patches_list = [pixel_values.shape[0]]
47 + for num_patches in num_patches_list:
48 + context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
49 + image_tokens = IMG_START + context_tokens + IMG_END
50 + text = text.replace('<image>', image_tokens, 1)
51 + prompt = self.tokenizer(text, return_tensors="pt")
52 + prompt.update({"pixel_values": pixel_values})
53 + return prompt
54 +
55 +
56 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py
57 +def generate(
58 + self,
59 + pixel_values: torch.FloatTensor,
60 + input_ids: torch.FloatTensor,
61 + attention_mask: Optional[torch.LongTensor] = None,
62 + **generate_kwargs,
63 +) -> torch.LongTensor:
64 + """Generate method for InternVL2 model without fixed use_cache."""
65 + assert self.img_context_token_id is not None
66 + vit_embeds = self.extract_feature(pixel_values)
67 + input_embeds = self.language_model.get_input_embeddings()(input_ids)
68 + B, N, C = input_embeds.shape
69 + input_embeds = input_embeds.reshape(B * N, C)
70 +
71 + input_ids = input_ids.reshape(B * N)
72 + selected = (input_ids == self.img_context_token_id)
73 + assert selected.sum() != 0
74 + input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
75 +
76 + input_embeds = input_embeds.reshape(B, N, C)
77 +
78 + outputs = self.language_model.generate(
79 + inputs_embeds=input_embeds,
80 + attention_mask=attention_mask,
81 + **generate_kwargs,
82 + )
83 +
84 + return outputs
85 +
86 +
87 +def run_test(
88 + hf_runner: Type[HfRunner],
89 + vllm_runner: Type[VllmRunner],
90 + image_assets: _ImageAssets,
91 + model: str,
92 + *,
93 + size_factors: List[float],
94 + dtype: str,
95 + max_tokens: int,
96 + num_logprobs: int,
97 + tensor_parallel_size: int,
98 + distributed_executor_backend: Optional[str] = None,
99 +):
100 + """Inference result should be the same between hf and vllm.
101 +
102 + All the image fixtures for the test is under tests/images.
103 + For huggingface runner, we provide the PIL images as input.
104 + For vllm runner, we provide MultiModalDataDict objects
105 + and corresponding vision language config as input.
106 + Note, the text input is also adjusted to abide by vllm contract.
107 + The text output is sanitized to be able to compare with hf.
108 + """
109 + images = [asset.pil_image for asset in image_assets]
110 +
111 + inputs_per_image = [(
112 + [prompt for _ in size_factors],
113 + [rescale_image_size(image, factor) for factor in size_factors],
114 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
115 +
116 + # NOTE: take care of the order. run vLLM first, and then run HF.
117 + # vLLM needs a fresh new process without cuda initialization.
118 + # if we run HF first, the cuda initialization will be done and it
119 + # will hurt multiprocessing backend with fork method (the default method).
120 +
121 + # max_model_len should be greater than image_feature_size
122 + with vllm_runner(model,
123 + max_model_len=4096,
124 + dtype=dtype,
125 + tensor_parallel_size=tensor_parallel_size,
126 + distributed_executor_backend=distributed_executor_backend,
127 + enforce_eager=True) as vllm_model:
128 + vllm_outputs_per_image = [
129 + vllm_model.generate_greedy_logprobs(prompts,
130 + max_tokens,
131 + num_logprobs=num_logprobs,
132 + images=images)
133 + for prompts, images in inputs_per_image
134 + ]
135 +
136 + with hf_runner(model, dtype=dtype) as hf_model:
137 + img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids(
138 + "<IMG_CONTEXT>")
139 + hf_model.model.img_context_token_id = img_context_token_id
140 + hf_model.processor = InternVLProcessor(hf_model)
141 + hf_model.model.get_output_embeddings = lambda: \
142 + hf_model.model.language_model.get_output_embeddings()
143 + hf_model.model.generate = types.MethodType(generate, hf_model.model)
144 + eos_token_id = hf_model.tokenizer.eos_token_id
145 + hf_outputs_per_image = [
146 + hf_model.generate_greedy_logprobs_limit(prompts,
147 + max_tokens,
148 + num_logprobs=num_logprobs,
149 + images=hf_images,
150 + eos_token_id=eos_token_id)
151 + for prompts, hf_images in inputs_per_image
152 + ]
153 +
154 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
155 + vllm_outputs_per_image):
156 + # TODO: Check whether using original CLIPVisionModel can improve
157 + # consistency against HF
158 + check_logprobs_close(
159 + outputs_0_lst=hf_outputs,
160 + outputs_1_lst=vllm_outputs,
161 + name_0="hf",
162 + name_1="vllm",
163 + )
164 +
165 +
166 +target_dtype = "half"
167 +if is_cpu():
168 + target_dtype = "bfloat16"
169 +
170 +
171 +@pytest.mark.parametrize("model", models)
172 +@pytest.mark.parametrize(
173 + "size_factors",
174 + [
175 + # No image
176 + [],
177 + # Single-scale
178 + [1.0],
179 + # Single-scale, batched
180 + [1.0, 1.0, 1.0],
181 + # Multi-scale
182 + [0.25, 0.5, 1.0],
183 + ],
184 +)
185 +@pytest.mark.parametrize("dtype", [target_dtype])
186 +@pytest.mark.parametrize("max_tokens", [128])
187 +@pytest.mark.parametrize("num_logprobs", [5])
188 +@torch.inference_mode()
189 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
190 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
191 + run_test(
192 + hf_runner,
193 + vllm_runner,
194 + image_assets,
195 + model,
196 + size_factors=size_factors,
197 + dtype=dtype,
198 + max_tokens=max_tokens,
199 + num_logprobs=num_logprobs,
200 + tensor_parallel_size=1,
201 + )
vllm/entrypoints/chat_utils.py
+1 -1
@@ -107,7 +107,7 @@ def _image_token_str(model_config: ModelConfig,
107 return None
108 if model_type.startswith("llava"):
109 return tokenizer.decode(model_config.hf_config.image_token_index)
110 - if model_type == "chameleon":
110 + if model_type in ("chameleon", "internvl_chat"):
111 return "<image>"
112 raise TypeError(f"Unknown model type: {model_type}")
113
vllm/model_executor/models/__init__.py
+1
@@ -37,6 +37,7 @@ _GENERATION_MODELS = {
37 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
38 "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
39 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
40 + "InternVLChatModel": ("internvl", "InternVLChatModel"),
41 "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
42 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
43 "LlavaForConditionalGeneration":
vllm/model_executor/models/intern_vit.py new
+270
@@ -0,0 +1,270 @@
1 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py
2 +# --------------------------------------------------------
3 +# InternVL
4 +# Copyright (c) 2023 OpenGVLab
5 +# Licensed under The MIT License [see LICENSE for details]
6 +# --------------------------------------------------------
7 +from typing import Optional
8 +
9 +import torch
10 +import torch.nn as nn
11 +import torch.nn.functional as F
12 +from transformers import PretrainedConfig
13 +
14 +from vllm.model_executor.layers.activation import get_act_fn
15 +from vllm.model_executor.layers.layernorm import RMSNorm
16 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 + RowParallelLinear)
18 +from vllm.model_executor.layers.quantization import QuantizationConfig
19 +
20 +NORM2FN = {
21 + 'rms_norm': RMSNorm,
22 + 'layer_norm': nn.LayerNorm,
23 +}
24 +
25 +
26 +class InternVisionEmbeddings(nn.Module):
27 +
28 + def __init__(self, config: PretrainedConfig):
29 + super().__init__()
30 + self.config = config
31 + self.embed_dim = config.hidden_size
32 + self.image_size = config.image_size
33 + self.patch_size = config.patch_size
34 +
35 + self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim))
36 +
37 + self.patch_embedding = nn.Conv2d(in_channels=3,
38 + out_channels=self.embed_dim,
39 + kernel_size=self.patch_size,
40 + stride=self.patch_size)
41 +
42 + self.num_patches = (self.image_size // self.patch_size)**2
43 + self.num_positions = self.num_patches + 1
44 +
45 + self.position_embedding = nn.Parameter(
46 + torch.randn(1, self.num_positions, self.embed_dim))
47 +
48 + def _get_pos_embed(self, pos_embed, H, W):
49 + target_dtype = pos_embed.dtype
50 + pos_embed = pos_embed.float().reshape(
51 + 1, self.image_size // self.patch_size,
52 + self.image_size // self.patch_size, -1).permute(0, 3, 1, 2)
53 + pos_embed = F.interpolate(pos_embed,
54 + size=(H, W),
55 + mode='bicubic',
56 + align_corners=False)
57 + pos_embed = pos_embed.reshape(1, -1, H * W).permute(0, 2,
58 + 1).to(target_dtype)
59 + return pos_embed
60 +
61 + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
62 + target_dtype = self.patch_embedding.weight.dtype
63 + patch_embeds = self.patch_embedding(pixel_values.to(
64 + target_dtype)) # shape = [*, channel, width, height]
65 + batch_size, _, height, width = patch_embeds.shape
66 + patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
67 + class_embeds = self.class_embedding.expand(batch_size, 1,
68 + -1).to(target_dtype)
69 + embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
70 + position_embedding = torch.cat([
71 + self.position_embedding[:, :1, :],
72 + self._get_pos_embed(self.position_embedding[:, 1:, :], height,
73 + width)
74 + ],
75 + dim=1)
76 + embeddings = embeddings + position_embedding.to(target_dtype)
77 + return embeddings
78 +
79 +
80 +class InternAttention(nn.Module):
81 + """Multi-headed attention from 'Attention Is All You Need' paper"""
82 +
83 + def __init__(self, config: PretrainedConfig):
84 + super().__init__()
85 + self.config = config
86 + self.embed_dim = config.hidden_size
87 + self.num_heads = config.num_attention_heads
88 + self.head_dim = self.embed_dim // self.num_heads
89 + if self.head_dim * self.num_heads != self.embed_dim:
90 + raise ValueError(
91 + f'embed_dim must be divisible by num_heads '
92 + f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
93 + f' {self.num_heads}).')
94 +
95 + self.scale = self.head_dim**-0.5
96 + self.qkv = nn.Linear(self.embed_dim,
97 + 3 * self.embed_dim,
98 + bias=config.qkv_bias)
99 +
100 + self.qk_normalization = config.qk_normalization
101 +
102 + if self.qk_normalization:
103 + self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
104 + self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
105 +
106 + self.proj = nn.Linear(self.embed_dim, self.embed_dim)
107 +
108 + def forward(self, x):
109 + B, N, C = x.shape
110 + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads,
111 + C // self.num_heads).permute(2, 0, 3, 1, 4)
112 + q, k, v = qkv.unbind(0)
113 +
114 + if self.qk_normalization:
115 + B_, H_, N_, D_ = q.shape
116 + q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(
117 + B_, N_, H_, D_).transpose(1, 2)
118 + k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(
119 + B_, N_, H_, D_).transpose(1, 2)
120 +
121 + x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
122 + x = x.transpose(1, 2).reshape(B, N, C)
123 +
124 + x = self.proj(x)
125 + return x
126 +
127 +
128 +class InternMLP(nn.Module):
129 +
130 + def __init__(self,
131 + config: PretrainedConfig,
132 + quant_config: Optional[QuantizationConfig] = None):
133 + super().__init__()
134 + self.config = config
135 + self.activation_fn = get_act_fn(config.hidden_act)
136 + self.fc1 = ColumnParallelLinear(config.hidden_size,
137 + config.intermediate_size,
138 + bias=True,
139 + quant_config=quant_config)
140 + self.fc2 = RowParallelLinear(config.intermediate_size,
141 + config.hidden_size,
142 + bias=True,
143 + quant_config=quant_config)
144 +
145 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
146 + hidden_states, _ = self.fc1(hidden_states)
147 + hidden_states = self.activation_fn(hidden_states)
148 + hidden_states, _ = self.fc2(hidden_states)
149 +
150 + return hidden_states
151 +
152 +
153 +class InternVisionEncoderLayer(nn.Module):
154 +
155 + def __init__(self,
156 + config: PretrainedConfig,
157 + quant_config: Optional[QuantizationConfig] = None):
158 + super().__init__()
159 + self.embed_dim = config.hidden_size
160 + self.intermediate_size = config.intermediate_size
161 + self.norm_type = config.norm_type
162 +
163 + self.attn = InternAttention(config)
164 + self.mlp = InternMLP(config, quant_config=quant_config)
165 + self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
166 + eps=config.layer_norm_eps)
167 + self.norm2 = NORM2FN[self.norm_type](self.embed_dim,
168 + eps=config.layer_norm_eps)
169 +
170 + self.ls1 = nn.Parameter(config.initializer_factor *
171 + torch.ones(self.embed_dim))
172 + self.ls2 = nn.Parameter(config.initializer_factor *
173 + torch.ones(self.embed_dim))
174 +
175 + def forward(
176 + self,
177 + hidden_states: torch.Tensor,
178 + ):
179 + hidden_states = hidden_states + self.attn(
180 + self.norm1(hidden_states)) * self.ls1
181 +
182 + hidden_states = hidden_states + self.mlp(
183 + self.norm2(hidden_states)) * self.ls2
184 +
185 + return hidden_states
186 +
187 +
188 +class InternVisionEncoder(nn.Module):
189 +
190 + def __init__(self,
191 + config: PretrainedConfig,
192 + quant_config: Optional[QuantizationConfig] = None,
193 + num_hidden_layers_override: Optional[int] = None):
194 + super().__init__()
195 + self.config = config
196 +
197 + if num_hidden_layers_override is None:
198 + num_hidden_layers = config.num_hidden_layers
199 + else:
200 + num_hidden_layers = num_hidden_layers_override
201 + self.layers = nn.ModuleList([
202 + InternVisionEncoderLayer(config=config, quant_config=quant_config)
203 + for _ in range(num_hidden_layers)
204 + ])
205 +
206 + def forward(self, inputs_embeds: torch.Tensor):
207 +
208 + hidden_states = inputs_embeds
209 + for encoder_layer in self.layers:
210 + hidden_states = encoder_layer(hidden_states)
211 +
212 + return hidden_states
213 +
214 +
215 +class InternVisionModel(nn.Module):
216 +
217 + def __init__(self,
218 + config: PretrainedConfig,
219 + quant_config: Optional[QuantizationConfig] = None,
220 + num_hidden_layers_override: Optional[int] = None):
221 + super().__init__()
222 + self.config = config
223 +
224 + self.embeddings = InternVisionEmbeddings(config)
225 + self.encoder = InternVisionEncoder(
226 + config=config,
227 + quant_config=quant_config,
228 + num_hidden_layers_override=num_hidden_layers_override)
229 +
230 + def resize_pos_embeddings(self, old_size, new_size, patch_size):
231 + pos_emb = self.embeddings.position_embedding
232 + _, num_positions, embed_dim = pos_emb.shape
233 + cls_emb = pos_emb[:, :1, :]
234 + pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size,
235 + old_size // patch_size,
236 + -1).permute(0, 3, 1, 2)
237 + pos_emb = F.interpolate(pos_emb.float(),
238 + size=new_size // patch_size,
239 + mode='bicubic',
240 + align_corners=False)
241 + pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim,
242 + -1).permute(0, 2, 1)
243 + pos_emb = torch.cat([cls_emb, pos_emb], dim=1)
244 + self.embeddings.position_embedding = nn.Parameter(pos_emb)
245 + self.embeddings.image_size = new_size
246 +
247 + def get_input_embeddings(self):
248 + return self.embeddings
249 +
250 + def forward(
251 + self,
252 + pixel_values: Optional[torch.Tensor] = None,
253 + pixel_embeds: Optional[torch.Tensor] = None,
254 + ) -> torch.FloatTensor:
255 + if pixel_values is None and pixel_embeds is None:
256 + raise ValueError(
257 + 'You have to specify pixel_values or pixel_embeds')
258 +
259 + if pixel_embeds is not None:
260 + hidden_states = pixel_embeds
261 + elif pixel_values is not None:
262 + if pixel_values.ndim == 4:
263 + hidden_states = self.embeddings(pixel_values)
264 + else:
265 + raise ValueError(
266 + f'wrong pixel_values size: {pixel_values.shape}')
267 +
268 + encoder_outputs = self.encoder(inputs_embeds=hidden_states)
269 +
270 + return encoder_outputs
vllm/model_executor/models/internlm2.py
+9 -1
@@ -219,14 +219,22 @@ class InternLM2Model(nn.Module):
219 ])
220 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
221
222 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
223 + return self.tok_embeddings(input_ids)
224 +
225 def forward(
226 self,
227 input_ids: torch.Tensor,
228 positions: torch.Tensor,
229 kv_caches: List[torch.Tensor],
230 attn_metadata: AttentionMetadata,
231 + intermediate_tensors: IntermediateTensors = None,
232 + inputs_embeds: Optional[torch.Tensor] = None,
233 ) -> torch.Tensor:
229 - hidden_states = self.tok_embeddings(input_ids)
234 + if inputs_embeds is not None:
235 + hidden_states = inputs_embeds
236 + else:
237 + hidden_states = self.tok_embeddings(input_ids)
238 residual = None
239 for i in range(len(self.layers)):
240 layer = self.layers[i]
vllm/model_executor/models/internvl.py new
+471
@@ -0,0 +1,471 @@
1 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py
2 +# --------------------------------------------------------
3 +# InternVL
4 +# Copyright (c) 2023 OpenGVLab
5 +# Licensed under The MIT License [see LICENSE for details]
6 +# --------------------------------------------------------
7 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
8 +
9 +import torch
10 +import torch.nn as nn
11 +import torchvision.transforms as T
12 +from PIL import Image
13 +from transformers import PretrainedConfig
14 +
15 +from vllm.attention import AttentionMetadata
16 +from vllm.config import CacheConfig, MultiModalConfig
17 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
18 +from vllm.model_executor.layers.quantization import QuantizationConfig
19 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20 +from vllm.model_executor.models import ModelRegistry
21 +from vllm.model_executor.models.intern_vit import InternVisionModel
22 +from vllm.model_executor.sampling_metadata import SamplingMetadata
23 +from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensors
24 +from vllm.multimodal.base import MultiModalInputs
25 +from vllm.multimodal.image import cached_get_tokenizer
26 +from vllm.sequence import IntermediateTensors, SamplerOutput
27 +
28 +from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
29 + get_clip_num_patches)
30 +from .interfaces import SupportsVision
31 +from .utils import merge_vision_embeddings
32 +
33 +IMG_START = '<img>'
34 +IMG_END = '</img>'
35 +IMG_CONTEXT = '<IMG_CONTEXT>'
36 +
37 +IMAGENET_MEAN = (0.485, 0.456, 0.406)
38 +IMAGENET_STD = (0.229, 0.224, 0.225)
39 +
40 +MAX_IMAGE_FEATURE_SIZE_WIDTH = 3000
41 +MAX_IMAGE_FEATURE_SIZE_HEIGHT = 500
42 +
43 +
44 +class InternVLImagePixelInputs(TypedDict):
45 + type: Literal["pixel_values"]
46 + data: BatchedTensors
47 + """
48 + Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
49 +
50 + Note that `num_patches` may be different for each batch, in which case
51 + the data is passed as a list instead of a batched tensor.
52 + """
53 +
54 +
55 +# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
56 +def build_transform(input_size):
57 + MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
58 + transform = T.Compose([
59 + T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
60 + T.Resize((input_size, input_size),
61 + interpolation=T.InterpolationMode.BICUBIC),
62 + T.ToTensor(),
63 + T.Normalize(mean=MEAN, std=STD)
64 + ])
65 + return transform
66 +
67 +
68 +# copied from https://huggingface.co/OpenGVLab/InternVL2-1B
69 +def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
70 + image_size):
71 + best_ratio_diff = float('inf')
72 + best_ratio = (1, 1)
73 + area = width * height
74 + for ratio in target_ratios:
75 + target_aspect_ratio = ratio[0] / ratio[1]
76 + ratio_diff = abs(aspect_ratio - target_aspect_ratio)
77 + if ratio_diff < best_ratio_diff:
78 + best_ratio_diff = ratio_diff
79 + best_ratio = ratio
80 + elif ratio_diff == best_ratio_diff:
81 + if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
82 + best_ratio = ratio
83 + return best_ratio
84 +
85 +
86 +def calculate_num_blocks(orig_width: int,
87 + orig_height: int,
88 + min_num=1,
89 + max_num=6,
90 + image_size=448):
91 + aspect_ratio = orig_width / orig_height
92 +
93 + # calculate the existing image aspect ratio
94 + target_ratios = set((i, j) for n in range(min_num, max_num + 1)
95 + for i in range(1, n + 1) for j in range(1, n + 1)
96 + if i * j <= max_num and i * j >= min_num)
97 + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
98 +
99 + # find the closest aspect ratio to the target
100 + target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio,
101 + target_ratios, orig_width,
102 + orig_height, image_size)
103 +
104 + # calculate the target width and height
105 + target_width = image_size * target_aspect_ratio[0]
106 + target_height = image_size * target_aspect_ratio[1]
107 + blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
108 + return blocks, target_width, target_height
109 +
110 +
111 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
112 +def dynamic_preprocess(image,
113 + min_num=1,
114 + max_num=6,
115 + image_size=448,
116 + use_thumbnail=False):
117 + orig_width, orig_height = image.size
118 +
119 + blocks, target_width, target_height = calculate_num_blocks(
120 + orig_width, orig_height, min_num, max_num, image_size)
121 + # resize the image
122 + resized_img = image.resize((target_width, target_height))
123 + processed_images = []
124 + for i in range(blocks):
125 + box = ((i % (target_width // image_size)) * image_size,
126 + (i // (target_width // image_size)) * image_size,
127 + ((i % (target_width // image_size)) + 1) * image_size,
128 + ((i // (target_width // image_size)) + 1) * image_size)
129 + # split the image
130 + split_img = resized_img.crop(box)
131 + processed_images.append(split_img)
132 + assert len(processed_images) == blocks
133 + if use_thumbnail and len(processed_images) != 1:
134 + thumbnail_img = image.resize((image_size, image_size))
135 + processed_images.append(thumbnail_img)
136 + return processed_images
137 +
138 +
139 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
140 +def image_to_pixel_values(image: Image.Image, input_size=448, max_num=6):
141 + transform = build_transform(input_size=input_size)
142 + images = dynamic_preprocess(image,
143 + image_size=input_size,
144 + use_thumbnail=True,
145 + max_num=max_num)
146 + pixel_values = [transform(image) for image in images]
147 + pixel_values = torch.stack(pixel_values)
148 + return pixel_values
149 +
150 +
151 +def get_internvl_num_patches(image_size: int, patch_size: int,
152 + downsample_ratio: float):
153 + return int(
154 + get_clip_num_patches(image_size=image_size, patch_size=patch_size) *
155 + (downsample_ratio**2))
156 +
157 +
158 +def get_max_internvl_image_tokens(ctx: InputContext):
159 + hf_config = ctx.get_hf_config(PretrainedConfig)
160 + vision_config = hf_config.vision_config
161 + image_size = vision_config.image_size
162 + patch_size = vision_config.patch_size
163 + downsample_ratio = hf_config.downsample_ratio
164 + num_patches = get_internvl_num_patches(image_size, patch_size,
165 + downsample_ratio)
166 + return num_patches * 7
167 +
168 +
169 +def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
170 + multi_modal_data = llm_inputs.get("multi_modal_data")
171 + if multi_modal_data is None or "image" not in multi_modal_data:
172 + return llm_inputs
173 +
174 + model_config = ctx.model_config
175 + hf_config = ctx.get_hf_config(PretrainedConfig)
176 + vision_config = hf_config.vision_config
177 +
178 + image_data = multi_modal_data["image"]
179 + if isinstance(image_data, Image.Image):
180 + width, height = image_data.size
181 + num_blocks, _, _ = calculate_num_blocks(width, height)
182 + elif isinstance(image_data, torch.Tensor):
183 + raise NotImplementedError("Embeddings input is not supported yet")
184 + else:
185 + raise TypeError(f"Invalid image type: {type(image_data)}")
186 +
187 + image_size = vision_config.image_size
188 + patch_size = vision_config.patch_size
189 + downsample_ratio = hf_config.downsample_ratio
190 + num_patches = get_internvl_num_patches(image_size, patch_size,
191 + downsample_ratio)
192 +
193 + tokenizer = cached_get_tokenizer(model_config.tokenizer,
194 + trust_remote_code=True)
195 +
196 + prompt = llm_inputs["prompt"]
197 + prompt_token_ids = llm_inputs["prompt_token_ids"]
198 + if prompt is None:
199 + prompt = tokenizer.decode(prompt_token_ids)
200 + image_prompt = IMG_START + IMG_CONTEXT * (num_blocks +
201 + 1) * num_patches + IMG_END
202 + new_prompt = prompt.replace('<image>', image_prompt, 1)
203 + new_prompt_token_ids = tokenizer.encode(new_prompt)
204 +
205 + return LLMInputs(prompt=prompt,
206 + prompt_token_ids=new_prompt_token_ids,
207 + multi_modal_data=multi_modal_data)
208 +
209 +
210 +def input_mapper_for_internvl(ctx: InputContext, data: object):
211 + if isinstance(data, Image.Image):
212 + data = image_to_pixel_values(data)
213 + model_config = ctx.model_config
214 + tokenizer = cached_get_tokenizer(model_config.tokenizer,
215 + trust_remote_code=True)
216 + image_token_id = tokenizer.encode(IMG_CONTEXT,
217 + add_special_tokens=False,
218 + return_tensors="pt")[0]
219 +
220 + return MultiModalInputs({
221 + "pixel_values": data,
222 + "image_token_id": image_token_id
223 + })
224 +
225 +
226 +def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
227 +
228 + image_feature_size = get_max_internvl_image_tokens(ctx)
229 + model_config = ctx.model_config
230 + hf_config = ctx.get_hf_config(PretrainedConfig)
231 + vision_config = hf_config.vision_config
232 + tokenizer = cached_get_tokenizer(model_config.tokenizer,
233 + trust_remote_code=True)
234 +
235 + seq_data = dummy_seq_data_for_clip(
236 + vision_config,
237 + seq_len,
238 + image_token_id=tokenizer.encode(IMG_CONTEXT,
239 + add_special_tokens=False)[0],
240 + image_feature_size_override=image_feature_size,
241 + )
242 + mm_data = dummy_image_for_clip(
243 + vision_config,
244 + image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
245 + image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
246 + )
247 +
248 + return seq_data, mm_data
249 +
250 +
251 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_internvl)
252 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
253 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_internvl)
254 +@INPUT_REGISTRY.register_input_processor(input_processor_for_internvl)
255 +class InternVLChatModel(nn.Module, SupportsVision):
256 +
257 + def __init__(self,
258 + config: PretrainedConfig,
259 + multimodal_config: MultiModalConfig,
260 + cache_config: Optional[CacheConfig] = None,
261 + quant_config: Optional[QuantizationConfig] = None) -> None:
262 + super().__init__()
263 +
264 + self.config = config
265 + self.multimodal_config = multimodal_config
266 +
267 + image_size = config.force_image_size or config.vision_config.image_size
268 + patch_size = config.vision_config.patch_size
269 + self.patch_size = patch_size
270 + self.select_layer = config.select_layer
271 + self.num_image_token = int(
272 + (image_size // patch_size)**2 * (config.downsample_ratio**2))
273 + self.downsample_ratio = config.downsample_ratio
274 + self.ps_version = config.ps_version
275 +
276 + vision_feature_layer = self.select_layer
277 + if vision_feature_layer < 0:
278 + num_hidden_layers = config.vision_config.num_hidden_layers \
279 + + vision_feature_layer + 1
280 + else:
281 + num_hidden_layers = vision_feature_layer + 1
282 + self.vision_model = InternVisionModel(
283 + config.vision_config, num_hidden_layers_override=num_hidden_layers)
284 +
285 + llm_class = ModelRegistry.load_model_cls(
286 + config.text_config.architectures[0])
287 + self.language_model = llm_class(config.text_config, cache_config,
288 + quant_config)
289 +
290 + vit_hidden_size = config.vision_config.hidden_size
291 + llm_hidden_size = config.text_config.hidden_size
292 +
293 + self.mlp1 = nn.Sequential(
294 + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
295 + nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
296 + llm_hidden_size), nn.GELU(),
297 + nn.Linear(llm_hidden_size, llm_hidden_size))
298 +
299 + self.img_context_token_id = None
300 +
301 + def pixel_shuffle(self, x, scale_factor=0.5):
302 + n, w, h, c = x.size()
303 + # N, W, H, C --> N, W, H * scale, C // scale
304 + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
305 + # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
306 + x = x.permute(0, 2, 1, 3).contiguous()
307 + x = x.view(n, int(h * scale_factor), int(w * scale_factor),
308 + int(c / (scale_factor * scale_factor)))
309 + if self.ps_version == 'v1':
310 + pass
311 + else:
312 + x = x.permute(0, 2, 1, 3).contiguous()
313 + return x
314 +
315 + def extract_feature(self, pixel_values):
316 + vit_embeds = self.vision_model(pixel_values=pixel_values)
317 + vit_embeds = vit_embeds[:, 1:, :]
318 +
319 + h = w = int(vit_embeds.shape[1]**0.5)
320 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
321 + vit_embeds = self.pixel_shuffle(vit_embeds,
322 + scale_factor=self.downsample_ratio)
323 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1,
324 + vit_embeds.shape[-1])
325 + vit_embeds = self.mlp1(vit_embeds)
326 + return vit_embeds
327 +
328 + def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
329 + if list(data.shape[1:]) != [2]:
330 + raise ValueError(
331 + f"The expected image sizes shape is batch dimension plus "
332 + f"{[2]}. You supplied {data.shape}.")
333 +
334 + return data
335 +
336 + def _validate_pixel_values(
337 + self, data: Union[torch.Tensor, List[torch.Tensor]]
338 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
339 +
340 + h = w = self.config.vision_config.image_size
341 + expected_dims = (3, h, w)
342 +
343 + def _validate_shape(d: torch.Tensor):
344 + actual_dims = tuple(d.shape)
345 +
346 + if actual_dims != expected_dims:
347 + expected_expr = ("num_patches", *map(str, expected_dims))
348 + raise ValueError(
349 + "The expected shape of pixel values in each batch element "
350 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
351 +
352 + for d in data:
353 + _validate_shape(d)
354 +
355 + return data
356 +
357 + def _parse_and_validate_image_input(
358 + self, **kwargs: object) -> Optional[InternVLImagePixelInputs]:
359 + pixel_values = kwargs.pop("pixel_values", None)
360 + image_token_id = kwargs.pop("image_token_id", None)
361 +
362 + if pixel_values is None:
363 + return None
364 +
365 + self.img_context_token_id = image_token_id[0]
366 +
367 + if not isinstance(pixel_values, (torch.Tensor, list)):
368 + raise ValueError("Incorrect type of pixel values. "
369 + f"Got type: {type(pixel_values)}")
370 +
371 + return InternVLImagePixelInputs(
372 + type="pixel_values",
373 + data=self._validate_pixel_values(pixel_values),
374 + )
375 +
376 + def forward(
377 + self,
378 + input_ids: torch.Tensor,
379 + positions: torch.Tensor,
380 + kv_caches: List[torch.Tensor],
381 + attn_metadata: AttentionMetadata,
382 + intermediate_tensors: Optional[IntermediateTensors] = None,
383 + **kwargs: object,
384 + ) -> SamplerOutput:
385 + image_input = self._parse_and_validate_image_input(**kwargs)
386 + if image_input is not None:
387 + inputs_embeds = self.language_model.model.get_input_embeddings(
388 + input_ids)
389 + vit_embeds = self.extract_feature(image_input["data"])
390 + inputs_embeds = merge_vision_embeddings(input_ids, inputs_embeds,
391 + vit_embeds,
392 + self.img_context_token_id)
393 + input_ids = None
394 + else:
395 + inputs_embeds = None
396 +
397 + hidden_states = self.language_model.model(input_ids,
398 + positions,
399 + kv_caches,
400 + attn_metadata,
401 + None,
402 + inputs_embeds=inputs_embeds)
403 + return hidden_states
404 +
405 + def compute_logits(self, hidden_states: torch.Tensor,
406 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
407 + return self.language_model.compute_logits(hidden_states,
408 + sampling_metadata)
409 +
410 + def sample(
411 + self,
412 + logits: torch.Tensor,
413 + sampling_metadata: SamplingMetadata,
414 + ) -> Optional[SamplerOutput]:
415 + return self.language_model.sample(logits, sampling_metadata)
416 +
417 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
418 + stacked_params_mapping = [
419 + # (param_name, shard_name, shard_id)
420 + (".qkv_proj", ".q_proj", "q"),
421 + (".qkv_proj", ".k_proj", "k"),
422 + (".qkv_proj", ".v_proj", "v"),
423 + (".gate_up_proj", ".gate_proj", 0),
424 + (".gate_up_proj", ".up_proj", 1),
425 + (".gate_up_proj", ".w1", 0),
426 + (".gate_up_proj", ".w3", 1),
427 + ]
428 + params_dict = dict(self.named_parameters())
429 + for name, loaded_weight in weights:
430 + if "rotary_emb.inv_freq" in name:
431 + continue
432 + if self.config.text_config.tie_word_embeddings \
433 + and "lm_head.weight" in name:
434 + continue
435 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
436 + # We only do sharding for language model
437 + # and not vision model for now.
438 + if "vision_embed_tokens" in name and self.vision_embed_tokens:
439 + continue
440 + if weight_name not in name:
441 + continue
442 + param = params_dict[name.replace(weight_name, param_name)]
443 + weight_loader = param.weight_loader
444 + weight_loader(param, loaded_weight, shard_id)
445 + break
446 + else:
447 + # Skip loading extra bias for GPTQ models.
448 + if name.endswith(".bias") and name not in params_dict:
449 + continue
450 + param = params_dict[name]
451 + if "wqkv" in name:
452 + config = self.config.text_config
453 + kv_groups = (config.num_attention_heads //
454 + config.num_key_value_heads)
455 + head_dim = config.hidden_size // config.num_attention_heads
456 + loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
457 + head_dim,
458 + loaded_weight.shape[-1])
459 + wq, wk, wv = torch.split(loaded_weight, [kv_groups, 1, 1],
460 + dim=1)
461 + wq = wq.reshape(-1, wq.shape[-1])
462 + wk = wk.reshape(-1, wk.shape[-1])
463 + wv = wv.reshape(-1, wv.shape[-1])
464 + weight_loader = param.weight_loader
465 + weight_loader(param, wq, 'q')
466 + weight_loader(param, wk, 'k')
467 + weight_loader(param, wv, 'v')
468 + continue
469 + weight_loader = getattr(param, "weight_loader",
470 + default_weight_loader)
471 + weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen2.py
+9 -1
@@ -243,14 +243,22 @@ class Qwen2Model(nn.Module):
243 ])
244 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
245
246 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
247 + return self.embed_tokens(input_ids)
248 +
249 def forward(
250 self,
251 input_ids: torch.Tensor,
252 positions: torch.Tensor,
253 kv_caches: List[torch.Tensor],
254 attn_metadata: AttentionMetadata,
255 + intermediate_tensors: Optional[IntermediateTensors] = None,
256 + inputs_embeds: Optional[torch.Tensor] = None,
257 ) -> torch.Tensor:
253 - hidden_states = self.embed_tokens(input_ids)
258 + if inputs_embeds is not None:
259 + hidden_states = inputs_embeds
260 + else:
261 + hidden_states = self.embed_tokens(input_ids)
262 residual = None
263 for i in range(len(self.layers)):
264 layer = self.layers[i]
vllm/transformers_utils/config.py
+5 -3
@@ -6,9 +6,10 @@ from transformers import GenerationConfig, PretrainedConfig
6 from vllm.envs import VLLM_USE_MODELSCOPE
7 from vllm.logger import init_logger
8 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9 - JAISConfig, MedusaConfig,
10 - MLPSpeculatorConfig, MPTConfig,
11 - NemotronConfig, RWConfig)
9 + InternVLChatConfig, JAISConfig,
10 + MedusaConfig, MLPSpeculatorConfig,
11 + MPTConfig, NemotronConfig,
12 + RWConfig)
13
14 if VLLM_USE_MODELSCOPE:
15 from modelscope import AutoConfig
@@ -26,6 +27,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
27 "jais": JAISConfig,
28 "mlp_speculator": MLPSpeculatorConfig,
29 "medusa": MedusaConfig,
30 + "internvl_chat": InternVLChatConfig,
31 "nemotron": NemotronConfig,
32 }
33
vllm/transformers_utils/configs/__init__.py
+2
@@ -4,6 +4,7 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
4 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5 # `FalconConfig` class from the official HuggingFace transformers library.
6 from vllm.transformers_utils.configs.falcon import RWConfig
7 +from vllm.transformers_utils.configs.internvl import InternVLChatConfig
8 from vllm.transformers_utils.configs.jais import JAISConfig
9 from vllm.transformers_utils.configs.medusa import MedusaConfig
10 from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
@@ -15,6 +16,7 @@ __all__ = [
16 "DbrxConfig",
17 "MPTConfig",
18 "RWConfig",
19 + "InternVLChatConfig",
20 "JAISConfig",
21 "MedusaConfig",
22 "MLPSpeculatorConfig",
vllm/transformers_utils/configs/internvl.py new
+51
@@ -0,0 +1,51 @@
1 +# Adapted from
2 +# https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/configuration_internvl_chat.py
3 +# --------------------------------------------------------
4 +# InternVL
5 +# Copyright (c) 2024 OpenGVLab
6 +# Licensed under The MIT License [see LICENSE for details]
7 +# --------------------------------------------------------
8 +from transformers.configuration_utils import PretrainedConfig
9 +
10 +
11 +class InternVLChatConfig(PretrainedConfig):
12 + model_type = 'internvl_chat'
13 + is_composition = True
14 +
15 + def __init__(self,
16 + vision_config=None,
17 + llm_config=None,
18 + use_backbone_lora=0,
19 + use_llm_lora=0,
20 + select_layer=-1,
21 + force_image_size=None,
22 + downsample_ratio=0.5,
23 + template=None,
24 + dynamic_image_size=False,
25 + use_thumbnail=False,
26 + ps_version='v1',
27 + min_dynamic_patch=1,
28 + max_dynamic_patch=6,
29 + **kwargs):
30 + super().__init__(**kwargs)
31 +
32 + if vision_config is None:
33 + vision_config = {}
34 +
35 + if llm_config is None:
36 + llm_config = {}
37 +
38 + self.vision_config = PretrainedConfig(**vision_config)
39 + self.text_config = PretrainedConfig(**llm_config)
40 +
41 + self.use_backbone_lora = use_backbone_lora
42 + self.use_llm_lora = use_llm_lora
43 + self.select_layer = select_layer
44 + self.force_image_size = force_image_size
45 + self.downsample_ratio = downsample_ratio
46 + self.template = template
47 + self.dynamic_image_size = dynamic_image_size
48 + self.use_thumbnail = use_thumbnail
49 + self.ps_version = ps_version # pixel shuffle version
50 + self.min_dynamic_patch = min_dynamic_patch
51 + self.max_dynamic_patch = max_dynamic_patch