@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 151ef4efd2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Support NVLM-D and fix QK Norm in InternViT (#9045)

Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Cyrus Leung committed Oct 7, 2024 at 19:55 UTC 151ef4efd2fb52554f4d30408aca619e181ea751
12 files changed +518 -236
docs/source/models/supported_models.rst
+9
@@ -315,6 +315,9 @@ Multimodal Language Models
315
316 .. _supported_vlms:
317
318 +Text Generation
319 +---------------
320 +
321 .. list-table::
322 :widths: 25 25 25 25 5 5
323 :header-rows: 1
@@ -384,7 +387,13 @@ Multimodal Language Models
387 - Image
388 - :code:`meta-llama/Llama-3.2-90B-Vision-Instruct`, :code:`meta-llama/Llama-3.2-11B-Vision`, etc.
389 -
390 + -
391 + * - :code:`NVLM_D_Model`
392 + - NVLM-D 1.0
393 + - Image\ :sup:`E+`
394 + - :code:`nvidia/NVLM-D-72B`, etc.
395 -
396 + - ✅︎
397 * - :code:`PaliGemmaForConditionalGeneration`
398 - PaliGemma
399 - Image\ :sup:`E`
examples/offline_inference_vision_language.py
+40 -15
@@ -18,7 +18,7 @@ from vllm.utils import FlexibleArgumentParser
18
19
20 # LLaVA-1.5
21 -def run_llava(question, modality):
21 +def run_llava(question: str, modality: str):
22 assert modality == "image"
23
24 prompt = f"USER: <image>\n{question}\nASSISTANT:"
@@ -29,7 +29,7 @@ def run_llava(question, modality):
29
30
31 # LLaVA-1.6/LLaVA-NeXT
32 -def run_llava_next(question, modality):
32 +def run_llava_next(question: str, modality: str):
33 assert modality == "image"
34
35 prompt = f"[INST] <image>\n{question} [/INST]"
@@ -40,7 +40,7 @@ def run_llava_next(question, modality):
40
41 # LlaVA-NeXT-Video
42 # Currently only support for video input
43 -def run_llava_next_video(question, modality):
43 +def run_llava_next_video(question: str, modality: str):
44 assert modality == "video"
45
46 prompt = f"USER: <video>\n{question} ASSISTANT:"
@@ -50,7 +50,7 @@ def run_llava_next_video(question, modality):
50
51
52 # LLaVA-OneVision
53 -def run_llava_onevision(question, modality):
53 +def run_llava_onevision(question: str, modality: str):
54
55 if modality == "video":
56 prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
@@ -67,7 +67,7 @@ def run_llava_onevision(question, modality):
67
68
69 # Fuyu
70 -def run_fuyu(question, modality):
70 +def run_fuyu(question: str, modality: str):
71 assert modality == "image"
72
73 prompt = f"{question}\n"
@@ -77,7 +77,7 @@ def run_fuyu(question, modality):
77
78
79 # Phi-3-Vision
80 -def run_phi3v(question, modality):
80 +def run_phi3v(question: str, modality: str):
81 assert modality == "image"
82
83 prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n" # noqa: E501
@@ -112,7 +112,7 @@ def run_phi3v(question, modality):
112
113
114 # PaliGemma
115 -def run_paligemma(question, modality):
115 +def run_paligemma(question: str, modality: str):
116 assert modality == "image"
117
118 # PaliGemma has special prompt format for VQA
@@ -123,7 +123,7 @@ def run_paligemma(question, modality):
123
124
125 # Chameleon
126 -def run_chameleon(question, modality):
126 +def run_chameleon(question: str, modality: str):
127 assert modality == "image"
128
129 prompt = f"{question}<image>"
@@ -133,7 +133,7 @@ def run_chameleon(question, modality):
133
134
135 # MiniCPM-V
136 -def run_minicpmv(question, modality):
136 +def run_minicpmv(question: str, modality: str):
137 assert modality == "image"
138
139 # 2.0
@@ -176,7 +176,7 @@ def run_minicpmv(question, modality):
176
177
178 # InternVL
179 -def run_internvl(question, modality):
179 +def run_internvl(question: str, modality: str):
180 assert modality == "image"
181
182 model_name = "OpenGVLab/InternVL2-2B"
@@ -203,8 +203,32 @@ def run_internvl(question, modality):
203 return llm, prompt, stop_token_ids
204
205
206 +# NVLM-D
207 +def run_nvlm_d(question: str, modality: str):
208 + assert modality == "image"
209 +
210 + model_name = "nvidia/NVLM-D-72B"
211 +
212 + # Adjust this as necessary to fit in GPU
213 + llm = LLM(
214 + model=model_name,
215 + trust_remote_code=True,
216 + max_model_len=4096,
217 + tensor_parallel_size=4,
218 + )
219 +
220 + tokenizer = AutoTokenizer.from_pretrained(model_name,
221 + trust_remote_code=True)
222 + messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
223 + prompt = tokenizer.apply_chat_template(messages,
224 + tokenize=False,
225 + add_generation_prompt=True)
226 + stop_token_ids = None
227 + return llm, prompt, stop_token_ids
228 +
229 +
230 # BLIP-2
207 -def run_blip2(question, modality):
231 +def run_blip2(question: str, modality: str):
232 assert modality == "image"
233
234 # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
@@ -216,7 +240,7 @@ def run_blip2(question, modality):
240
241
242 # Qwen
219 -def run_qwen_vl(question, modality):
243 +def run_qwen_vl(question: str, modality: str):
244 assert modality == "image"
245
246 llm = LLM(
@@ -232,7 +256,7 @@ def run_qwen_vl(question, modality):
256
257
258 # Qwen2-VL
235 -def run_qwen2_vl(question, modality):
259 +def run_qwen2_vl(question: str, modality: str):
260 assert modality == "image"
261
262 model_name = "Qwen/Qwen2-VL-7B-Instruct"
@@ -252,8 +276,8 @@ def run_qwen2_vl(question, modality):
276 return llm, prompt, stop_token_ids
277
278
255 -# LLama
256 -def run_mllama(question, modality):
279 +# LLama 3.2
280 +def run_mllama(question: str, modality: str):
281 assert modality == "image"
282
283 model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
@@ -287,6 +311,7 @@ model_example_map = {
311 "minicpmv": run_minicpmv,
312 "blip-2": run_blip2,
313 "internvl_chat": run_internvl,
314 + "NVLM_D": run_nvlm_d,
315 "qwen_vl": run_qwen_vl,
316 "qwen2_vl": run_qwen2_vl,
317 "mllama": run_mllama,
examples/offline_inference_vision_language_multi_image.py
+34
@@ -144,6 +144,39 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
144 )
145
146
147 +def load_nvlm_d(question: str, image_urls: List[str]):
148 + model_name = "nvidia/NVLM-D-72B"
149 +
150 + # Adjust this as necessary to fit in GPU
151 + llm = LLM(
152 + model=model_name,
153 + trust_remote_code=True,
154 + max_model_len=8192,
155 + tensor_parallel_size=4,
156 + limit_mm_per_prompt={"image": len(image_urls)},
157 + mm_processor_kwargs={"max_dynamic_patch": 4},
158 + )
159 +
160 + placeholders = "\n".join(f"Image-{i}: <image>\n"
161 + for i, _ in enumerate(image_urls, start=1))
162 + messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
163 +
164 + tokenizer = AutoTokenizer.from_pretrained(model_name,
165 + trust_remote_code=True)
166 + prompt = tokenizer.apply_chat_template(messages,
167 + tokenize=False,
168 + add_generation_prompt=True)
169 + stop_token_ids = None
170 +
171 + return ModelRequestData(
172 + llm=llm,
173 + prompt=prompt,
174 + stop_token_ids=stop_token_ids,
175 + image_data=[fetch_image(url) for url in image_urls],
176 + chat_template=None,
177 + )
178 +
179 +
180 def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
181 try:
182 from qwen_vl_utils import process_vision_info
@@ -204,6 +237,7 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
237 model_example_map = {
238 "phi3_v": load_phi3v,
239 "internvl_chat": load_internvl,
240 + "NVLM_D": load_nvlm_d,
241 "qwen2_vl": load_qwen2_vl,
242 "qwen_vl_chat": load_qwenvl_chat,
243 }
vllm/entrypoints/chat_utils.py
+1 -1
@@ -157,7 +157,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
157 if model_type.startswith("llava"):
158 return self._cached_token_str(self._tokenizer,
159 hf_config.image_token_index)
160 - if model_type in ("chameleon", "internvl_chat"):
160 + if model_type in ("chameleon", "internvl_chat", "NVLM_D"):
161 return "<image>"
162 if model_type == "mllama":
163 return "<|image|>"
vllm/model_executor/layers/layernorm.py
+30 -2
@@ -18,10 +18,16 @@ class RMSNorm(CustomOp):
18 self,
19 hidden_size: int,
20 eps: float = 1e-6,
21 + var_hidden_size: Optional[int] = None,
22 ) -> None:
23 super().__init__()
23 - self.weight = nn.Parameter(torch.ones(hidden_size))
24 +
25 + self.hidden_size = hidden_size
26 self.variance_epsilon = eps
27 + self.variance_size_override = (None if var_hidden_size == hidden_size
28 + else var_hidden_size)
29 +
30 + self.weight = nn.Parameter(torch.ones(hidden_size))
31
32 def forward_native(
33 self,
@@ -35,7 +41,23 @@ class RMSNorm(CustomOp):
41 x = x + residual.to(torch.float32)
42 residual = x.to(orig_dtype)
43
38 - variance = x.pow(2).mean(dim=-1, keepdim=True)
44 + hidden_size = x.shape[-1]
45 + if hidden_size != self.hidden_size:
46 + raise ValueError("Expected hidden_size to be "
47 + f"{self.hidden_size}, but found: {hidden_size}")
48 +
49 + if self.variance_size_override is None:
50 + x_var = x
51 + else:
52 + if hidden_size < self.variance_size_override:
53 + raise ValueError(
54 + "Expected hidden_size to be at least "
55 + f"{self.variance_size_override}, but found: {hidden_size}")
56 +
57 + x_var = x[:, :, :self.variance_size_override]
58 +
59 + variance = x_var.pow(2).mean(dim=-1, keepdim=True)
60 +
61 x = x * torch.rsqrt(variance + self.variance_epsilon)
62 x = x.to(orig_dtype) * self.weight
63 if residual is None:
@@ -48,6 +70,9 @@ class RMSNorm(CustomOp):
70 x: torch.Tensor,
71 residual: Optional[torch.Tensor] = None,
72 ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
73 + if self.variance_size_override is not None:
74 + return self.forward_native(x, residual)
75 +
76 from vllm import _custom_ops as ops
77
78 if residual is not None:
@@ -72,6 +97,9 @@ class RMSNorm(CustomOp):
97 x: torch.Tensor,
98 residual: Optional[torch.Tensor] = None,
99 ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
100 + if self.variance_size_override is not None:
101 + return self.forward_native(x, residual)
102 +
103 from vllm._ipex_ops import ipex_ops as ops
104
105 if residual is not None:
vllm/model_executor/models/intern_vit.py
+134 -72
@@ -4,6 +4,7 @@
4 # Copyright (c) 2023 OpenGVLab
5 # Licensed under The MIT License [see LICENSE for details]
6 # --------------------------------------------------------
7 +from functools import partial
8 from typing import Iterable, Optional, Tuple
9
10 import torch
@@ -11,7 +12,10 @@ import torch.nn as nn
12 import torch.nn.functional as F
13 from transformers import PretrainedConfig
14
14 -from vllm.distributed import divide, get_tensor_model_parallel_world_size
15 +from vllm.distributed import (divide, get_tensor_model_parallel_rank,
16 + get_tensor_model_parallel_world_size,
17 + split_tensor_along_last_dim,
18 + tensor_model_parallel_all_gather)
19 from vllm.model_executor.layers.activation import get_act_fn
20 from vllm.model_executor.layers.layernorm import RMSNorm
21 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -54,7 +58,7 @@ class InternVisionEmbeddings(nn.Module):
58 self.position_embedding = nn.Parameter(
59 torch.randn(1, self.num_positions, self.embed_dim))
60
57 - def _get_pos_embed(self, pos_embed, H, W):
61 + def _get_pos_embed(self, pos_embed: torch.Tensor, H: int, W: int):
62 target_dtype = pos_embed.dtype
63 pos_embed = pos_embed.float().reshape(
64 1, self.image_size // self.patch_size,
@@ -63,9 +67,21 @@ class InternVisionEmbeddings(nn.Module):
67 size=(H, W),
68 mode='bicubic',
69 align_corners=False)
66 - pos_embed = pos_embed.reshape(1, -1, H * W).permute(0, 2,
67 - 1).to(target_dtype)
68 - return pos_embed
70 + return pos_embed.reshape(1, -1, H * W).permute(0, 2,
71 + 1).to(target_dtype)
72 +
73 + def _get_position_embedding(self, H: int, W: int) -> torch.Tensor:
74 + position_embedding = self.position_embedding
75 + if self.num_patches == H * W:
76 + return position_embedding
77 +
78 + return torch.cat(
79 + [
80 + position_embedding[:, :1, :],
81 + self._get_pos_embed(position_embedding[:, 1:, :], H, W),
82 + ],
83 + dim=1,
84 + )
85
86 def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
87 target_dtype = self.patch_embedding.weight.dtype
@@ -76,12 +92,7 @@ class InternVisionEmbeddings(nn.Module):
92 class_embeds = self.class_embedding.expand(batch_size, 1,
93 -1).to(target_dtype)
94 embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
79 - position_embedding = torch.cat([
80 - self.position_embedding[:, :1, :],
81 - self._get_pos_embed(self.position_embedding[:, 1:, :], height,
82 - width)
83 - ],
84 - dim=1)
95 + position_embedding = self._get_position_embedding(height, width)
96 embeddings = embeddings + position_embedding.to(target_dtype)
97 return embeddings
98
@@ -93,8 +104,11 @@ class InternParallelAttention(nn.Module):
104 self,
105 config: PretrainedConfig,
106 quant_config: Optional[QuantizationConfig] = None,
96 - ):
107 + *,
108 + num_dummy_heads: int = 0,
109 + ) -> None:
110 super().__init__()
111 +
112 self.config = config
113 self.embed_dim = config.hidden_size
114 self.num_heads = config.num_attention_heads
@@ -105,11 +119,19 @@ class InternParallelAttention(nn.Module):
119 f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
120 f' {self.num_heads}).')
121
122 + self.tp_size = get_tensor_model_parallel_world_size()
123 + self.tp_rank = get_tensor_model_parallel_rank()
124 +
125 + # Additional dummy heads are used to enable TP for common GPU counts.
126 + self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim
127 + self.num_heads_per_partition = divide(num_dummy_heads + self.num_heads,
128 + self.tp_size)
129 +
130 self.scale = self.head_dim**-0.5
131 self.qkv = QKVParallelLinear(
132 self.embed_dim,
133 self.head_dim,
112 - self.num_heads,
134 + num_dummy_heads + self.num_heads,
135 bias=config.qkv_bias,
136 quant_config=quant_config,
137 )
@@ -117,34 +139,44 @@ class InternParallelAttention(nn.Module):
139 self.qk_normalization = config.qk_normalization
140
141 if self.qk_normalization:
120 - self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
121 - self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
142 + self.q_norm = RMSNorm(self.dummy_dim,
143 + eps=config.layer_norm_eps,
144 + var_hidden_size=self.embed_dim)
145 + self.k_norm = RMSNorm(self.dummy_dim,
146 + eps=config.layer_norm_eps,
147 + var_hidden_size=self.embed_dim)
148
149 self.proj = RowParallelLinear(
124 - self.embed_dim,
150 + self.dummy_dim,
151 self.embed_dim,
152 quant_config=quant_config,
153 )
154
129 - self.tp_size = get_tensor_model_parallel_world_size()
130 - self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
131 -
132 - def forward(self, x):
133 - B, N, C = x.shape
155 + def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
156 + if self.tp_size > 1:
157 + q = tensor_model_parallel_all_gather(q.contiguous())
158 + k = tensor_model_parallel_all_gather(k.contiguous())
159 + q = self.q_norm.forward_native(q)
160 + k = self.k_norm.forward_native(k)
161 + if self.tp_size > 1:
162 + splitter = partial(split_tensor_along_last_dim,
163 + num_partitions=self.tp_size)
164 + q = splitter(q)[self.tp_rank]
165 + k = splitter(k)[self.tp_rank]
166 + return q, k
167 +
168 + def forward(self, x: torch.Tensor) -> torch.Tensor:
169 + B, N, _ = x.shape
170 qkv, _ = self.qkv(x)
171 q, k, v = qkv.chunk(3, dim=-1)
172
173 + if self.qk_normalization:
174 + q, k = self._apply_qk_norm(q, k)
175 +
176 q = q.view(B, N, self.num_heads_per_partition, self.head_dim)
177 k = k.view(B, N, self.num_heads_per_partition, self.head_dim)
178 v = v.view(B, N, self.num_heads_per_partition, self.head_dim)
179
141 - if self.qk_normalization:
142 - B_, N_, H_, D_ = q.shape
143 - q = self.q_norm.forward_native(q.flatten(-2,
144 - -1)).view(B_, N_, H_, D_)
145 - k = self.k_norm.forward_native(k.flatten(-2,
146 - -1)).view(B_, N_, H_, D_)
147 -
180 x = xops.memory_efficient_attention_forward(q, k, v, scale=self.scale)
181 x = x.view(B, N, -1)
182
@@ -155,8 +187,14 @@ class InternParallelAttention(nn.Module):
187 class InternSdpaAttention(nn.Module):
188 """Multi-headed attention from 'Attention Is All You Need' paper"""
189
158 - def __init__(self, config: PretrainedConfig):
190 + def __init__(
191 + self,
192 + config: PretrainedConfig,
193 + *,
194 + num_dummy_heads: int = 0,
195 + ) -> None:
196 super().__init__()
197 +
198 self.config = config
199 self.embed_dim = config.hidden_size
200 self.num_heads = config.num_attention_heads
@@ -167,20 +205,27 @@ class InternSdpaAttention(nn.Module):
205 f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
206 f' {self.num_heads}).')
207
208 + # Additional dummy heads are used to enable TP for common GPU counts.
209 + self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim
210 +
211 self.scale = self.head_dim**-0.5
212 self.qkv = nn.Linear(self.embed_dim,
172 - 3 * self.embed_dim,
213 + 3 * self.dummy_dim,
214 bias=config.qkv_bias)
215
216 self.qk_normalization = config.qk_normalization
217
218 if self.qk_normalization:
178 - self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
179 - self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
219 + self.q_norm = RMSNorm(self.dummy_dim,
220 + eps=config.layer_norm_eps,
221 + var_hidden_size=self.embed_dim)
222 + self.k_norm = RMSNorm(self.dummy_dim,
223 + eps=config.layer_norm_eps,
224 + var_hidden_size=self.embed_dim)
225
181 - self.proj = nn.Linear(self.embed_dim, self.embed_dim)
226 + self.proj = nn.Linear(self.dummy_dim, self.embed_dim)
227
183 - def forward(self, x):
228 + def forward(self, x: torch.Tensor) -> torch.Tensor:
229 B, N, C = x.shape
230 qkv = self.qkv(x)
231 q, k, v = qkv.chunk(3, dim=-1)
@@ -233,22 +278,23 @@ class InternMLP(nn.Module):
278
279 class InternVisionEncoderLayer(nn.Module):
280
236 - def __init__(self,
237 - config: PretrainedConfig,
238 - quant_config: Optional[QuantizationConfig] = None):
281 + def __init__(
282 + self,
283 + config: PretrainedConfig,
284 + quant_config: Optional[QuantizationConfig] = None,
285 + *,
286 + num_dummy_heads: int = 0,
287 + ) -> None:
288 super().__init__()
289 +
290 self.embed_dim = config.hidden_size
291 self.intermediate_size = config.intermediate_size
292 self.norm_type = config.norm_type
293
244 - # fallback to sdpa attention if tp unavailable
245 - tp_size = get_tensor_model_parallel_world_size()
246 - num_heads = config.num_attention_heads
247 - if USE_XFORMERS_OPS and num_heads % tp_size == 0:
248 - self.attn = InternParallelAttention(config,
249 - quant_config=quant_config)
250 - else:
251 - self.attn = InternSdpaAttention(config)
294 + self.attn = self._init_attn(config,
295 + quant_config,
296 + num_dummy_heads=num_dummy_heads)
297 +
298 self.mlp = InternMLP(config, quant_config=quant_config)
299 self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
300 eps=config.layer_norm_eps)
@@ -260,6 +306,24 @@ class InternVisionEncoderLayer(nn.Module):
306 self.ls2 = nn.Parameter(config.initializer_factor *
307 torch.ones(self.embed_dim))
308
309 + def _init_attn(
310 + self,
311 + config: PretrainedConfig,
312 + quant_config: Optional[QuantizationConfig],
313 + *,
314 + num_dummy_heads: int,
315 + ):
316 + # fallback to sdpa attention if tp unavailable
317 + tp_size = get_tensor_model_parallel_world_size()
318 + num_heads = config.num_attention_heads
319 +
320 + if USE_XFORMERS_OPS and (num_heads + num_dummy_heads) % tp_size == 0:
321 + return InternParallelAttention(config,
322 + quant_config=quant_config,
323 + num_dummy_heads=num_dummy_heads)
324 +
325 + return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads)
326 +
327 def forward(
328 self,
329 hidden_states: torch.Tensor,
@@ -275,19 +339,27 @@ class InternVisionEncoderLayer(nn.Module):
339
340 class InternVisionEncoder(nn.Module):
341
278 - def __init__(self,
279 - config: PretrainedConfig,
280 - quant_config: Optional[QuantizationConfig] = None,
281 - num_hidden_layers_override: Optional[int] = None):
342 + def __init__(
343 + self,
344 + config: PretrainedConfig,
345 + quant_config: Optional[QuantizationConfig] = None,
346 + *,
347 + num_hidden_layers_override: Optional[int] = None,
348 + num_dummy_heads: int = 0,
349 + ):
350 super().__init__()
351 +
352 self.config = config
353
354 if num_hidden_layers_override is None:
355 num_hidden_layers = config.num_hidden_layers
356 else:
357 num_hidden_layers = num_hidden_layers_override
358 +
359 self.layers = nn.ModuleList([
290 - InternVisionEncoderLayer(config=config, quant_config=quant_config)
360 + InternVisionEncoderLayer(config,
361 + quant_config,
362 + num_dummy_heads=num_dummy_heads)
363 for _ in range(num_hidden_layers)
364 ])
365
@@ -302,35 +374,25 @@ class InternVisionEncoder(nn.Module):
374
375 class InternVisionModel(nn.Module):
376
305 - def __init__(self,
306 - config: PretrainedConfig,
307 - quant_config: Optional[QuantizationConfig] = None,
308 - num_hidden_layers_override: Optional[int] = None):
377 + def __init__(
378 + self,
379 + config: PretrainedConfig,
380 + quant_config: Optional[QuantizationConfig] = None,
381 + *,
382 + num_hidden_layers_override: Optional[int] = None,
383 + num_dummy_heads: int = 0,
384 + ):
385 super().__init__()
386 +
387 self.config = config
388
389 self.embeddings = InternVisionEmbeddings(config)
390 self.encoder = InternVisionEncoder(
391 config=config,
392 quant_config=quant_config,
316 - num_hidden_layers_override=num_hidden_layers_override)
317 -
318 - def resize_pos_embeddings(self, old_size, new_size, patch_size):
319 - pos_emb = self.embeddings.position_embedding
320 - _, num_positions, embed_dim = pos_emb.shape
321 - cls_emb = pos_emb[:, :1, :]
322 - pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size,
323 - old_size // patch_size,
324 - -1).permute(0, 3, 1, 2)
325 - pos_emb = F.interpolate(pos_emb.float(),
326 - size=new_size // patch_size,
327 - mode='bicubic',
328 - align_corners=False)
329 - pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim,
330 - -1).permute(0, 2, 1)
331 - pos_emb = torch.cat([cls_emb, pos_emb], dim=1)
332 - self.embeddings.position_embedding = nn.Parameter(pos_emb)
333 - self.embeddings.image_size = new_size
393 + num_hidden_layers_override=num_hidden_layers_override,
394 + num_dummy_heads=num_dummy_heads,
395 + )
396
397 def get_input_embeddings(self):
398 return self.embeddings
vllm/model_executor/models/internvl.py
+173 -121
@@ -237,130 +237,173 @@ def get_max_internvl_image_size(ctx: InputContext,
237 return width, height
238
239
240 -def input_processor_for_internvl(ctx: InputContext,
241 - llm_inputs: LLMInputs,
242 - *,
243 - max_dynamic_patch: Optional[int] = None):
244 - multi_modal_data = llm_inputs.get("multi_modal_data")
245 - if multi_modal_data is None or "image" not in multi_modal_data:
246 - return llm_inputs
240 +class InternVLInputPipeline:
241
248 - model_config = ctx.model_config
249 - hf_config = ctx.get_hf_config()
242 + def __init__(
243 + self,
244 + img_start_token: str,
245 + img_end_token: str,
246 + img_context_token: str,
247 + ) -> None:
248 + super().__init__()
249
251 - image_data = multi_modal_data["image"]
252 - num_patches = get_internvl_num_patches(hf_config)
253 - num_blocks_calculator = calculate_num_blocks_wrapper(
254 - hf_config, max_dynamic_patch)
255 - if isinstance(image_data, Image.Image):
256 - width, height = image_data.size
257 - num_blocks, _, _ = num_blocks_calculator(width, height)
258 - image_feature_size = [num_blocks * num_patches]
259 - elif is_list_of(image_data, Image.Image):
260 - image_feature_size = []
261 - for image in image_data:
262 - width, height = image.size
250 + self.img_start_token = img_start_token
251 + self.img_end_token = img_end_token
252 + self.img_context_token = img_context_token
253 +
254 + def _create_image_prompt(self, feature_size: int, num_patches: int) -> str:
255 + return (self.img_start_token + self.img_context_token * feature_size +
256 + self.img_end_token)
257 +
258 + def _expand_image_prompt(
259 + self,
260 + prompt: str,
261 + feature_sizes: List[int],
262 + num_patches: int,
263 + ) -> str:
264 + image_idx = sorted(
265 + map(int, re.findall(r"Image-(\d+): <image>\n", prompt)))
266 +
267 + new_prompt = prompt
268 + for idx, feature_size in enumerate(feature_sizes, start=1):
269 + image_prompt = self._create_image_prompt(feature_size, num_patches)
270 + if not image_idx:
271 + image_prompt = f"Image-{idx}: {image_prompt}"
272 +
273 + new_prompt = new_prompt.replace('<image>', image_prompt, 1)
274 +
275 + return new_prompt
276 +
277 + def input_processor(
278 + self,
279 + ctx: InputContext,
280 + llm_inputs: LLMInputs,
281 + *,
282 + max_dynamic_patch: Optional[int] = None,
283 + ) -> LLMInputs:
284 + multi_modal_data = llm_inputs.get("multi_modal_data")
285 + if multi_modal_data is None or "image" not in multi_modal_data:
286 + return llm_inputs
287 +
288 + model_config = ctx.model_config
289 + hf_config = ctx.get_hf_config()
290 +
291 + image_data = multi_modal_data["image"]
292 + num_patches = get_internvl_num_patches(hf_config)
293 + num_blocks_calculator = calculate_num_blocks_wrapper(
294 + hf_config, max_dynamic_patch)
295 + if isinstance(image_data, Image.Image):
296 + width, height = image_data.size
297 num_blocks, _, _ = num_blocks_calculator(width, height)
264 - image_feature_size.append(num_blocks * num_patches)
265 - elif isinstance(image_data, torch.Tensor):
266 - num_images, image_feature_size, hidden_size = image_data.shape
267 - else:
268 - raise TypeError(f"Invalid image type: {type(image_data)}")
269 -
270 - tokenizer = cached_get_tokenizer(
271 - model_config.tokenizer,
272 - trust_remote_code=model_config.trust_remote_code)
273 -
274 - prompt = llm_inputs.get("prompt")
275 - prompt_token_ids = llm_inputs["prompt_token_ids"]
276 - if prompt is None:
277 - prompt = tokenizer.decode(prompt_token_ids)
278 -
279 - new_prompt = prompt
280 - image_idx = sorted(map(int, re.findall(r"Image-(\d+): <image>\n", prompt)))
281 - for idx, feature_size in enumerate(image_feature_size, start=1):
282 - image_prompt = IMG_START + IMG_CONTEXT * feature_size + IMG_END
283 - if not image_idx:
284 - image_prompt = f"Image-{idx}: {image_prompt}"
285 - new_prompt = new_prompt.replace('<image>', image_prompt, 1)
286 - new_prompt_token_ids = tokenizer.encode(new_prompt)
287 -
288 - return LLMInputs(prompt=prompt,
289 - prompt_token_ids=new_prompt_token_ids,
290 - multi_modal_data=multi_modal_data)
291 -
292 -
293 -def input_mapper_for_internvl(ctx: InputContext,
294 - data: object,
295 - *,
296 - max_dynamic_patch: Optional[int] = None):
297 - hf_config = ctx.get_hf_config()
298 + image_feature_sizes = [num_blocks * num_patches]
299 + elif is_list_of(image_data, Image.Image):
300 + image_feature_sizes = []
301 + for image in image_data:
302 + width, height = image.size
303 + num_blocks, _, _ = num_blocks_calculator(width, height)
304 + image_feature_sizes.append(num_blocks * num_patches)
305 + elif isinstance(image_data, torch.Tensor):
306 + num_images, image_feature_size, hidden_size = image_data.shape
307 + image_feature_sizes = [image_feature_size]
308 + else:
309 + raise TypeError(f"Invalid image type: {type(image_data)}")
310
299 - image_pixel_values_mapper = image_to_pixel_values_wrapper(
300 - hf_config, max_dynamic_patch)
301 - if isinstance(data, Image.Image):
302 - data = image_pixel_values_mapper(data)
303 - # Add an N dimension for number of images per prompt (currently 1).
304 - data = data.unsqueeze(0)
305 - elif is_list_of(data, Image.Image):
306 - # we can't stack here because the images may have different num_patches
307 - data = [image_pixel_values_mapper(img) for img in data]
308 - model_config = ctx.model_config
309 - tokenizer = cached_get_tokenizer(
310 - model_config.tokenizer,
311 - trust_remote_code=model_config.trust_remote_code)
312 - image_token_id = tokenizer.encode(IMG_CONTEXT,
313 - add_special_tokens=False,
314 - return_tensors="pt")[0]
315 -
316 - return MultiModalInputs({
317 - "pixel_values": data,
318 - "image_token_id": image_token_id
319 - })
320 -
321 -
322 -def dummy_data_for_internvl(ctx: InputContext,
323 - seq_len: int,
324 - mm_counts: Mapping[str, int],
325 - *,
326 - max_dynamic_patch: Optional[int] = None):
327 - num_images = mm_counts["image"]
311 + tokenizer = cached_get_tokenizer(
312 + model_config.tokenizer,
313 + trust_remote_code=model_config.trust_remote_code)
314
329 - hf_config = ctx.get_hf_config()
315 + prompt = llm_inputs.get("prompt")
316 + prompt_token_ids = llm_inputs["prompt_token_ids"]
317 + if prompt is None:
318 + prompt = tokenizer.decode(prompt_token_ids)
319 +
320 + new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
321 + num_patches)
322 + new_prompt_token_ids = tokenizer.encode(new_prompt)
323 +
324 + return LLMInputs(prompt=prompt,
325 + prompt_token_ids=new_prompt_token_ids,
326 + multi_modal_data=multi_modal_data)
327
331 - image_feature_size = get_max_internvl_image_tokens(
332 - ctx, max_dynamic_patch=max_dynamic_patch)
333 - model_config = ctx.model_config
334 - tokenizer = cached_get_tokenizer(
335 - model_config.tokenizer,
336 - trust_remote_code=model_config.trust_remote_code)
328 + def input_mapper(
329 + self,
330 + ctx: InputContext,
331 + data: object,
332 + *,
333 + max_dynamic_patch: Optional[int] = None,
334 + ):
335 + hf_config = ctx.get_hf_config()
336 +
337 + image_pixel_values_mapper = image_to_pixel_values_wrapper(
338 + hf_config, max_dynamic_patch)
339 + if isinstance(data, Image.Image):
340 + data = image_pixel_values_mapper(data)
341 + # Add an N dimension for number of images per prompt (currently 1).
342 + data = data.unsqueeze(0)
343 + elif is_list_of(data, Image.Image):
344 + # we can't stack here because images may have different num_patches
345 + data = [image_pixel_values_mapper(img) for img in data]
346 + model_config = ctx.model_config
347 + tokenizer = cached_get_tokenizer(
348 + model_config.tokenizer,
349 + trust_remote_code=model_config.trust_remote_code)
350 + image_token_id = tokenizer.encode(self.img_context_token,
351 + add_special_tokens=False,
352 + return_tensors="pt")[0]
353 +
354 + return MultiModalInputs({
355 + "pixel_values": data,
356 + "image_token_id": image_token_id
357 + })
358 +
359 + def dummy_data(
360 + self,
361 + ctx: InputContext,
362 + seq_len: int,
363 + mm_counts: Mapping[str, int],
364 + *,
365 + max_dynamic_patch: Optional[int] = None,
366 + ):
367 + num_images = mm_counts["image"]
368 +
369 + hf_config = ctx.get_hf_config()
370 +
371 + image_feature_size = get_max_internvl_image_tokens(
372 + ctx, max_dynamic_patch=max_dynamic_patch)
373 + model_config = ctx.model_config
374 + tokenizer = cached_get_tokenizer(
375 + model_config.tokenizer,
376 + trust_remote_code=model_config.trust_remote_code)
377 +
378 + seq_data = dummy_seq_data_for_clip(
379 + hf_config.vision_config,
380 + seq_len,
381 + num_images,
382 + image_token_id=tokenizer.encode(self.img_context_token,
383 + add_special_tokens=False)[0],
384 + image_feature_size_override=image_feature_size,
385 + )
386
338 - seq_data = dummy_seq_data_for_clip(
339 - hf_config.vision_config,
340 - seq_len,
341 - num_images,
342 - image_token_id=tokenizer.encode(IMG_CONTEXT,
343 - add_special_tokens=False)[0],
344 - image_feature_size_override=image_feature_size,
345 - )
387 + max_image_width, max_image_height = get_max_internvl_image_size(
388 + ctx, max_dynamic_patch=max_dynamic_patch)
389
347 - max_image_width, max_image_height = get_max_internvl_image_size(
348 - ctx, max_dynamic_patch=max_dynamic_patch)
390 + mm_data = dummy_image_for_clip(
391 + hf_config.vision_config,
392 + num_images,
393 + image_width_override=max_image_width,
394 + image_height_override=max_image_height,
395 + )
396
350 - mm_data = dummy_image_for_clip(
351 - hf_config.vision_config,
352 - num_images,
353 - image_width_override=max_image_width,
354 - image_height_override=max_image_height,
355 - )
397 + return seq_data, mm_data
398
357 - return seq_data, mm_data
399
400 +input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
401
360 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_internvl)
402 +
403 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
404 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
362 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_internvl)
363 -@INPUT_REGISTRY.register_input_processor(input_processor_for_internvl)
405 +@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
406 +@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
407 class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
408
409 def __init__(self,
@@ -388,20 +431,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
431 + vision_feature_layer + 1
432 else:
433 num_hidden_layers = vision_feature_layer + 1
391 - self.vision_model = InternVisionModel(
392 - config.vision_config, num_hidden_layers_override=num_hidden_layers)
434 + self.vision_model = self._init_vision_model(config, num_hidden_layers)
435
436 self.language_model = init_vllm_registered_model(
437 config.text_config, cache_config, quant_config)
438
397 - vit_hidden_size = config.vision_config.hidden_size
398 - llm_hidden_size = config.text_config.hidden_size
399 -
400 - self.mlp1 = nn.Sequential(
401 - nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
402 - nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
403 - llm_hidden_size), nn.GELU(),
404 - nn.Linear(llm_hidden_size, llm_hidden_size))
439 + self.mlp1 = self._init_mlp1(config)
440
441 self.img_context_token_id = None
442 self.make_empty_intermediate_tensors = (
@@ -414,6 +449,23 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
449
450 return Sampler()
451
452 + def _init_vision_model(self, config: PretrainedConfig,
453 + num_hidden_layers: int):
454 + return InternVisionModel(config.vision_config,
455 + num_hidden_layers_override=num_hidden_layers)
456 +
457 + def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
458 + vit_hidden_size = config.vision_config.hidden_size
459 + llm_hidden_size = config.text_config.hidden_size
460 +
461 + return nn.Sequential(
462 + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
463 + nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
464 + llm_hidden_size),
465 + nn.GELU(),
466 + nn.Linear(llm_hidden_size, llm_hidden_size),
467 + )
468 +
469 def pixel_shuffle(self, x, scale_factor=0.5):
470 n, w, h, c = x.size()
471 # N, W, H, C --> N, W, H * scale, C // scale
vllm/model_executor/models/nvlm_d.py new
+64
@@ -0,0 +1,64 @@
1 +# adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py
2 +# --------------------------------------------------------
3 +# NVLM-D
4 +# Copyright (c) 2024 NVIDIA
5 +# Licensed under Apache 2.0 License [see LICENSE for details]
6 +# --------------------------------------------------------
7 +import torch.nn as nn
8 +from transformers import PretrainedConfig
9 +
10 +from vllm.inputs import INPUT_REGISTRY
11 +from vllm.multimodal import MULTIMODAL_REGISTRY
12 +
13 +from .intern_vit import InternVisionModel
14 +from .internvl import (InternVLChatModel, InternVLInputPipeline,
15 + get_max_internvl_image_tokens)
16 +
17 +IMG_START = '<|vision_start|>'
18 +IMG_END = '<|vision_end|>'
19 +IMG_CONTEXT = '<|vision_pad|>'
20 +
21 +
22 +class NVLMInputPipeline(InternVLInputPipeline):
23 +
24 + def _create_image_prompt(self, feature_size: int, num_patches: int) -> str:
25 + tile_pos_identifiers = ([f"<tile_{i}>"
26 + for i in range(1, num_patches)] +
27 + ["<tile_global_thumbnail>"])
28 + context_size = feature_size // num_patches
29 +
30 + return '<Image>' + ''.join(
31 + tile_pos_identifier + self.img_context_token * context_size
32 + for tile_pos_identifier in tile_pos_identifiers) + '</Image>'
33 +
34 +
35 +input_pipeline = NVLMInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
36 +
37 +
38 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
39 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
40 +@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
41 +@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
42 +class NVLM_D_Model(InternVLChatModel):
43 +
44 + def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
45 + vit_hidden_size = config.vision_config.hidden_size
46 + llm_intermediate_size = config.text_config.intermediate_size
47 + llm_hidden_size = config.text_config.hidden_size
48 +
49 + return nn.Sequential(
50 + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
51 + nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
52 + llm_intermediate_size,
53 + bias=False),
54 + nn.GELU(),
55 + nn.Linear(llm_intermediate_size, llm_hidden_size, bias=False),
56 + )
57 +
58 + def _init_vision_model(self, config: PretrainedConfig,
59 + num_hidden_layers: int):
60 + # We added additional dummy heads to the original num of heads to make
61 + # the number of heads divisible by 8.
62 + return InternVisionModel(config.vision_config,
63 + num_hidden_layers_override=num_hidden_layers,
64 + num_dummy_heads=7)
vllm/model_executor/models/registry.py
+15 -22
@@ -16,6 +16,7 @@ from .interfaces_base import is_embedding_model, is_text_generation_model
16
17 logger = init_logger(__name__)
18
19 +# yapf: disable
20 _TEXT_GENERATION_MODELS = {
21 # [Decoder-only]
22 "AquilaModel": ("llama", "LlamaForCausalLM"),
@@ -68,8 +69,6 @@ _TEXT_GENERATION_MODELS = {
69 "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
70 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
71 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
71 - "Qwen2VLForConditionalGeneration":
72 - ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
72 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
73 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
74 "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
@@ -88,32 +87,25 @@ _EMBEDDING_MODELS = {
87 }
88
89 _MULTIMODAL_MODELS = {
91 - "Blip2ForConditionalGeneration":
92 - ("blip2", "Blip2ForConditionalGeneration"),
93 - "ChameleonForConditionalGeneration":
94 - ("chameleon", "ChameleonForConditionalGeneration"),
90 + # [Decoder-only]
91 + "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
92 + "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
93 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
94 "InternVLChatModel": ("internvl", "InternVLChatModel"),
97 - "LlavaForConditionalGeneration": ("llava",
98 - "LlavaForConditionalGeneration"),
99 - "LlavaNextForConditionalGeneration": ("llava_next",
100 - "LlavaNextForConditionalGeneration"),
101 - "LlavaNextVideoForConditionalGeneration":
102 - ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
103 - "LlavaOnevisionForConditionalGeneration":
104 - ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
95 + "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
96 + "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
97 + "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
98 + "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
99 "MiniCPMV": ("minicpmv", "MiniCPMV"),
106 - "PaliGemmaForConditionalGeneration": ("paligemma",
107 - "PaliGemmaForConditionalGeneration"),
100 + "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
101 + "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
102 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
109 - "PixtralForConditionalGeneration": ("pixtral",
110 - "PixtralForConditionalGeneration"),
103 + "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
104 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
112 - "Qwen2VLForConditionalGeneration": ("qwen2_vl",
113 - "Qwen2VLForConditionalGeneration"),
105 + "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
106 "UltravoxModel": ("ultravox", "UltravoxModel"),
115 - "MllamaForConditionalGeneration": ("mllama",
116 - "MllamaForConditionalGeneration"),
107 + # [Encoder-decoder]
108 + "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
109 }
110
111 _SPECULATIVE_DECODING_MODELS = {
@@ -121,6 +113,7 @@ _SPECULATIVE_DECODING_MODELS = {
113 "MedusaModel": ("medusa", "Medusa"),
114 "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
115 }
116 +# yapf: enable
117
118 _MODELS = {
119 **_TEXT_GENERATION_MODELS,
vllm/transformers_utils/config.py
+4 -3
@@ -22,9 +22,9 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
22 InternVLChatConfig, JAISConfig,
23 MedusaConfig, MllamaConfig,
24 MLPSpeculatorConfig, MPTConfig,
25 - NemotronConfig, Qwen2VLConfig,
26 - RWConfig, SolarConfig,
27 - UltravoxConfig)
25 + NemotronConfig, NVLM_D_Config,
26 + Qwen2VLConfig, RWConfig,
27 + SolarConfig, UltravoxConfig)
28 # yapf: enable
29 from vllm.transformers_utils.utils import check_gguf_file
30
@@ -54,6 +54,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
54 "exaone": ExaoneConfig,
55 "internvl_chat": InternVLChatConfig,
56 "nemotron": NemotronConfig,
57 + "NVLM_D": NVLM_D_Config,
58 "solar": SolarConfig,
59 "ultravox": UltravoxConfig,
60 "qwen2_vl": Qwen2VLConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.transformers_utils.configs.mllama import MllamaConfig
13 from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
14 from vllm.transformers_utils.configs.mpt import MPTConfig
15 from vllm.transformers_utils.configs.nemotron import NemotronConfig
16 +from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config
17 from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig,
18 Qwen2VLVisionConfig)
19 from vllm.transformers_utils.configs.solar import SolarConfig
@@ -31,6 +32,7 @@ __all__ = [
32 "MllamaConfig",
33 "MLPSpeculatorConfig",
34 "NemotronConfig",
35 + "NVLM_D_Config",
36 "SolarConfig",
37 "UltravoxConfig",
38 "Qwen2VLConfig",
vllm/transformers_utils/configs/nvlm_d.py new
+12
@@ -0,0 +1,12 @@
1 +# Adapted from
2 +# https://huggingface.co/nvidia/NVLM-D-72B/blob/main/configuration_nvlm_d.py
3 +# --------------------------------------------------------
4 +# NVLM-D
5 +# Copyright (c) 2024 NVIDIA
6 +# Licensed under Apache 2.0 License [see LICENSE for details]
7 +# --------------------------------------------------------
8 +from .internvl import InternVLChatConfig
9 +
10 +
11 +class NVLM_D_Config(InternVLChatConfig):
12 + model_type = 'NVLM_D'