A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] Support NVLM-D and fix QK Norm in InternViT (#9045)
Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Cyrus Leung committed
Oct 7, 2024 at 19:55 UTC
151ef4efd2fb52554f4d30408aca619e181ea751
12 files changed
+518
-236
docs/source/models/supported_models.rst
+9
@@ -315,6 +315,9 @@ Multimodal Language Models
315
316
.. _supported_vlms:
317
318
+Text Generation
319
+---------------
320
+
321
.. list-table::
322
:widths: 25 25 25 25 5 5
323
:header-rows: 1
@@ -384,7 +387,13 @@ Multimodal Language Models
387
- Image
388
- :code:`meta-llama/Llama-3.2-90B-Vision-Instruct`, :code:`meta-llama/Llama-3.2-11B-Vision`, etc.
389
-
390
+ -
391
+ * - :code:`NVLM_D_Model`
392
+ - NVLM-D 1.0
393
+ - Image\ :sup:`E+`
394
+ - :code:`nvidia/NVLM-D-72B`, etc.
395
-
396
+ - ✅︎
397
* - :code:`PaliGemmaForConditionalGeneration`
398
- PaliGemma
399
- Image\ :sup:`E`
examples/offline_inference_vision_language.py
+40
-15
@@ -18,7 +18,7 @@ from vllm.utils import FlexibleArgumentParser
18
19
20
# LLaVA-1.5
21
-def run_llava(question, modality):
21
+def run_llava(question: str, modality: str):
22
assert modality == "image"
23
24
prompt = f"USER: <image>\n{question}\nASSISTANT:"
@@ -29,7 +29,7 @@ def run_llava(question, modality):
29
30
31
# LLaVA-1.6/LLaVA-NeXT
32
-def run_llava_next(question, modality):
32
+def run_llava_next(question: str, modality: str):
33
assert modality == "image"
34
35
prompt = f"[INST] <image>\n{question} [/INST]"
@@ -40,7 +40,7 @@ def run_llava_next(question, modality):
40
41
# LlaVA-NeXT-Video
42
# Currently only support for video input
43
-def run_llava_next_video(question, modality):
43
+def run_llava_next_video(question: str, modality: str):
44
assert modality == "video"
45
46
prompt = f"USER: <video>\n{question} ASSISTANT:"
@@ -50,7 +50,7 @@ def run_llava_next_video(question, modality):
50
51
52
# LLaVA-OneVision
53
-def run_llava_onevision(question, modality):
53
+def run_llava_onevision(question: str, modality: str):
54
55
if modality == "video":
56
prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
@@ -67,7 +67,7 @@ def run_llava_onevision(question, modality):
67
68
69
# Fuyu
70
-def run_fuyu(question, modality):
70
+def run_fuyu(question: str, modality: str):
71
assert modality == "image"
72
73
prompt = f"{question}\n"
@@ -77,7 +77,7 @@ def run_fuyu(question, modality):
77
78
79
# Phi-3-Vision
80
-def run_phi3v(question, modality):
80
+def run_phi3v(question: str, modality: str):
81
assert modality == "image"
82
83
prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n" # noqa: E501
@@ -112,7 +112,7 @@ def run_phi3v(question, modality):
112
113
114
# PaliGemma
115
-def run_paligemma(question, modality):
115
+def run_paligemma(question: str, modality: str):
116
assert modality == "image"
117
118
# PaliGemma has special prompt format for VQA
@@ -123,7 +123,7 @@ def run_paligemma(question, modality):
123
124
125
# Chameleon
126
-def run_chameleon(question, modality):
126
+def run_chameleon(question: str, modality: str):
127
assert modality == "image"
128
129
prompt = f"{question}<image>"
@@ -133,7 +133,7 @@ def run_chameleon(question, modality):
133
134
135
# MiniCPM-V
136
-def run_minicpmv(question, modality):
136
+def run_minicpmv(question: str, modality: str):
137
assert modality == "image"
138
139
# 2.0
@@ -176,7 +176,7 @@ def run_minicpmv(question, modality):
176
177
178
# InternVL
179
-def run_internvl(question, modality):
179
+def run_internvl(question: str, modality: str):
180
assert modality == "image"
181
182
model_name = "OpenGVLab/InternVL2-2B"
@@ -203,8 +203,32 @@ def run_internvl(question, modality):
203
return llm, prompt, stop_token_ids
204
205
206
+# NVLM-D
207
+def run_nvlm_d(question: str, modality: str):
208
+ assert modality == "image"
209
+
210
+ model_name = "nvidia/NVLM-D-72B"
211
+
212
+ # Adjust this as necessary to fit in GPU
213
+ llm = LLM(
214
+ model=model_name,
215
+ trust_remote_code=True,
216
+ max_model_len=4096,
217
+ tensor_parallel_size=4,
218
+ )
219
+
220
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
221
+ trust_remote_code=True)
222
+ messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
223
+ prompt = tokenizer.apply_chat_template(messages,
224
+ tokenize=False,
225
+ add_generation_prompt=True)
226
+ stop_token_ids = None
227
+ return llm, prompt, stop_token_ids
228
+
229
+
230
# BLIP-2
207
-def run_blip2(question, modality):
231
+def run_blip2(question: str, modality: str):
232
assert modality == "image"
233
234
# BLIP-2 prompt format is inaccurate on HuggingFace model repository.
@@ -216,7 +240,7 @@ def run_blip2(question, modality):
240
241
242
# Qwen
219
-def run_qwen_vl(question, modality):
243
+def run_qwen_vl(question: str, modality: str):
244
assert modality == "image"
245
246
llm = LLM(
@@ -232,7 +256,7 @@ def run_qwen_vl(question, modality):
256
257
258
# Qwen2-VL
235
-def run_qwen2_vl(question, modality):
259
+def run_qwen2_vl(question: str, modality: str):
260
assert modality == "image"
261
262
model_name = "Qwen/Qwen2-VL-7B-Instruct"
@@ -252,8 +276,8 @@ def run_qwen2_vl(question, modality):
276
return llm, prompt, stop_token_ids
277
278
255
-# LLama
256
-def run_mllama(question, modality):
279
+# LLama 3.2
280
+def run_mllama(question: str, modality: str):
281
assert modality == "image"
282
283
model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
@@ -287,6 +311,7 @@ model_example_map = {
311
"minicpmv": run_minicpmv,
312
"blip-2": run_blip2,
313
"internvl_chat": run_internvl,
314
+ "NVLM_D": run_nvlm_d,
315
"qwen_vl": run_qwen_vl,
316
"qwen2_vl": run_qwen2_vl,
317
"mllama": run_mllama,
examples/offline_inference_vision_language_multi_image.py
+34
@@ -144,6 +144,39 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
144
)
145
146
147
+def load_nvlm_d(question: str, image_urls: List[str]):
148
+ model_name = "nvidia/NVLM-D-72B"
149
+
150
+ # Adjust this as necessary to fit in GPU
151
+ llm = LLM(
152
+ model=model_name,
153
+ trust_remote_code=True,
154
+ max_model_len=8192,
155
+ tensor_parallel_size=4,
156
+ limit_mm_per_prompt={"image": len(image_urls)},
157
+ mm_processor_kwargs={"max_dynamic_patch": 4},
158
+ )
159
+
160
+ placeholders = "\n".join(f"Image-{i}: <image>\n"
161
+ for i, _ in enumerate(image_urls, start=1))
162
+ messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
163
+
164
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
165
+ trust_remote_code=True)
166
+ prompt = tokenizer.apply_chat_template(messages,
167
+ tokenize=False,
168
+ add_generation_prompt=True)
169
+ stop_token_ids = None
170
+
171
+ return ModelRequestData(
172
+ llm=llm,
173
+ prompt=prompt,
174
+ stop_token_ids=stop_token_ids,
175
+ image_data=[fetch_image(url) for url in image_urls],
176
+ chat_template=None,
177
+ )
178
+
179
+
180
def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
181
try:
182
from qwen_vl_utils import process_vision_info
@@ -204,6 +237,7 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
237
model_example_map = {
238
"phi3_v": load_phi3v,
239
"internvl_chat": load_internvl,
240
+ "NVLM_D": load_nvlm_d,
241
"qwen2_vl": load_qwen2_vl,
242
"qwen_vl_chat": load_qwenvl_chat,
243
}
vllm/entrypoints/chat_utils.py
+1
-1
@@ -157,7 +157,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
157
if model_type.startswith("llava"):
158
return self._cached_token_str(self._tokenizer,
159
hf_config.image_token_index)
160
- if model_type in ("chameleon", "internvl_chat"):
160
+ if model_type in ("chameleon", "internvl_chat", "NVLM_D"):
161
return "<image>"
162
if model_type == "mllama":
163
return "<|image|>"
vllm/model_executor/layers/layernorm.py
+30
-2
@@ -18,10 +18,16 @@ class RMSNorm(CustomOp):
18
self,
19
hidden_size: int,
20
eps: float = 1e-6,
21
+ var_hidden_size: Optional[int] = None,
22
) -> None:
23
super().__init__()
23
- self.weight = nn.Parameter(torch.ones(hidden_size))
24
+
25
+ self.hidden_size = hidden_size
26
self.variance_epsilon = eps
27
+ self.variance_size_override = (None if var_hidden_size == hidden_size
28
+ else var_hidden_size)
29
+
30
+ self.weight = nn.Parameter(torch.ones(hidden_size))
31
32
def forward_native(
33
self,
@@ -35,7 +41,23 @@ class RMSNorm(CustomOp):
41
x = x + residual.to(torch.float32)
42
residual = x.to(orig_dtype)
43
38
- variance = x.pow(2).mean(dim=-1, keepdim=True)
44
+ hidden_size = x.shape[-1]
45
+ if hidden_size != self.hidden_size:
46
+ raise ValueError("Expected hidden_size to be "
47
+ f"{self.hidden_size}, but found: {hidden_size}")
48
+
49
+ if self.variance_size_override is None:
50
+ x_var = x
51
+ else:
52
+ if hidden_size < self.variance_size_override:
53
+ raise ValueError(
54
+ "Expected hidden_size to be at least "
55
+ f"{self.variance_size_override}, but found: {hidden_size}")
56
+
57
+ x_var = x[:, :, :self.variance_size_override]
58
+
59
+ variance = x_var.pow(2).mean(dim=-1, keepdim=True)
60
+
61
x = x * torch.rsqrt(variance + self.variance_epsilon)
62
x = x.to(orig_dtype) * self.weight
63
if residual is None:
@@ -48,6 +70,9 @@ class RMSNorm(CustomOp):
70
x: torch.Tensor,
71
residual: Optional[torch.Tensor] = None,
72
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
73
+ if self.variance_size_override is not None:
74
+ return self.forward_native(x, residual)
75
+
76
from vllm import _custom_ops as ops
77
78
if residual is not None:
@@ -72,6 +97,9 @@ class RMSNorm(CustomOp):
97
x: torch.Tensor,
98
residual: Optional[torch.Tensor] = None,
99
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
100
+ if self.variance_size_override is not None:
101
+ return self.forward_native(x, residual)
102
+
103
from vllm._ipex_ops import ipex_ops as ops
104
105
if residual is not None:
vllm/model_executor/models/intern_vit.py
+134
-72
@@ -4,6 +4,7 @@
4
# Copyright (c) 2023 OpenGVLab
5
# Licensed under The MIT License [see LICENSE for details]
6
# --------------------------------------------------------
7
+from functools import partial
8
from typing import Iterable, Optional, Tuple
9
10
import torch
@@ -11,7 +12,10 @@ import torch.nn as nn
12
import torch.nn.functional as F
13
from transformers import PretrainedConfig
14
14
-from vllm.distributed import divide, get_tensor_model_parallel_world_size
15
+from vllm.distributed import (divide, get_tensor_model_parallel_rank,
16
+ get_tensor_model_parallel_world_size,
17
+ split_tensor_along_last_dim,
18
+ tensor_model_parallel_all_gather)
19
from vllm.model_executor.layers.activation import get_act_fn
20
from vllm.model_executor.layers.layernorm import RMSNorm
21
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -54,7 +58,7 @@ class InternVisionEmbeddings(nn.Module):
58
self.position_embedding = nn.Parameter(
59
torch.randn(1, self.num_positions, self.embed_dim))
60
57
- def _get_pos_embed(self, pos_embed, H, W):
61
+ def _get_pos_embed(self, pos_embed: torch.Tensor, H: int, W: int):
62
target_dtype = pos_embed.dtype
63
pos_embed = pos_embed.float().reshape(
64
1, self.image_size // self.patch_size,
@@ -63,9 +67,21 @@ class InternVisionEmbeddings(nn.Module):
67
size=(H, W),
68
mode='bicubic',
69
align_corners=False)
66
- pos_embed = pos_embed.reshape(1, -1, H * W).permute(0, 2,
67
- 1).to(target_dtype)
68
- return pos_embed
70
+ return pos_embed.reshape(1, -1, H * W).permute(0, 2,
71
+ 1).to(target_dtype)
72
+
73
+ def _get_position_embedding(self, H: int, W: int) -> torch.Tensor:
74
+ position_embedding = self.position_embedding
75
+ if self.num_patches == H * W:
76
+ return position_embedding
77
+
78
+ return torch.cat(
79
+ [
80
+ position_embedding[:, :1, :],
81
+ self._get_pos_embed(position_embedding[:, 1:, :], H, W),
82
+ ],
83
+ dim=1,
84
+ )
85
86
def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
87
target_dtype = self.patch_embedding.weight.dtype
@@ -76,12 +92,7 @@ class InternVisionEmbeddings(nn.Module):
92
class_embeds = self.class_embedding.expand(batch_size, 1,
93
-1).to(target_dtype)
94
embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
79
- position_embedding = torch.cat([
80
- self.position_embedding[:, :1, :],
81
- self._get_pos_embed(self.position_embedding[:, 1:, :], height,
82
- width)
83
- ],
84
- dim=1)
95
+ position_embedding = self._get_position_embedding(height, width)
96
embeddings = embeddings + position_embedding.to(target_dtype)
97
return embeddings
98
@@ -93,8 +104,11 @@ class InternParallelAttention(nn.Module):
104
self,
105
config: PretrainedConfig,
106
quant_config: Optional[QuantizationConfig] = None,
96
- ):
107
+ *,
108
+ num_dummy_heads: int = 0,
109
+ ) -> None:
110
super().__init__()
111
+
112
self.config = config
113
self.embed_dim = config.hidden_size
114
self.num_heads = config.num_attention_heads
@@ -105,11 +119,19 @@ class InternParallelAttention(nn.Module):
119
f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
120
f' {self.num_heads}).')
121
122
+ self.tp_size = get_tensor_model_parallel_world_size()
123
+ self.tp_rank = get_tensor_model_parallel_rank()
124
+
125
+ # Additional dummy heads are used to enable TP for common GPU counts.
126
+ self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim
127
+ self.num_heads_per_partition = divide(num_dummy_heads + self.num_heads,
128
+ self.tp_size)
129
+
130
self.scale = self.head_dim**-0.5
131
self.qkv = QKVParallelLinear(
132
self.embed_dim,
133
self.head_dim,
112
- self.num_heads,
134
+ num_dummy_heads + self.num_heads,
135
bias=config.qkv_bias,
136
quant_config=quant_config,
137
)
@@ -117,34 +139,44 @@ class InternParallelAttention(nn.Module):
139
self.qk_normalization = config.qk_normalization
140
141
if self.qk_normalization:
120
- self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
121
- self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
142
+ self.q_norm = RMSNorm(self.dummy_dim,
143
+ eps=config.layer_norm_eps,
144
+ var_hidden_size=self.embed_dim)
145
+ self.k_norm = RMSNorm(self.dummy_dim,
146
+ eps=config.layer_norm_eps,
147
+ var_hidden_size=self.embed_dim)
148
149
self.proj = RowParallelLinear(
124
- self.embed_dim,
150
+ self.dummy_dim,
151
self.embed_dim,
152
quant_config=quant_config,
153
)
154
129
- self.tp_size = get_tensor_model_parallel_world_size()
130
- self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
131
-
132
- def forward(self, x):
133
- B, N, C = x.shape
155
+ def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
156
+ if self.tp_size > 1:
157
+ q = tensor_model_parallel_all_gather(q.contiguous())
158
+ k = tensor_model_parallel_all_gather(k.contiguous())
159
+ q = self.q_norm.forward_native(q)
160
+ k = self.k_norm.forward_native(k)
161
+ if self.tp_size > 1:
162
+ splitter = partial(split_tensor_along_last_dim,
163
+ num_partitions=self.tp_size)
164
+ q = splitter(q)[self.tp_rank]
165
+ k = splitter(k)[self.tp_rank]
166
+ return q, k
167
+
168
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
169
+ B, N, _ = x.shape
170
qkv, _ = self.qkv(x)
171
q, k, v = qkv.chunk(3, dim=-1)
172
173
+ if self.qk_normalization:
174
+ q, k = self._apply_qk_norm(q, k)
175
+
176
q = q.view(B, N, self.num_heads_per_partition, self.head_dim)
177
k = k.view(B, N, self.num_heads_per_partition, self.head_dim)
178
v = v.view(B, N, self.num_heads_per_partition, self.head_dim)
179
141
- if self.qk_normalization:
142
- B_, N_, H_, D_ = q.shape
143
- q = self.q_norm.forward_native(q.flatten(-2,
144
- -1)).view(B_, N_, H_, D_)
145
- k = self.k_norm.forward_native(k.flatten(-2,
146
- -1)).view(B_, N_, H_, D_)
147
-
180
x = xops.memory_efficient_attention_forward(q, k, v, scale=self.scale)
181
x = x.view(B, N, -1)
182
@@ -155,8 +187,14 @@ class InternParallelAttention(nn.Module):
187
class InternSdpaAttention(nn.Module):
188
"""Multi-headed attention from 'Attention Is All You Need' paper"""
189
158
- def __init__(self, config: PretrainedConfig):
190
+ def __init__(
191
+ self,
192
+ config: PretrainedConfig,
193
+ *,
194
+ num_dummy_heads: int = 0,
195
+ ) -> None:
196
super().__init__()
197
+
198
self.config = config
199
self.embed_dim = config.hidden_size
200
self.num_heads = config.num_attention_heads
@@ -167,20 +205,27 @@ class InternSdpaAttention(nn.Module):
205
f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
206
f' {self.num_heads}).')
207
208
+ # Additional dummy heads are used to enable TP for common GPU counts.
209
+ self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim
210
+
211
self.scale = self.head_dim**-0.5
212
self.qkv = nn.Linear(self.embed_dim,
172
- 3 * self.embed_dim,
213
+ 3 * self.dummy_dim,
214
bias=config.qkv_bias)
215
216
self.qk_normalization = config.qk_normalization
217
218
if self.qk_normalization:
178
- self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
179
- self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
219
+ self.q_norm = RMSNorm(self.dummy_dim,
220
+ eps=config.layer_norm_eps,
221
+ var_hidden_size=self.embed_dim)
222
+ self.k_norm = RMSNorm(self.dummy_dim,
223
+ eps=config.layer_norm_eps,
224
+ var_hidden_size=self.embed_dim)
225
181
- self.proj = nn.Linear(self.embed_dim, self.embed_dim)
226
+ self.proj = nn.Linear(self.dummy_dim, self.embed_dim)
227
183
- def forward(self, x):
228
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
229
B, N, C = x.shape
230
qkv = self.qkv(x)
231
q, k, v = qkv.chunk(3, dim=-1)
@@ -233,22 +278,23 @@ class InternMLP(nn.Module):
278
279
class InternVisionEncoderLayer(nn.Module):
280
236
- def __init__(self,
237
- config: PretrainedConfig,
238
- quant_config: Optional[QuantizationConfig] = None):
281
+ def __init__(
282
+ self,
283
+ config: PretrainedConfig,
284
+ quant_config: Optional[QuantizationConfig] = None,
285
+ *,
286
+ num_dummy_heads: int = 0,
287
+ ) -> None:
288
super().__init__()
289
+
290
self.embed_dim = config.hidden_size
291
self.intermediate_size = config.intermediate_size
292
self.norm_type = config.norm_type
293
244
- # fallback to sdpa attention if tp unavailable
245
- tp_size = get_tensor_model_parallel_world_size()
246
- num_heads = config.num_attention_heads
247
- if USE_XFORMERS_OPS and num_heads % tp_size == 0:
248
- self.attn = InternParallelAttention(config,
249
- quant_config=quant_config)
250
- else:
251
- self.attn = InternSdpaAttention(config)
294
+ self.attn = self._init_attn(config,
295
+ quant_config,
296
+ num_dummy_heads=num_dummy_heads)
297
+
298
self.mlp = InternMLP(config, quant_config=quant_config)
299
self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
300
eps=config.layer_norm_eps)
@@ -260,6 +306,24 @@ class InternVisionEncoderLayer(nn.Module):
306
self.ls2 = nn.Parameter(config.initializer_factor *
307
torch.ones(self.embed_dim))
308
309
+ def _init_attn(
310
+ self,
311
+ config: PretrainedConfig,
312
+ quant_config: Optional[QuantizationConfig],
313
+ *,
314
+ num_dummy_heads: int,
315
+ ):
316
+ # fallback to sdpa attention if tp unavailable
317
+ tp_size = get_tensor_model_parallel_world_size()
318
+ num_heads = config.num_attention_heads
319
+
320
+ if USE_XFORMERS_OPS and (num_heads + num_dummy_heads) % tp_size == 0:
321
+ return InternParallelAttention(config,
322
+ quant_config=quant_config,
323
+ num_dummy_heads=num_dummy_heads)
324
+
325
+ return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads)
326
+
327
def forward(
328
self,
329
hidden_states: torch.Tensor,
@@ -275,19 +339,27 @@ class InternVisionEncoderLayer(nn.Module):
339
340
class InternVisionEncoder(nn.Module):
341
278
- def __init__(self,
279
- config: PretrainedConfig,
280
- quant_config: Optional[QuantizationConfig] = None,
281
- num_hidden_layers_override: Optional[int] = None):
342
+ def __init__(
343
+ self,
344
+ config: PretrainedConfig,
345
+ quant_config: Optional[QuantizationConfig] = None,
346
+ *,
347
+ num_hidden_layers_override: Optional[int] = None,
348
+ num_dummy_heads: int = 0,
349
+ ):
350
super().__init__()
351
+
352
self.config = config
353
354
if num_hidden_layers_override is None:
355
num_hidden_layers = config.num_hidden_layers
356
else:
357
num_hidden_layers = num_hidden_layers_override
358
+
359
self.layers = nn.ModuleList([
290
- InternVisionEncoderLayer(config=config, quant_config=quant_config)
360
+ InternVisionEncoderLayer(config,
361
+ quant_config,
362
+ num_dummy_heads=num_dummy_heads)
363
for _ in range(num_hidden_layers)
364
])
365
@@ -302,35 +374,25 @@ class InternVisionEncoder(nn.Module):
374
375
class InternVisionModel(nn.Module):
376
305
- def __init__(self,
306
- config: PretrainedConfig,
307
- quant_config: Optional[QuantizationConfig] = None,
308
- num_hidden_layers_override: Optional[int] = None):
377
+ def __init__(
378
+ self,
379
+ config: PretrainedConfig,
380
+ quant_config: Optional[QuantizationConfig] = None,
381
+ *,
382
+ num_hidden_layers_override: Optional[int] = None,
383
+ num_dummy_heads: int = 0,
384
+ ):
385
super().__init__()
386
+
387
self.config = config
388
389
self.embeddings = InternVisionEmbeddings(config)
390
self.encoder = InternVisionEncoder(
391
config=config,
392
quant_config=quant_config,
316
- num_hidden_layers_override=num_hidden_layers_override)
317
-
318
- def resize_pos_embeddings(self, old_size, new_size, patch_size):
319
- pos_emb = self.embeddings.position_embedding
320
- _, num_positions, embed_dim = pos_emb.shape
321
- cls_emb = pos_emb[:, :1, :]
322
- pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size,
323
- old_size // patch_size,
324
- -1).permute(0, 3, 1, 2)
325
- pos_emb = F.interpolate(pos_emb.float(),
326
- size=new_size // patch_size,
327
- mode='bicubic',
328
- align_corners=False)
329
- pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim,
330
- -1).permute(0, 2, 1)
331
- pos_emb = torch.cat([cls_emb, pos_emb], dim=1)
332
- self.embeddings.position_embedding = nn.Parameter(pos_emb)
333
- self.embeddings.image_size = new_size
393
+ num_hidden_layers_override=num_hidden_layers_override,
394
+ num_dummy_heads=num_dummy_heads,
395
+ )
396
397
def get_input_embeddings(self):
398
return self.embeddings
vllm/model_executor/models/internvl.py
+173
-121
@@ -237,130 +237,173 @@ def get_max_internvl_image_size(ctx: InputContext,
237
return width, height
238
239
240
-def input_processor_for_internvl(ctx: InputContext,
241
- llm_inputs: LLMInputs,
242
- *,
243
- max_dynamic_patch: Optional[int] = None):
244
- multi_modal_data = llm_inputs.get("multi_modal_data")
245
- if multi_modal_data is None or "image" not in multi_modal_data:
246
- return llm_inputs
240
+class InternVLInputPipeline:
241
248
- model_config = ctx.model_config
249
- hf_config = ctx.get_hf_config()
242
+ def __init__(
243
+ self,
244
+ img_start_token: str,
245
+ img_end_token: str,
246
+ img_context_token: str,
247
+ ) -> None:
248
+ super().__init__()
249
251
- image_data = multi_modal_data["image"]
252
- num_patches = get_internvl_num_patches(hf_config)
253
- num_blocks_calculator = calculate_num_blocks_wrapper(
254
- hf_config, max_dynamic_patch)
255
- if isinstance(image_data, Image.Image):
256
- width, height = image_data.size
257
- num_blocks, _, _ = num_blocks_calculator(width, height)
258
- image_feature_size = [num_blocks * num_patches]
259
- elif is_list_of(image_data, Image.Image):
260
- image_feature_size = []
261
- for image in image_data:
262
- width, height = image.size
250
+ self.img_start_token = img_start_token
251
+ self.img_end_token = img_end_token
252
+ self.img_context_token = img_context_token
253
+
254
+ def _create_image_prompt(self, feature_size: int, num_patches: int) -> str:
255
+ return (self.img_start_token + self.img_context_token * feature_size +
256
+ self.img_end_token)
257
+
258
+ def _expand_image_prompt(
259
+ self,
260
+ prompt: str,
261
+ feature_sizes: List[int],
262
+ num_patches: int,
263
+ ) -> str:
264
+ image_idx = sorted(
265
+ map(int, re.findall(r"Image-(\d+): <image>\n", prompt)))
266
+
267
+ new_prompt = prompt
268
+ for idx, feature_size in enumerate(feature_sizes, start=1):
269
+ image_prompt = self._create_image_prompt(feature_size, num_patches)
270
+ if not image_idx:
271
+ image_prompt = f"Image-{idx}: {image_prompt}"
272
+
273
+ new_prompt = new_prompt.replace('<image>', image_prompt, 1)
274
+
275
+ return new_prompt
276
+
277
+ def input_processor(
278
+ self,
279
+ ctx: InputContext,
280
+ llm_inputs: LLMInputs,
281
+ *,
282
+ max_dynamic_patch: Optional[int] = None,
283
+ ) -> LLMInputs:
284
+ multi_modal_data = llm_inputs.get("multi_modal_data")
285
+ if multi_modal_data is None or "image" not in multi_modal_data:
286
+ return llm_inputs
287
+
288
+ model_config = ctx.model_config
289
+ hf_config = ctx.get_hf_config()
290
+
291
+ image_data = multi_modal_data["image"]
292
+ num_patches = get_internvl_num_patches(hf_config)
293
+ num_blocks_calculator = calculate_num_blocks_wrapper(
294
+ hf_config, max_dynamic_patch)
295
+ if isinstance(image_data, Image.Image):
296
+ width, height = image_data.size
297
num_blocks, _, _ = num_blocks_calculator(width, height)
264
- image_feature_size.append(num_blocks * num_patches)
265
- elif isinstance(image_data, torch.Tensor):
266
- num_images, image_feature_size, hidden_size = image_data.shape
267
- else:
268
- raise TypeError(f"Invalid image type: {type(image_data)}")
269
-
270
- tokenizer = cached_get_tokenizer(
271
- model_config.tokenizer,
272
- trust_remote_code=model_config.trust_remote_code)
273
-
274
- prompt = llm_inputs.get("prompt")
275
- prompt_token_ids = llm_inputs["prompt_token_ids"]
276
- if prompt is None:
277
- prompt = tokenizer.decode(prompt_token_ids)
278
-
279
- new_prompt = prompt
280
- image_idx = sorted(map(int, re.findall(r"Image-(\d+): <image>\n", prompt)))
281
- for idx, feature_size in enumerate(image_feature_size, start=1):
282
- image_prompt = IMG_START + IMG_CONTEXT * feature_size + IMG_END
283
- if not image_idx:
284
- image_prompt = f"Image-{idx}: {image_prompt}"
285
- new_prompt = new_prompt.replace('<image>', image_prompt, 1)
286
- new_prompt_token_ids = tokenizer.encode(new_prompt)
287
-
288
- return LLMInputs(prompt=prompt,
289
- prompt_token_ids=new_prompt_token_ids,
290
- multi_modal_data=multi_modal_data)
291
-
292
-
293
-def input_mapper_for_internvl(ctx: InputContext,
294
- data: object,
295
- *,
296
- max_dynamic_patch: Optional[int] = None):
297
- hf_config = ctx.get_hf_config()
298
+ image_feature_sizes = [num_blocks * num_patches]
299
+ elif is_list_of(image_data, Image.Image):
300
+ image_feature_sizes = []
301
+ for image in image_data:
302
+ width, height = image.size
303
+ num_blocks, _, _ = num_blocks_calculator(width, height)
304
+ image_feature_sizes.append(num_blocks * num_patches)
305
+ elif isinstance(image_data, torch.Tensor):
306
+ num_images, image_feature_size, hidden_size = image_data.shape
307
+ image_feature_sizes = [image_feature_size]
308
+ else:
309
+ raise TypeError(f"Invalid image type: {type(image_data)}")
310
299
- image_pixel_values_mapper = image_to_pixel_values_wrapper(
300
- hf_config, max_dynamic_patch)
301
- if isinstance(data, Image.Image):
302
- data = image_pixel_values_mapper(data)
303
- # Add an N dimension for number of images per prompt (currently 1).
304
- data = data.unsqueeze(0)
305
- elif is_list_of(data, Image.Image):
306
- # we can't stack here because the images may have different num_patches
307
- data = [image_pixel_values_mapper(img) for img in data]
308
- model_config = ctx.model_config
309
- tokenizer = cached_get_tokenizer(
310
- model_config.tokenizer,
311
- trust_remote_code=model_config.trust_remote_code)
312
- image_token_id = tokenizer.encode(IMG_CONTEXT,
313
- add_special_tokens=False,
314
- return_tensors="pt")[0]
315
-
316
- return MultiModalInputs({
317
- "pixel_values": data,
318
- "image_token_id": image_token_id
319
- })
320
-
321
-
322
-def dummy_data_for_internvl(ctx: InputContext,
323
- seq_len: int,
324
- mm_counts: Mapping[str, int],
325
- *,
326
- max_dynamic_patch: Optional[int] = None):
327
- num_images = mm_counts["image"]
311
+ tokenizer = cached_get_tokenizer(
312
+ model_config.tokenizer,
313
+ trust_remote_code=model_config.trust_remote_code)
314
329
- hf_config = ctx.get_hf_config()
315
+ prompt = llm_inputs.get("prompt")
316
+ prompt_token_ids = llm_inputs["prompt_token_ids"]
317
+ if prompt is None:
318
+ prompt = tokenizer.decode(prompt_token_ids)
319
+
320
+ new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
321
+ num_patches)
322
+ new_prompt_token_ids = tokenizer.encode(new_prompt)
323
+
324
+ return LLMInputs(prompt=prompt,
325
+ prompt_token_ids=new_prompt_token_ids,
326
+ multi_modal_data=multi_modal_data)
327
331
- image_feature_size = get_max_internvl_image_tokens(
332
- ctx, max_dynamic_patch=max_dynamic_patch)
333
- model_config = ctx.model_config
334
- tokenizer = cached_get_tokenizer(
335
- model_config.tokenizer,
336
- trust_remote_code=model_config.trust_remote_code)
328
+ def input_mapper(
329
+ self,
330
+ ctx: InputContext,
331
+ data: object,
332
+ *,
333
+ max_dynamic_patch: Optional[int] = None,
334
+ ):
335
+ hf_config = ctx.get_hf_config()
336
+
337
+ image_pixel_values_mapper = image_to_pixel_values_wrapper(
338
+ hf_config, max_dynamic_patch)
339
+ if isinstance(data, Image.Image):
340
+ data = image_pixel_values_mapper(data)
341
+ # Add an N dimension for number of images per prompt (currently 1).
342
+ data = data.unsqueeze(0)
343
+ elif is_list_of(data, Image.Image):
344
+ # we can't stack here because images may have different num_patches
345
+ data = [image_pixel_values_mapper(img) for img in data]
346
+ model_config = ctx.model_config
347
+ tokenizer = cached_get_tokenizer(
348
+ model_config.tokenizer,
349
+ trust_remote_code=model_config.trust_remote_code)
350
+ image_token_id = tokenizer.encode(self.img_context_token,
351
+ add_special_tokens=False,
352
+ return_tensors="pt")[0]
353
+
354
+ return MultiModalInputs({
355
+ "pixel_values": data,
356
+ "image_token_id": image_token_id
357
+ })
358
+
359
+ def dummy_data(
360
+ self,
361
+ ctx: InputContext,
362
+ seq_len: int,
363
+ mm_counts: Mapping[str, int],
364
+ *,
365
+ max_dynamic_patch: Optional[int] = None,
366
+ ):
367
+ num_images = mm_counts["image"]
368
+
369
+ hf_config = ctx.get_hf_config()
370
+
371
+ image_feature_size = get_max_internvl_image_tokens(
372
+ ctx, max_dynamic_patch=max_dynamic_patch)
373
+ model_config = ctx.model_config
374
+ tokenizer = cached_get_tokenizer(
375
+ model_config.tokenizer,
376
+ trust_remote_code=model_config.trust_remote_code)
377
+
378
+ seq_data = dummy_seq_data_for_clip(
379
+ hf_config.vision_config,
380
+ seq_len,
381
+ num_images,
382
+ image_token_id=tokenizer.encode(self.img_context_token,
383
+ add_special_tokens=False)[0],
384
+ image_feature_size_override=image_feature_size,
385
+ )
386
338
- seq_data = dummy_seq_data_for_clip(
339
- hf_config.vision_config,
340
- seq_len,
341
- num_images,
342
- image_token_id=tokenizer.encode(IMG_CONTEXT,
343
- add_special_tokens=False)[0],
344
- image_feature_size_override=image_feature_size,
345
- )
387
+ max_image_width, max_image_height = get_max_internvl_image_size(
388
+ ctx, max_dynamic_patch=max_dynamic_patch)
389
347
- max_image_width, max_image_height = get_max_internvl_image_size(
348
- ctx, max_dynamic_patch=max_dynamic_patch)
390
+ mm_data = dummy_image_for_clip(
391
+ hf_config.vision_config,
392
+ num_images,
393
+ image_width_override=max_image_width,
394
+ image_height_override=max_image_height,
395
+ )
396
350
- mm_data = dummy_image_for_clip(
351
- hf_config.vision_config,
352
- num_images,
353
- image_width_override=max_image_width,
354
- image_height_override=max_image_height,
355
- )
397
+ return seq_data, mm_data
398
357
- return seq_data, mm_data
399
400
+input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
401
360
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_internvl)
402
+
403
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
404
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
362
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_internvl)
363
-@INPUT_REGISTRY.register_input_processor(input_processor_for_internvl)
405
+@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
406
+@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
407
class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
408
409
def __init__(self,
@@ -388,20 +431,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
431
+ vision_feature_layer + 1
432
else:
433
num_hidden_layers = vision_feature_layer + 1
391
- self.vision_model = InternVisionModel(
392
- config.vision_config, num_hidden_layers_override=num_hidden_layers)
434
+ self.vision_model = self._init_vision_model(config, num_hidden_layers)
435
436
self.language_model = init_vllm_registered_model(
437
config.text_config, cache_config, quant_config)
438
397
- vit_hidden_size = config.vision_config.hidden_size
398
- llm_hidden_size = config.text_config.hidden_size
399
-
400
- self.mlp1 = nn.Sequential(
401
- nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
402
- nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
403
- llm_hidden_size), nn.GELU(),
404
- nn.Linear(llm_hidden_size, llm_hidden_size))
439
+ self.mlp1 = self._init_mlp1(config)
440
441
self.img_context_token_id = None
442
self.make_empty_intermediate_tensors = (
@@ -414,6 +449,23 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
449
450
return Sampler()
451
452
+ def _init_vision_model(self, config: PretrainedConfig,
453
+ num_hidden_layers: int):
454
+ return InternVisionModel(config.vision_config,
455
+ num_hidden_layers_override=num_hidden_layers)
456
+
457
+ def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
458
+ vit_hidden_size = config.vision_config.hidden_size
459
+ llm_hidden_size = config.text_config.hidden_size
460
+
461
+ return nn.Sequential(
462
+ nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
463
+ nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
464
+ llm_hidden_size),
465
+ nn.GELU(),
466
+ nn.Linear(llm_hidden_size, llm_hidden_size),
467
+ )
468
+
469
def pixel_shuffle(self, x, scale_factor=0.5):
470
n, w, h, c = x.size()
471
# N, W, H, C --> N, W, H * scale, C // scale
vllm/model_executor/models/nvlm_d.py
new
+64
@@ -0,0 +1,64 @@
1
+# adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py
2
+# --------------------------------------------------------
3
+# NVLM-D
4
+# Copyright (c) 2024 NVIDIA
5
+# Licensed under Apache 2.0 License [see LICENSE for details]
6
+# --------------------------------------------------------
7
+import torch.nn as nn
8
+from transformers import PretrainedConfig
9
+
10
+from vllm.inputs import INPUT_REGISTRY
11
+from vllm.multimodal import MULTIMODAL_REGISTRY
12
+
13
+from .intern_vit import InternVisionModel
14
+from .internvl import (InternVLChatModel, InternVLInputPipeline,
15
+ get_max_internvl_image_tokens)
16
+
17
+IMG_START = '<|vision_start|>'
18
+IMG_END = '<|vision_end|>'
19
+IMG_CONTEXT = '<|vision_pad|>'
20
+
21
+
22
+class NVLMInputPipeline(InternVLInputPipeline):
23
+
24
+ def _create_image_prompt(self, feature_size: int, num_patches: int) -> str:
25
+ tile_pos_identifiers = ([f"<tile_{i}>"
26
+ for i in range(1, num_patches)] +
27
+ ["<tile_global_thumbnail>"])
28
+ context_size = feature_size // num_patches
29
+
30
+ return '<Image>' + ''.join(
31
+ tile_pos_identifier + self.img_context_token * context_size
32
+ for tile_pos_identifier in tile_pos_identifiers) + '</Image>'
33
+
34
+
35
+input_pipeline = NVLMInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
36
+
37
+
38
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
39
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
40
+@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
41
+@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
42
+class NVLM_D_Model(InternVLChatModel):
43
+
44
+ def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
45
+ vit_hidden_size = config.vision_config.hidden_size
46
+ llm_intermediate_size = config.text_config.intermediate_size
47
+ llm_hidden_size = config.text_config.hidden_size
48
+
49
+ return nn.Sequential(
50
+ nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
51
+ nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio)**2,
52
+ llm_intermediate_size,
53
+ bias=False),
54
+ nn.GELU(),
55
+ nn.Linear(llm_intermediate_size, llm_hidden_size, bias=False),
56
+ )
57
+
58
+ def _init_vision_model(self, config: PretrainedConfig,
59
+ num_hidden_layers: int):
60
+ # We added additional dummy heads to the original num of heads to make
61
+ # the number of heads divisible by 8.
62
+ return InternVisionModel(config.vision_config,
63
+ num_hidden_layers_override=num_hidden_layers,
64
+ num_dummy_heads=7)
vllm/model_executor/models/registry.py
+15
-22
@@ -16,6 +16,7 @@ from .interfaces_base import is_embedding_model, is_text_generation_model
16
17
logger = init_logger(__name__)
18
19
+# yapf: disable
20
_TEXT_GENERATION_MODELS = {
21
# [Decoder-only]
22
"AquilaModel": ("llama", "LlamaForCausalLM"),
@@ -68,8 +69,6 @@ _TEXT_GENERATION_MODELS = {
69
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
70
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
71
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
71
- "Qwen2VLForConditionalGeneration":
72
- ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
72
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
73
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
74
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
@@ -88,32 +87,25 @@ _EMBEDDING_MODELS = {
87
}
88
89
_MULTIMODAL_MODELS = {
91
- "Blip2ForConditionalGeneration":
92
- ("blip2", "Blip2ForConditionalGeneration"),
93
- "ChameleonForConditionalGeneration":
94
- ("chameleon", "ChameleonForConditionalGeneration"),
90
+ # [Decoder-only]
91
+ "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
92
+ "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
93
"FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
94
"InternVLChatModel": ("internvl", "InternVLChatModel"),
97
- "LlavaForConditionalGeneration": ("llava",
98
- "LlavaForConditionalGeneration"),
99
- "LlavaNextForConditionalGeneration": ("llava_next",
100
- "LlavaNextForConditionalGeneration"),
101
- "LlavaNextVideoForConditionalGeneration":
102
- ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
103
- "LlavaOnevisionForConditionalGeneration":
104
- ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
95
+ "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
96
+ "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
97
+ "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
98
+ "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
99
"MiniCPMV": ("minicpmv", "MiniCPMV"),
106
- "PaliGemmaForConditionalGeneration": ("paligemma",
107
- "PaliGemmaForConditionalGeneration"),
100
+ "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
101
+ "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
102
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
109
- "PixtralForConditionalGeneration": ("pixtral",
110
- "PixtralForConditionalGeneration"),
103
+ "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
104
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
112
- "Qwen2VLForConditionalGeneration": ("qwen2_vl",
113
- "Qwen2VLForConditionalGeneration"),
105
+ "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
106
"UltravoxModel": ("ultravox", "UltravoxModel"),
115
- "MllamaForConditionalGeneration": ("mllama",
116
- "MllamaForConditionalGeneration"),
107
+ # [Encoder-decoder]
108
+ "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
109
}
110
111
_SPECULATIVE_DECODING_MODELS = {
@@ -121,6 +113,7 @@ _SPECULATIVE_DECODING_MODELS = {
113
"MedusaModel": ("medusa", "Medusa"),
114
"MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
115
}
116
+# yapf: enable
117
118
_MODELS = {
119
**_TEXT_GENERATION_MODELS,
vllm/transformers_utils/config.py
+4
-3
@@ -22,9 +22,9 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
22
InternVLChatConfig, JAISConfig,
23
MedusaConfig, MllamaConfig,
24
MLPSpeculatorConfig, MPTConfig,
25
- NemotronConfig, Qwen2VLConfig,
26
- RWConfig, SolarConfig,
27
- UltravoxConfig)
25
+ NemotronConfig, NVLM_D_Config,
26
+ Qwen2VLConfig, RWConfig,
27
+ SolarConfig, UltravoxConfig)
28
# yapf: enable
29
from vllm.transformers_utils.utils import check_gguf_file
30
@@ -54,6 +54,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
54
"exaone": ExaoneConfig,
55
"internvl_chat": InternVLChatConfig,
56
"nemotron": NemotronConfig,
57
+ "NVLM_D": NVLM_D_Config,
58
"solar": SolarConfig,
59
"ultravox": UltravoxConfig,
60
"qwen2_vl": Qwen2VLConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.transformers_utils.configs.mllama import MllamaConfig
13
from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
14
from vllm.transformers_utils.configs.mpt import MPTConfig
15
from vllm.transformers_utils.configs.nemotron import NemotronConfig
16
+from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config
17
from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig,
18
Qwen2VLVisionConfig)
19
from vllm.transformers_utils.configs.solar import SolarConfig
@@ -31,6 +32,7 @@ __all__ = [
32
"MllamaConfig",
33
"MLPSpeculatorConfig",
34
"NemotronConfig",
35
+ "NVLM_D_Config",
36
"SolarConfig",
37
"UltravoxConfig",
38
"Qwen2VLConfig",
vllm/transformers_utils/configs/nvlm_d.py
new
+12
@@ -0,0 +1,12 @@
1
+# Adapted from
2
+# https://huggingface.co/nvidia/NVLM-D-72B/blob/main/configuration_nvlm_d.py
3
+# --------------------------------------------------------
4
+# NVLM-D
5
+# Copyright (c) 2024 NVIDIA
6
+# Licensed under Apache 2.0 License [see LICENSE for details]
7
+# --------------------------------------------------------
8
+from .internvl import InternVLChatConfig
9
+
10
+
11
+class NVLM_D_Config(InternVLChatConfig):
12
+ model_type = 'NVLM_D'