A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 01:30 UTC
[Model] Add Step3vl 10b (#32329)
Signed-off-by: luotingdan <luotingdan@stepfun.com> Signed-off-by: ltd0924 <32387785+ltd0924@users.noreply.github.com> Co-authored-by: luotingdan <luotingdan@stepfun.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
ltd0924 committed
Jan 16, 2026 at 11:04 UTC
709502558c3efa658b2b49c7019f9dc67929764d
4 files changed
+554
docs/models/supported_models.md
+1
@@ -720,6 +720,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
720
| `SkyworkR1VChatModel` | Skywork-R1V-38B | T + I | `Skywork/Skywork-R1V-38B` | | ✅︎ |
721
| `SmolVLMForConditionalGeneration` | SmolVLM2 | T + I | `SmolVLM2-2.2B-Instruct` | ✅︎ | |
722
| `Step3VLForConditionalGeneration` | Step3-VL | T + I<sup>+</sup> | `stepfun-ai/step3` | | ✅︎ |
723
+| `StepVLForConditionalGeneration` | Step3-VL-10B | T + I<sup>+</sup> | `stepfun-ai/Step3-VL-10B` | | ✅︎ |
724
| `TarsierForConditionalGeneration` | Tarsier | T + I<sup>E+</sup> | `omni-search/Tarsier-7b`, `omni-search/Tarsier-34b` | | ✅︎ |
725
| `Tarsier2ForConditionalGeneration`<sup>^</sup> | Tarsier2 | T + I<sup>E+</sup> + V<sup>E+</sup> | `omni-research/Tarsier2-Recap-7b`, `omni-research/Tarsier2-7b-0115` | | ✅︎ |
726
| `UltravoxModel` | Ultravox | T + A<sup>E+</sup> | `fixie-ai/ultravox-v0_5-llama-3_2-1b` | ✅︎ | ✅︎ |
tests/models/registry.py
+3
@@ -925,6 +925,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
925
"Step3VLForConditionalGeneration": _HfExamplesInfo(
926
"stepfun-ai/step3", trust_remote_code=True
927
),
928
+ "StepVLForConditionalGeneration": _HfExamplesInfo(
929
+ "stepfun-ai/Step3-VL-10B", trust_remote_code=True
930
+ ),
931
"UltravoxModel": _HfExamplesInfo(
932
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
933
trust_remote_code=True,
vllm/model_executor/models/registry.py
+1
@@ -323,6 +323,7 @@ _MULTIMODAL_MODELS = {
323
"hunyuan_vision",
324
"HunYuanVLForConditionalGeneration",
325
),
326
+ "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),
327
"InternVLChatModel": ("internvl", "InternVLChatModel"),
328
"NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
329
"OpenCUAForConditionalGeneration": (
vllm/model_executor/models/step_vl.py
new
+549
@@ -0,0 +1,549 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""This is basically a copy from perception_models/core/vision_encoder/pe.py"""
4
+
5
+from collections.abc import Callable
6
+from functools import partial
7
+
8
+import torch
9
+from einops import rearrange, repeat
10
+from torch import nn
11
+from torch.nn import functional as F
12
+
13
+from vllm.config import VllmConfig
14
+from vllm.distributed import get_tensor_model_parallel_world_size
15
+from vllm.model_executor.layers.activation import get_act_fn
16
+from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
17
+from vllm.model_executor.layers.conv import Conv2dLayer
18
+from vllm.model_executor.layers.linear import (
19
+ ColumnParallelLinear,
20
+ QKVParallelLinear,
21
+ RowParallelLinear,
22
+)
23
+from vllm.model_executor.layers.quantization import QuantizationConfig
24
+
25
+from .step3_vl import Step3VLForConditionalGeneration
26
+from .utils import WeightsMapper, init_vllm_registered_model, maybe_prefix
27
+from .vision import run_dp_sharded_vision_model
28
+
29
+_DEFAULT_NORM_LAYER = partial(nn.LayerNorm, eps=1e-5)
30
+
31
+
32
+def rotate_half(x):
33
+ x = rearrange(x, "... (d r) -> ... d r", r=2)
34
+ x1, x2 = x.unbind(dim=-1)
35
+ x = torch.stack((-x2, x1), dim=-1)
36
+ return rearrange(x, "... d r -> ... (d r)")
37
+
38
+
39
+def apply_rotary_emb(freqs, t, start_index=0, scale=1.0, seq_dim=-2):
40
+ dtype = t.dtype
41
+
42
+ if t.ndim == 3:
43
+ seq_len = t.shape[seq_dim]
44
+ freqs = freqs[-seq_len:]
45
+
46
+ rot_dim = freqs.shape[-1]
47
+ end_index = start_index + rot_dim
48
+
49
+ assert rot_dim <= t.shape[-1], (
50
+ "feature dimension {} is not of sufficient size to rotate in all the "
51
+ "positions {}".format(t.shape[-1], rot_dim)
52
+ )
53
+
54
+ t_left, t, t_right = (
55
+ t[..., :start_index],
56
+ t[..., start_index:end_index],
57
+ t[..., end_index:],
58
+ )
59
+ t = (t * freqs.cos() * scale) + (rotate_half(t) * freqs.sin() * scale)
60
+ out = torch.cat((t_left, t, t_right), dim=-1)
61
+
62
+ return out.type(dtype)
63
+
64
+
65
+class PerceptionEncoderRope2D(nn.Module):
66
+ def __init__(
67
+ self,
68
+ dim: int,
69
+ max_grid_height: int,
70
+ max_grid_width: int,
71
+ use_cls_token: bool = False,
72
+ theta=10000,
73
+ max_freq=10,
74
+ num_freqs=1,
75
+ theta_rescale_factor=1.0,
76
+ ):
77
+ super().__init__()
78
+ self.dim = dim
79
+ self.max_grid_height = max_grid_height
80
+ self.max_grid_width = max_grid_width
81
+ self.use_cls_token = use_cls_token
82
+ self.theta = theta * theta_rescale_factor ** (dim / (dim - 2))
83
+ self.max_freq = max_freq
84
+ self.num_freqs = num_freqs
85
+ cache = self._compute_2d_freqs()
86
+ self.register_buffer("freqs_cache", cache, persistent=False)
87
+
88
+ def _compute_inv_freq(self, base: int | float, dim: int) -> torch.Tensor:
89
+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))
90
+ return freqs
91
+
92
+ def _compute_freqs(self, t: torch.Tensor, inv_freq: torch.Tensor):
93
+ freqs = torch.einsum("..., f -> ... f", t.type(inv_freq.dtype), inv_freq)
94
+ freqs = repeat(freqs, "... n -> ... (n r)", r=2)
95
+ return freqs
96
+
97
+ def _compute_2d_freqs(self) -> torch.Tensor:
98
+ grid_h_range = torch.arange(self.max_grid_height, dtype=torch.float)
99
+ grid_w_range = torch.arange(self.max_grid_width, dtype=torch.float)
100
+ if self.use_cls_token:
101
+ grid_h_range += 1
102
+ grid_w_range += 1
103
+ inv_freq = self._compute_inv_freq(self.theta, self.dim // 2)
104
+ freqs_h = self._compute_freqs(grid_h_range, inv_freq)[:, None].expand(
105
+ self.max_grid_height, self.max_grid_width, -1
106
+ )
107
+ freqs_w = self._compute_freqs(grid_w_range, inv_freq)[None, :].expand(
108
+ self.max_grid_height, self.max_grid_width, -1
109
+ )
110
+ freqs = torch.cat([freqs_w, freqs_h], dim=-1).reshape(
111
+ self.max_grid_height * self.max_grid_width, -1
112
+ )
113
+ if self.use_cls_token:
114
+ freqs = torch.cat([torch.zeros(1, freqs.shape[-1]), freqs], dim=0)
115
+ freqs = freqs[None, None, ...]
116
+ return freqs
117
+
118
+ def forward(self, q: torch.Tensor, k: torch.Tensor, grid_hw: tuple[int, int]):
119
+ if grid_hw[0] != self.max_grid_height or grid_hw[1] != self.max_grid_width:
120
+ rows = torch.arange(grid_hw[0], device=q.device).view(-1, 1)
121
+ cols = torch.arange(grid_hw[1], device=q.device).view(1, -1)
122
+ positions = (rows * self.max_grid_width + cols).reshape(-1).to(torch.long)
123
+ if self.use_cls_token:
124
+ positions = torch.cat(
125
+ [torch.zeros(1, device=q.device), positions + 1], dim=0
126
+ )
127
+ positions = positions.to(torch.long)
128
+ freqs = self.freqs_cache.index_select(2, positions)
129
+ else:
130
+ freqs = self.freqs_cache
131
+ q = apply_rotary_emb(freqs, q)
132
+ k = apply_rotary_emb(freqs, k)
133
+ return q, k
134
+
135
+
136
+class PerceptionEncoderLayerScale(nn.Module):
137
+ def __init__(self, dim, init_values=1e-5, inplace=False):
138
+ super().__init__()
139
+ self.inplace = inplace
140
+ self.gamma = nn.Parameter(init_values * torch.ones(dim))
141
+
142
+ def forward(self, x):
143
+ return x.mul_(self.gamma) if self.inplace else x * self.gamma
144
+
145
+
146
+class PerceptionEncoderMLP(nn.Module):
147
+ def __init__(
148
+ self,
149
+ input_dim: int,
150
+ hidden_dim: int,
151
+ act_layer: Callable[[], nn.Module],
152
+ quant_config: QuantizationConfig | None = None,
153
+ prefix: str = "",
154
+ use_data_parallel: bool = False,
155
+ ):
156
+ super().__init__()
157
+ self.fc1 = ColumnParallelLinear(
158
+ input_dim,
159
+ hidden_dim,
160
+ bias=True,
161
+ quant_config=quant_config,
162
+ prefix=f"{prefix}.fc1",
163
+ disable_tp=use_data_parallel,
164
+ )
165
+ self.activation = act_layer
166
+ self.fc2 = RowParallelLinear(
167
+ hidden_dim,
168
+ input_dim,
169
+ bias=True,
170
+ quant_config=quant_config,
171
+ prefix=f"{prefix}.fc2",
172
+ disable_tp=use_data_parallel,
173
+ )
174
+
175
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
176
+ x, _ = self.fc1(x)
177
+ x = self.activation(x)
178
+ x, _ = self.fc2(x)
179
+ return x
180
+
181
+
182
+class PerceptionEncoderVisionAttention(nn.Module):
183
+ def __init__(
184
+ self,
185
+ embed_dim: int,
186
+ num_heads: int,
187
+ max_grid_height: int,
188
+ max_grid_width: int,
189
+ use_cls_token: bool = False,
190
+ quant_config: QuantizationConfig | None = None,
191
+ prefix: str = "",
192
+ use_data_parallel: bool = False,
193
+ ):
194
+ super().__init__()
195
+ self.embed_dim = embed_dim
196
+ self.total_num_heads = num_heads
197
+ self.head_dim = embed_dim // num_heads
198
+ self.scale = self.head_dim**-0.5
199
+
200
+ tp_size = 1 if use_data_parallel else get_tensor_model_parallel_world_size()
201
+ assert self.total_num_heads % tp_size == 0, (
202
+ "embed_dim must be divisible by num_heads"
203
+ )
204
+ self.num_heads = self.total_num_heads // tp_size
205
+
206
+ self.qkv_proj = QKVParallelLinear(
207
+ embed_dim,
208
+ self.head_dim,
209
+ self.total_num_heads,
210
+ bias=True,
211
+ quant_config=quant_config,
212
+ prefix=f"{prefix}.qkv_proj",
213
+ disable_tp=use_data_parallel,
214
+ )
215
+ self.out_proj = RowParallelLinear(
216
+ embed_dim,
217
+ embed_dim,
218
+ bias=True,
219
+ quant_config=quant_config,
220
+ prefix=f"{prefix}.out_proj",
221
+ disable_tp=use_data_parallel,
222
+ )
223
+ self.attn = MMEncoderAttention(self.num_heads, self.head_dim, self.scale)
224
+ self.rope = PerceptionEncoderRope2D(
225
+ dim=self.head_dim,
226
+ max_grid_height=max_grid_height,
227
+ max_grid_width=max_grid_width,
228
+ use_cls_token=use_cls_token,
229
+ )
230
+
231
+ def forward(self, x: torch.Tensor, grid_hw: tuple[int, int]) -> torch.Tensor:
232
+ bsz, seq_len, _ = x.shape
233
+ qkv, _ = self.qkv_proj(x)
234
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
235
+
236
+ q = q.view(bsz, seq_len, self.num_heads, self.head_dim).permute(0, 2, 1, 3)
237
+ k = k.view(bsz, seq_len, self.num_heads, self.head_dim).permute(0, 2, 1, 3)
238
+ q, k = self.rope(q, k, grid_hw=grid_hw)
239
+ q = q.permute(0, 2, 1, 3).reshape(bsz, seq_len, self.num_heads * self.head_dim)
240
+ k = k.permute(0, 2, 1, 3).reshape(bsz, seq_len, self.num_heads * self.head_dim)
241
+
242
+ attn_output = self.attn(q, k, v)
243
+ attn_output, _ = self.out_proj(attn_output)
244
+ return attn_output
245
+
246
+
247
+class PerceptionEncoderVisionBlock(nn.Module):
248
+ def __init__(
249
+ self,
250
+ d_model: int,
251
+ n_head: int,
252
+ max_grid_height: int,
253
+ max_grid_width: int,
254
+ mlp_ratio: float = 4.0,
255
+ ls_init_value: float = None,
256
+ act_layer: Callable = nn.GELU,
257
+ norm_layer: Callable = nn.LayerNorm,
258
+ use_cls_token: bool = False,
259
+ quant_config: QuantizationConfig | None = None,
260
+ prefix: str = "",
261
+ use_data_parallel: bool = False,
262
+ ):
263
+ super().__init__()
264
+ self.attn = PerceptionEncoderVisionAttention(
265
+ d_model,
266
+ n_head,
267
+ max_grid_height=max_grid_height,
268
+ max_grid_width=max_grid_width,
269
+ use_cls_token=use_cls_token,
270
+ quant_config=quant_config,
271
+ prefix=f"{prefix}.attn",
272
+ use_data_parallel=use_data_parallel,
273
+ )
274
+ self.ls_1 = (
275
+ PerceptionEncoderLayerScale(d_model, ls_init_value)
276
+ if ls_init_value is not None
277
+ else nn.Identity()
278
+ )
279
+ self.ls_2 = (
280
+ PerceptionEncoderLayerScale(d_model, ls_init_value)
281
+ if ls_init_value is not None
282
+ else nn.Identity()
283
+ )
284
+ self.ln_1 = norm_layer(d_model)
285
+ self.ln_2 = norm_layer(d_model)
286
+ hidden_dim = int(d_model * mlp_ratio)
287
+ self.mlp = PerceptionEncoderMLP(
288
+ d_model,
289
+ hidden_dim,
290
+ act_layer,
291
+ quant_config=quant_config,
292
+ prefix=f"{prefix}.mlp",
293
+ use_data_parallel=use_data_parallel,
294
+ )
295
+
296
+ def forward(self, x: torch.Tensor, grid_hw: tuple[int, int]):
297
+ x = x + self.ls_1(self.attn(self.ln_1(x), grid_hw=grid_hw))
298
+ x = x + self.ls_2(self.mlp(self.ln_2(x)))
299
+ return x
300
+
301
+
302
+class PerceptionEncoderVisionTransformer(nn.Module):
303
+ def __init__(
304
+ self,
305
+ width: int,
306
+ layers: int,
307
+ heads: int,
308
+ max_grid_height: int,
309
+ max_grid_width: int,
310
+ mlp_ratio: float = 4.0,
311
+ ls_init_value: float = None,
312
+ act_layer: Callable = nn.GELU,
313
+ norm_layer: Callable = nn.LayerNorm,
314
+ use_cls_token: bool = False,
315
+ quant_config: QuantizationConfig | None = None,
316
+ prefix: str = "",
317
+ use_data_parallel: bool = False,
318
+ ):
319
+ super().__init__()
320
+ self.width = width
321
+ self.layers = layers
322
+ self.resblocks = nn.ModuleList(
323
+ [
324
+ PerceptionEncoderVisionBlock(
325
+ d_model=width,
326
+ n_head=heads,
327
+ max_grid_height=max_grid_height,
328
+ max_grid_width=max_grid_width,
329
+ mlp_ratio=mlp_ratio,
330
+ ls_init_value=ls_init_value,
331
+ act_layer=act_layer,
332
+ norm_layer=norm_layer,
333
+ use_cls_token=use_cls_token,
334
+ quant_config=quant_config,
335
+ prefix=f"{prefix}.resblocks.{i}",
336
+ use_data_parallel=use_data_parallel,
337
+ )
338
+ for i in range(layers)
339
+ ]
340
+ )
341
+
342
+ def forward(self, x: torch.Tensor, grid_hw: tuple[int, int]):
343
+ for block in self.resblocks:
344
+ x = block(x, grid_hw=grid_hw)
345
+ return x
346
+
347
+
348
+class PerceptionEncoder(nn.Module):
349
+ def __init__(
350
+ self,
351
+ config,
352
+ act_layer: Callable,
353
+ norm_layer: Callable = _DEFAULT_NORM_LAYER,
354
+ quant_config: QuantizationConfig | None = None,
355
+ prefix: str = "",
356
+ use_data_parallel: bool = False,
357
+ ):
358
+ super().__init__()
359
+ self.patch_size = config.patch_size
360
+
361
+ self.output_dim = config.output_dim or config.width
362
+ self.heads = config.heads
363
+ self.width = config.width
364
+ self.layers = config.layers
365
+
366
+ self.use_abs_posemb = config.use_abs_posemb
367
+ self.use_cls_token = config.use_cls_token
368
+ self.use_rope2d = config.use_rope2d
369
+ if not self.use_rope2d:
370
+ raise ValueError("use_rope2d must be True")
371
+ self.image_size = config.image_size
372
+
373
+ self.conv1 = Conv2dLayer(
374
+ in_channels=3,
375
+ out_channels=config.width,
376
+ kernel_size=config.patch_size,
377
+ stride=config.patch_size,
378
+ bias=False,
379
+ )
380
+
381
+ self.ln_pre = norm_layer(config.width) if config.use_ln_pre else nn.Identity()
382
+ self.ln_post = norm_layer(self.width) if config.use_ln_post else nn.Identity()
383
+
384
+ self.transformer = PerceptionEncoderVisionTransformer(
385
+ config.width,
386
+ config.layers,
387
+ config.heads,
388
+ max_grid_height=self.image_size // self.patch_size,
389
+ max_grid_width=self.image_size // self.patch_size,
390
+ mlp_ratio=config.mlp_ratio,
391
+ ls_init_value=config.ls_init_value,
392
+ act_layer=act_layer,
393
+ norm_layer=norm_layer,
394
+ use_cls_token=self.use_cls_token,
395
+ quant_config=quant_config,
396
+ prefix=f"{prefix}.transformer",
397
+ use_data_parallel=use_data_parallel,
398
+ )
399
+
400
+ self.vit_downsampler1 = Conv2dLayer(
401
+ config.width, config.width * 2, kernel_size=3, stride=2, padding=1
402
+ )
403
+ self.vit_downsampler2 = Conv2dLayer(
404
+ config.width * 2, config.width * 4, kernel_size=3, stride=2, padding=1
405
+ )
406
+
407
+ if self.use_cls_token:
408
+ self.class_embedding = nn.Parameter(
409
+ (self.width**-0.5) * torch.randn(self.width)
410
+ )
411
+
412
+ if self.use_abs_posemb:
413
+ self.posemb_grid_size = self.image_size // self.patch_size
414
+ self.positional_embedding = nn.Parameter(
415
+ (self.width**-0.5)
416
+ * torch.randn(
417
+ int(self.use_cls_token) + self.posemb_grid_size**2,
418
+ self.width,
419
+ )
420
+ )
421
+
422
+ def sample_abs_posemb(self, grid_h: int, grid_w: int):
423
+ if self.posemb_grid_size == grid_h and self.posemb_grid_size == grid_w:
424
+ return self.positional_embedding[None, ...]
425
+
426
+ pos_embed = self.positional_embedding
427
+ if self.use_cls_token:
428
+ cls_token_embed, pos_embed = pos_embed[:1], pos_embed[1:]
429
+
430
+ pos_embed = (
431
+ pos_embed.reshape(1, self.posemb_grid_size, self.posemb_grid_size, -1)
432
+ .permute(0, 3, 1, 2)
433
+ .contiguous()
434
+ )
435
+ pos_embed = F.interpolate(
436
+ pos_embed, size=(grid_h, grid_w), mode="bilinear", align_corners=False
437
+ )
438
+ pos_embed = pos_embed.permute(0, 2, 3, 1).reshape(-1, self.width)
439
+
440
+ if self.use_cls_token:
441
+ pos_embed = torch.cat([cls_token_embed, pos_embed], dim=0)
442
+
443
+ return pos_embed[None, ...]
444
+
445
+ def forward_features(self, x: torch.Tensor):
446
+ batch, _, h, w = x.shape
447
+ grid_h, grid_w = h // self.patch_size, w // self.patch_size
448
+
449
+ x = self.conv1(x)
450
+ x = x.permute(0, 2, 3, 1).reshape(batch, -1, self.width)
451
+
452
+ if self.use_cls_token:
453
+ x = torch.cat(
454
+ [self.class_embedding.view(1, 1, -1).expand(batch, -1, -1), x], dim=1
455
+ )
456
+
457
+ if self.use_abs_posemb:
458
+ x = x + self.sample_abs_posemb(grid_h, grid_w)
459
+
460
+ x = self.ln_pre(x)
461
+ x = self.transformer(x, grid_hw=(grid_h, grid_w))
462
+ x = self.ln_post(x)
463
+
464
+ if self.use_cls_token:
465
+ x = x[:, 1:, :]
466
+
467
+ return x
468
+
469
+ def forward(self, x: torch.Tensor):
470
+ x = self.forward_features(x)
471
+ B, P, C = x.shape
472
+ T = int(P**0.5)
473
+ x = x.transpose(2, 1).contiguous()
474
+ x = x.view(B, C, T, T)
475
+
476
+ x = self.vit_downsampler1(x)
477
+ x = self.vit_downsampler2(x)
478
+
479
+ B, C, T, T = x.shape
480
+ return x.view(B, -1, T * T).transpose(1, 2)
481
+
482
+
483
+class StepVLForConditionalGeneration(Step3VLForConditionalGeneration):
484
+ hf_to_vllm_mapper = WeightsMapper(
485
+ orig_to_new_prefix={
486
+ "model.": "language_model.model.",
487
+ "lm_head.": "language_model.lm_head.",
488
+ },
489
+ orig_to_new_substr={
490
+ ".attn.in_proj_weight": ".attn.qkv_proj.weight",
491
+ ".attn.in_proj_bias": ".attn.qkv_proj.bias",
492
+ ".mlp.c_fc": ".mlp.fc1",
493
+ ".mlp.c_proj": ".mlp.fc2",
494
+ },
495
+ )
496
+
497
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
498
+ super(Step3VLForConditionalGeneration, self).__init__()
499
+
500
+ config = vllm_config.model_config.hf_config
501
+ multimodal_config = vllm_config.model_config.multimodal_config
502
+ quant_config = vllm_config.quant_config
503
+
504
+ self.config = config
505
+ self.multimodal_config = multimodal_config
506
+ self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
507
+ if multimodal_config.get_limit_per_prompt("image"):
508
+ self.vision_model = PerceptionEncoder(
509
+ config.vision_config,
510
+ get_act_fn(config.vision_config.hidden_act),
511
+ quant_config=quant_config,
512
+ prefix=maybe_prefix(prefix, "vision_model"),
513
+ use_data_parallel=self.use_data_parallel,
514
+ )
515
+ self.vit_large_projector = ColumnParallelLinear(
516
+ config.vision_config.width * 4,
517
+ config.text_config.hidden_size,
518
+ bias=config.projector_bias,
519
+ gather_output=True,
520
+ quant_config=quant_config,
521
+ prefix=maybe_prefix(prefix, "vit_large_projector"),
522
+ disable_tp=self.use_data_parallel,
523
+ )
524
+ else:
525
+ self.vision_model = None
526
+ self.vit_large_projector = None
527
+
528
+ self.language_model = init_vllm_registered_model(
529
+ vllm_config=vllm_config,
530
+ hf_config=config.text_config,
531
+ prefix=maybe_prefix(prefix, "language_model"),
532
+ )
533
+
534
+ self.make_empty_intermediate_tensors = (
535
+ self.language_model.make_empty_intermediate_tensors
536
+ )
537
+
538
+ def _get_vision_model_output(
539
+ self, input_tensor: torch.Tensor | None
540
+ ) -> torch.Tensor | None:
541
+ if input_tensor is None:
542
+ return None
543
+ if self.use_data_parallel:
544
+ return run_dp_sharded_vision_model(input_tensor, self.vision_model)
545
+ return self.vision_model(input_tensor)
546
+
547
+ def _process_image_features(self, image_features: torch.Tensor) -> torch.Tensor:
548
+ image_features, _ = self.vit_large_projector(image_features)
549
+ return image_features