[Model] Add tower and connector LoRA support for LFM2-VL (#51498)
Signed-off-by: zupengwang <71580390+zupengwang@users.noreply.github.com>
Zupeng Wang committed
Aug 20, 2026 at 20:33 UTC
df1376907b16b8b57a6c08fe074f62015d81cde3
2 files changed
+22
-5
vllm/model_executor/models/lfm2_siglip2.py
+5
-3
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention import MMEncoderAttention
20
from vllm.model_executor.layers.linear import (
21
ColumnParallelLinear,
22
QKVParallelLinear,
23
+ ReplicatedLinear,
24
RowParallelLinear,
25
)
26
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -37,9 +38,10 @@ class Siglip2VisionEmbeddings(nn.Module):
38
self.config = config
39
self.embed_dim = config.hidden_size
40
self.patch_size = config.patch_size
40
- self.patch_embedding = nn.Linear(
41
- in_features=config.num_channels * self.patch_size * self.patch_size,
42
- out_features=self.embed_dim,
41
+ self.patch_embedding = ReplicatedLinear(
42
+ input_size=config.num_channels * self.patch_size * self.patch_size,
43
+ output_size=self.embed_dim,
44
+ return_bias=False,
45
)
46
self.num_patches = config.num_patches
47
self.position_embedding_size = int(self.num_patches**0.5)
vllm/model_executor/models/lfm2_vl.py
+17
-2
@@ -22,6 +22,7 @@ from vllm.config import VllmConfig
22
from vllm.config.multimodal import BaseDummyOptions
23
from vllm.forward_context import set_forward_context
24
from vllm.inputs import MultiModalDataDict
25
+from vllm.model_executor.layers.linear import ReplicatedLinear
26
from vllm.model_executor.layers.mamba.mamba_utils import (
27
MambaStateCopyFunc,
28
MambaStateCopyFuncCalculator,
@@ -499,16 +500,20 @@ class Lfm2VLMultiModalProjector(nn.Module):
500
self.projector_use_layernorm = config.projector_use_layernorm
501
if self.projector_use_layernorm:
502
self.layer_norm = nn.LayerNorm(in_channels)
502
- self.linear_1 = nn.Linear(
503
+ self.linear_1 = ReplicatedLinear(
504
in_channels,
505
config.projector_hidden_size,
506
bias=config.projector_bias,
507
+ prefix=maybe_prefix(prefix, "linear_1"),
508
+ return_bias=False,
509
)
510
self.act = ACT2FN[config.projector_hidden_act]
508
- self.linear_2 = nn.Linear(
511
+ self.linear_2 = ReplicatedLinear(
512
config.projector_hidden_size,
513
config.text_config.hidden_size,
514
bias=config.projector_bias,
515
+ prefix=maybe_prefix(prefix, "linear_2"),
516
+ return_bias=False,
517
)
518
519
def forward(
@@ -1259,3 +1264,13 @@ class Lfm2VLForConditionalGeneration(
1264
connector="multi_modal_projector",
1265
tower_model="vision_tower",
1266
)
1267
+
1268
+ def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
1269
+ downsample_factor = self.config.downsample_factor
1270
+
1271
+ return num_image_tokens * downsample_factor**2
1272
+
1273
+ def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
1274
+ downsample_factor = self.config.downsample_factor
1275
+
1276
+ return num_vision_tokens // downsample_factor**2