A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Model] Modify MolmoForCausalLM MLP (#11510)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 26, 2024 at 21:12 UTC
f57ee5650dd402c6147980824c6936c96cfa59fe
1 file changed
+24
-18
vllm/model_executor/models/molmo.py
+24
-18
@@ -464,24 +464,27 @@ class MolmoAttention(nn.Module):
464
class MolmoMLP(nn.Module):
465
"""Molmo's LLM mlp."""
466
467
- def __init__(
468
- self,
469
- config: PretrainedConfig,
470
- input_dim: Optional[int] = None,
471
- quant_config: Optional[QuantizationConfig] = None,
472
- ) -> None:
467
+ def __init__(self,
468
+ config: PretrainedConfig,
469
+ input_dim: Optional[int] = None,
470
+ quant_config: Optional[QuantizationConfig] = None,
471
+ proj_name: str = "gate_up_proj") -> None:
472
super().__init__()
473
self.hidden_size = config.hidden_size
474
self.intermediate_size = config.intermediate_size // 2
475
477
- # Feed-forward input projection.
478
- self.gate_up_proj = MergedColumnParallelLinear(
479
- input_dim or self.hidden_size,
480
- [self.intermediate_size] * 2,
481
- bias=False,
482
- quant_config=quant_config,
483
- )
484
-
476
+ # Molmo's LLM proj weights are already merged into the disk, while
477
+ # image_projector proj is separate. If the same proj_name were used, it
478
+ # would create ambiguity and make it difficult to support BNB and LoRA.
479
+ self.proj_name = proj_name
480
+ setattr(
481
+ self, proj_name,
482
+ MergedColumnParallelLinear(
483
+ input_dim or self.hidden_size,
484
+ [self.intermediate_size] * 2,
485
+ bias=False,
486
+ quant_config=quant_config,
487
+ ))
488
# Activation function.
489
self.act_fn = SiluAndMul()
490
@@ -497,7 +500,7 @@ class MolmoMLP(nn.Module):
500
self,
501
x: torch.Tensor,
502
) -> torch.Tensor:
500
- gate_up, _ = self.gate_up_proj(x)
503
+ gate_up, _ = getattr(self, self.proj_name)(x)
504
x = self.act_fn(gate_up)
505
x, _ = self.down_proj(x)
506
return x
@@ -520,7 +523,9 @@ class MolmoDecoderLayer(nn.Module):
523
prefix=f"{prefix}.self_attn")
524
525
# MLP block.
523
- self.mlp = MolmoMLP(config, quant_config=quant_config)
526
+ self.mlp = MolmoMLP(config,
527
+ quant_config=quant_config,
528
+ proj_name="gate_up_proj")
529
530
# LayerNorm
531
assert config.layer_norm_type == "rms"
@@ -616,6 +621,7 @@ class MolmoVisionBackbone(nn.Module):
621
config,
622
input_dim=vision_config.image_emb_dim,
623
quant_config=quant_config,
624
+ proj_name="merged_linear",
625
)
626
627
image_dim = vision_config.image_emb_dim * len(self.vit_layers)
@@ -714,8 +720,8 @@ class MolmoVisionBackbone(nn.Module):
720
torch.Tensor]]) -> Set[str]:
721
stacked_params_mapping = [
722
# (param_name, shard_name, shard_id)
717
- ("gate_up_proj", "gate_proj", 0),
718
- ("gate_up_proj", "up_proj", 1),
723
+ ("merged_linear", "gate_proj", 0),
724
+ ("merged_linear", "up_proj", 1),
725
]
726
params_dict = dict(self.named_parameters())
727
loaded_params: Set[str] = set()