@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f57ee5650d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Model] Modify MolmoForCausalLM MLP (#11510)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 26, 2024 at 21:12 UTC f57ee5650dd402c6147980824c6936c96cfa59fe
1 file changed +24 -18
vllm/model_executor/models/molmo.py
+24 -18
@@ -464,24 +464,27 @@ class MolmoAttention(nn.Module):
464 class MolmoMLP(nn.Module):
465 """Molmo's LLM mlp."""
466
467 - def __init__(
468 - self,
469 - config: PretrainedConfig,
470 - input_dim: Optional[int] = None,
471 - quant_config: Optional[QuantizationConfig] = None,
472 - ) -> None:
467 + def __init__(self,
468 + config: PretrainedConfig,
469 + input_dim: Optional[int] = None,
470 + quant_config: Optional[QuantizationConfig] = None,
471 + proj_name: str = "gate_up_proj") -> None:
472 super().__init__()
473 self.hidden_size = config.hidden_size
474 self.intermediate_size = config.intermediate_size // 2
475
477 - # Feed-forward input projection.
478 - self.gate_up_proj = MergedColumnParallelLinear(
479 - input_dim or self.hidden_size,
480 - [self.intermediate_size] * 2,
481 - bias=False,
482 - quant_config=quant_config,
483 - )
484 -
476 + # Molmo's LLM proj weights are already merged into the disk, while
477 + # image_projector proj is separate. If the same proj_name were used, it
478 + # would create ambiguity and make it difficult to support BNB and LoRA.
479 + self.proj_name = proj_name
480 + setattr(
481 + self, proj_name,
482 + MergedColumnParallelLinear(
483 + input_dim or self.hidden_size,
484 + [self.intermediate_size] * 2,
485 + bias=False,
486 + quant_config=quant_config,
487 + ))
488 # Activation function.
489 self.act_fn = SiluAndMul()
490
@@ -497,7 +500,7 @@ class MolmoMLP(nn.Module):
500 self,
501 x: torch.Tensor,
502 ) -> torch.Tensor:
500 - gate_up, _ = self.gate_up_proj(x)
503 + gate_up, _ = getattr(self, self.proj_name)(x)
504 x = self.act_fn(gate_up)
505 x, _ = self.down_proj(x)
506 return x
@@ -520,7 +523,9 @@ class MolmoDecoderLayer(nn.Module):
523 prefix=f"{prefix}.self_attn")
524
525 # MLP block.
523 - self.mlp = MolmoMLP(config, quant_config=quant_config)
526 + self.mlp = MolmoMLP(config,
527 + quant_config=quant_config,
528 + proj_name="gate_up_proj")
529
530 # LayerNorm
531 assert config.layer_norm_type == "rms"
@@ -616,6 +621,7 @@ class MolmoVisionBackbone(nn.Module):
621 config,
622 input_dim=vision_config.image_emb_dim,
623 quant_config=quant_config,
624 + proj_name="merged_linear",
625 )
626
627 image_dim = vision_config.image_emb_dim * len(self.vit_layers)
@@ -714,8 +720,8 @@ class MolmoVisionBackbone(nn.Module):
720 torch.Tensor]]) -> Set[str]:
721 stacked_params_mapping = [
722 # (param_name, shard_name, shard_id)
717 - ("gate_up_proj", "gate_proj", 0),
718 - ("gate_up_proj", "up_proj", 1),
723 + ("merged_linear", "gate_proj", 0),
724 + ("merged_linear", "up_proj", 1),
725 ]
726 params_dict = dict(self.named_parameters())
727 loaded_params: Set[str] = set()