A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Misc]Add BNB quantization for MolmoForCausalLM (#11551)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 28, 2024 at 02:48 UTC
0240402c4632604c9cd02f7eae4ae36fa990b38f
2 files changed
+83
-33
vllm/model_executor/model_loader/loader.py
+18
-8
@@ -11,7 +11,8 @@ import os
11
import warnings
12
from abc import ABC, abstractmethod
13
from contextlib import contextmanager
14
-from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, cast
14
+from typing import (Any, Callable, Dict, Generator, Iterable, List, Optional,
15
+ Tuple, cast)
16
17
import gguf
18
import huggingface_hub
@@ -706,6 +707,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
707
# Store all module names (from transformers) that support
708
# BNB quantization.
709
self.target_modules: List[str] = []
710
+ # mapping weight names from transformers to vllm.
711
+ self.weight_mapper: Callable = lambda name: name
712
713
def _get_weight_files(
714
self,
@@ -763,9 +766,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
766
767
def _hf_weight_iter(self, hf_weights_files, use_safetensors: bool):
768
if use_safetensors:
766
- return safetensors_weights_iterator(hf_weights_files)
769
+ iterator = safetensors_weights_iterator(hf_weights_files)
770
else:
768
- return pt_weights_iterator(hf_weights_files)
771
+ iterator = pt_weights_iterator(hf_weights_files)
772
+ for name, param in iterator:
773
+ # mapping weight names from transformers to vllm.
774
+ yield self.weight_mapper(name), param
775
776
def _get_quantized_weights_iterator(
777
self,
@@ -782,12 +788,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
788
try:
789
import bitsandbytes
790
785
- if bitsandbytes.__version__ < "0.44.0":
791
+ if bitsandbytes.__version__ < "0.45.0":
792
raise ImportError("bitsandbytes version is wrong. Please "
787
- "install bitsandbytes>=0.44.0.")
793
+ "install bitsandbytes>=0.45.0.")
794
except ImportError as err:
789
- raise ImportError("Please install bitsandbytes>=0.44.0 via "
790
- "`pip install bitsandbytes>=0.44.0` to use "
795
+ raise ImportError("Please install bitsandbytes>=0.45.0 via "
796
+ "`pip install bitsandbytes>=0.45.0` to use "
797
"bitsandbytes quantizer.") from err
798
799
hf_weights_files, use_safetensors = self._prepare_weights(
@@ -991,7 +997,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
997
if isinstance(module, (LinearBase, )):
998
last_name = name.split(".")[-1]
999
if sub_modules := inverse_stacked_mapping.get(last_name, []):
994
- # Map vllm's names to transformers' names.
1000
+ # Map vllm's names to transformers's names.
1001
for sub_name in sub_modules:
1002
self.target_modules.append(
1003
name.replace(last_name, sub_name))
@@ -1013,6 +1019,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1019
f"Model {type(model).__name__} does not support BitsAndBytes "
1020
"quantization yet.")
1021
1022
+ # For some models like Molmo, we need to use hf_to_vllm_mapper
1023
+ # to ensure correct loading of weights.
1024
+ if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None):
1025
+ self.weight_mapper = lambda name: hf_to_vllm_mapper._map_name(name)
1026
# Modules whose weights might have fused on disk
1027
# we need their output_sizes to make shard in flight correctly with TP
1028
self.maybe_fused_weights_modules: Dict[str, List[int]] = {}
vllm/model_executor/models/molmo.py
+65
-25
@@ -461,30 +461,71 @@ class MolmoAttention(nn.Module):
461
return output
462
463
464
-class MolmoMLP(nn.Module):
464
+class SwiGLU(nn.Module):
465
+
466
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
467
+ x, gate = x.chunk(2, dim=-1)
468
+ # Note that the order is reversed compared to
469
+ # SiluAndMul.
470
+ return x * F.silu(gate)
471
+
472
+
473
+class LanuageModelMLP(nn.Module):
474
"""Molmo's LLM mlp."""
475
476
def __init__(self,
477
config: PretrainedConfig,
478
input_dim: Optional[int] = None,
470
- quant_config: Optional[QuantizationConfig] = None,
471
- proj_name: str = "gate_up_proj") -> None:
479
+ quant_config: Optional[QuantizationConfig] = None) -> None:
480
super().__init__()
481
self.hidden_size = config.hidden_size
482
self.intermediate_size = config.intermediate_size // 2
483
476
- # Molmo's LLM proj weights are already merged into the disk, while
477
- # image_projector proj is separate. If the same proj_name were used, it
478
- # would create ambiguity and make it difficult to support BNB and LoRA.
479
- self.proj_name = proj_name
480
- setattr(
481
- self, proj_name,
482
- MergedColumnParallelLinear(
483
- input_dim or self.hidden_size,
484
- [self.intermediate_size] * 2,
485
- bias=False,
486
- quant_config=quant_config,
487
- ))
484
+ self.gate_up_proj = MergedColumnParallelLinear(
485
+ input_dim or self.hidden_size,
486
+ [self.intermediate_size] * 2,
487
+ bias=False,
488
+ quant_config=quant_config,
489
+ )
490
+ # Activation function.
491
+ self.act_fn = SwiGLU()
492
+ # Feed-forward output projection.
493
+ self.down_proj = RowParallelLinear(
494
+ self.intermediate_size,
495
+ self.hidden_size,
496
+ bias=False,
497
+ quant_config=quant_config,
498
+ )
499
+
500
+ def forward(
501
+ self,
502
+ x: torch.Tensor,
503
+ ) -> torch.Tensor:
504
+ gate_up, _ = self.gate_up_proj(x)
505
+ x = self.act_fn(gate_up)
506
+ x, _ = self.down_proj(x)
507
+ return x
508
+
509
+
510
+class ImageProjectorMLP(nn.Module):
511
+ """Molmo's image_projector mlp."""
512
+
513
+ def __init__(
514
+ self,
515
+ config: PretrainedConfig,
516
+ input_dim: Optional[int] = None,
517
+ quant_config: Optional[QuantizationConfig] = None,
518
+ ) -> None:
519
+ super().__init__()
520
+ self.hidden_size = config.hidden_size
521
+ self.intermediate_size = config.intermediate_size // 2
522
+
523
+ self.merged_linear = MergedColumnParallelLinear(
524
+ input_dim or self.hidden_size,
525
+ [self.intermediate_size] * 2,
526
+ bias=False,
527
+ quant_config=quant_config,
528
+ )
529
# Activation function.
530
self.act_fn = SiluAndMul()
531
@@ -500,7 +541,7 @@ class MolmoMLP(nn.Module):
541
self,
542
x: torch.Tensor,
543
) -> torch.Tensor:
503
- gate_up, _ = getattr(self, self.proj_name)(x)
544
+ gate_up, _ = self.merged_linear(x)
545
x = self.act_fn(gate_up)
546
x, _ = self.down_proj(x)
547
return x
@@ -523,9 +564,7 @@ class MolmoDecoderLayer(nn.Module):
564
prefix=f"{prefix}.self_attn")
565
566
# MLP block.
526
- self.mlp = MolmoMLP(config,
527
- quant_config=quant_config,
528
- proj_name="gate_up_proj")
567
+ self.mlp = LanuageModelMLP(config, quant_config=quant_config)
568
569
# LayerNorm
570
assert config.layer_norm_type == "rms"
@@ -617,11 +656,10 @@ class MolmoVisionBackbone(nn.Module):
656
vision_config,
657
nlayers=len(self.vit_layers),
658
quant_config=quant_config)
620
- self.image_projector = MolmoMLP(
659
+ self.image_projector = ImageProjectorMLP(
660
config,
661
input_dim=vision_config.image_emb_dim,
662
quant_config=quant_config,
624
- proj_name="merged_linear",
663
)
664
665
image_dim = vision_config.image_emb_dim * len(self.vit_layers)
@@ -842,10 +880,6 @@ class MolmoModel(nn.Module):
880
loaded_params: Set[str] = set()
881
882
for name, loaded_weight in weights:
845
- if "gate_up_proj" in name:
846
- up_proj, gate_proj = loaded_weight.chunk(2, dim=0)
847
- loaded_weight = torch.cat([gate_proj, up_proj], dim=0)
848
-
883
if name.endswith(".bias") and name not in params_dict:
884
continue
885
if is_pp_missing_parameter(name, self):
@@ -1157,6 +1191,12 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1191
},
1192
)
1193
1194
+ # BitandBytes specific attributes
1195
+ bitsandbytes_stacked_params_mapping = {
1196
+ "gate_proj": ("merged_linear", 0),
1197
+ "up_proj": ("merged_linear", 1),
1198
+ }
1199
+
1200
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1201
super().__init__()
1202
config = vllm_config.model_config.hf_config