@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0240402c46
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Misc]Add BNB quantization for MolmoForCausalLM (#11551)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 28, 2024 at 02:48 UTC 0240402c4632604c9cd02f7eae4ae36fa990b38f
2 files changed +83 -33
vllm/model_executor/model_loader/loader.py
+18 -8
@@ -11,7 +11,8 @@ import os
11 import warnings
12 from abc import ABC, abstractmethod
13 from contextlib import contextmanager
14 -from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, cast
14 +from typing import (Any, Callable, Dict, Generator, Iterable, List, Optional,
15 + Tuple, cast)
16
17 import gguf
18 import huggingface_hub
@@ -706,6 +707,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
707 # Store all module names (from transformers) that support
708 # BNB quantization.
709 self.target_modules: List[str] = []
710 + # mapping weight names from transformers to vllm.
711 + self.weight_mapper: Callable = lambda name: name
712
713 def _get_weight_files(
714 self,
@@ -763,9 +766,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
766
767 def _hf_weight_iter(self, hf_weights_files, use_safetensors: bool):
768 if use_safetensors:
766 - return safetensors_weights_iterator(hf_weights_files)
769 + iterator = safetensors_weights_iterator(hf_weights_files)
770 else:
768 - return pt_weights_iterator(hf_weights_files)
771 + iterator = pt_weights_iterator(hf_weights_files)
772 + for name, param in iterator:
773 + # mapping weight names from transformers to vllm.
774 + yield self.weight_mapper(name), param
775
776 def _get_quantized_weights_iterator(
777 self,
@@ -782,12 +788,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
788 try:
789 import bitsandbytes
790
785 - if bitsandbytes.__version__ < "0.44.0":
791 + if bitsandbytes.__version__ < "0.45.0":
792 raise ImportError("bitsandbytes version is wrong. Please "
787 - "install bitsandbytes>=0.44.0.")
793 + "install bitsandbytes>=0.45.0.")
794 except ImportError as err:
789 - raise ImportError("Please install bitsandbytes>=0.44.0 via "
790 - "`pip install bitsandbytes>=0.44.0` to use "
795 + raise ImportError("Please install bitsandbytes>=0.45.0 via "
796 + "`pip install bitsandbytes>=0.45.0` to use "
797 "bitsandbytes quantizer.") from err
798
799 hf_weights_files, use_safetensors = self._prepare_weights(
@@ -991,7 +997,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
997 if isinstance(module, (LinearBase, )):
998 last_name = name.split(".")[-1]
999 if sub_modules := inverse_stacked_mapping.get(last_name, []):
994 - # Map vllm's names to transformers' names.
1000 + # Map vllm's names to transformers's names.
1001 for sub_name in sub_modules:
1002 self.target_modules.append(
1003 name.replace(last_name, sub_name))
@@ -1013,6 +1019,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1019 f"Model {type(model).__name__} does not support BitsAndBytes "
1020 "quantization yet.")
1021
1022 + # For some models like Molmo, we need to use hf_to_vllm_mapper
1023 + # to ensure correct loading of weights.
1024 + if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None):
1025 + self.weight_mapper = lambda name: hf_to_vllm_mapper._map_name(name)
1026 # Modules whose weights might have fused on disk
1027 # we need their output_sizes to make shard in flight correctly with TP
1028 self.maybe_fused_weights_modules: Dict[str, List[int]] = {}
vllm/model_executor/models/molmo.py
+65 -25
@@ -461,30 +461,71 @@ class MolmoAttention(nn.Module):
461 return output
462
463
464 -class MolmoMLP(nn.Module):
464 +class SwiGLU(nn.Module):
465 +
466 + def forward(self, x: torch.Tensor) -> torch.Tensor:
467 + x, gate = x.chunk(2, dim=-1)
468 + # Note that the order is reversed compared to
469 + # SiluAndMul.
470 + return x * F.silu(gate)
471 +
472 +
473 +class LanuageModelMLP(nn.Module):
474 """Molmo's LLM mlp."""
475
476 def __init__(self,
477 config: PretrainedConfig,
478 input_dim: Optional[int] = None,
470 - quant_config: Optional[QuantizationConfig] = None,
471 - proj_name: str = "gate_up_proj") -> None:
479 + quant_config: Optional[QuantizationConfig] = None) -> None:
480 super().__init__()
481 self.hidden_size = config.hidden_size
482 self.intermediate_size = config.intermediate_size // 2
483
476 - # Molmo's LLM proj weights are already merged into the disk, while
477 - # image_projector proj is separate. If the same proj_name were used, it
478 - # would create ambiguity and make it difficult to support BNB and LoRA.
479 - self.proj_name = proj_name
480 - setattr(
481 - self, proj_name,
482 - MergedColumnParallelLinear(
483 - input_dim or self.hidden_size,
484 - [self.intermediate_size] * 2,
485 - bias=False,
486 - quant_config=quant_config,
487 - ))
484 + self.gate_up_proj = MergedColumnParallelLinear(
485 + input_dim or self.hidden_size,
486 + [self.intermediate_size] * 2,
487 + bias=False,
488 + quant_config=quant_config,
489 + )
490 + # Activation function.
491 + self.act_fn = SwiGLU()
492 + # Feed-forward output projection.
493 + self.down_proj = RowParallelLinear(
494 + self.intermediate_size,
495 + self.hidden_size,
496 + bias=False,
497 + quant_config=quant_config,
498 + )
499 +
500 + def forward(
501 + self,
502 + x: torch.Tensor,
503 + ) -> torch.Tensor:
504 + gate_up, _ = self.gate_up_proj(x)
505 + x = self.act_fn(gate_up)
506 + x, _ = self.down_proj(x)
507 + return x
508 +
509 +
510 +class ImageProjectorMLP(nn.Module):
511 + """Molmo's image_projector mlp."""
512 +
513 + def __init__(
514 + self,
515 + config: PretrainedConfig,
516 + input_dim: Optional[int] = None,
517 + quant_config: Optional[QuantizationConfig] = None,
518 + ) -> None:
519 + super().__init__()
520 + self.hidden_size = config.hidden_size
521 + self.intermediate_size = config.intermediate_size // 2
522 +
523 + self.merged_linear = MergedColumnParallelLinear(
524 + input_dim or self.hidden_size,
525 + [self.intermediate_size] * 2,
526 + bias=False,
527 + quant_config=quant_config,
528 + )
529 # Activation function.
530 self.act_fn = SiluAndMul()
531
@@ -500,7 +541,7 @@ class MolmoMLP(nn.Module):
541 self,
542 x: torch.Tensor,
543 ) -> torch.Tensor:
503 - gate_up, _ = getattr(self, self.proj_name)(x)
544 + gate_up, _ = self.merged_linear(x)
545 x = self.act_fn(gate_up)
546 x, _ = self.down_proj(x)
547 return x
@@ -523,9 +564,7 @@ class MolmoDecoderLayer(nn.Module):
564 prefix=f"{prefix}.self_attn")
565
566 # MLP block.
526 - self.mlp = MolmoMLP(config,
527 - quant_config=quant_config,
528 - proj_name="gate_up_proj")
567 + self.mlp = LanuageModelMLP(config, quant_config=quant_config)
568
569 # LayerNorm
570 assert config.layer_norm_type == "rms"
@@ -617,11 +656,10 @@ class MolmoVisionBackbone(nn.Module):
656 vision_config,
657 nlayers=len(self.vit_layers),
658 quant_config=quant_config)
620 - self.image_projector = MolmoMLP(
659 + self.image_projector = ImageProjectorMLP(
660 config,
661 input_dim=vision_config.image_emb_dim,
662 quant_config=quant_config,
624 - proj_name="merged_linear",
663 )
664
665 image_dim = vision_config.image_emb_dim * len(self.vit_layers)
@@ -842,10 +880,6 @@ class MolmoModel(nn.Module):
880 loaded_params: Set[str] = set()
881
882 for name, loaded_weight in weights:
845 - if "gate_up_proj" in name:
846 - up_proj, gate_proj = loaded_weight.chunk(2, dim=0)
847 - loaded_weight = torch.cat([gate_proj, up_proj], dim=0)
848 -
883 if name.endswith(".bias") and name not in params_dict:
884 continue
885 if is_pp_missing_parameter(name, self):
@@ -1157,6 +1191,12 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1191 },
1192 )
1193
1194 + # BitandBytes specific attributes
1195 + bitsandbytes_stacked_params_mapping = {
1196 + "gate_proj": ("merged_linear", 0),
1197 + "up_proj": ("merged_linear", 1),
1198 + }
1199 +
1200 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1201 super().__init__()
1202 config = vllm_config.model_config.hf_config