@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09500f7dde
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Add BNB quantization support for Mllama (#9720)

Isotr0py committed Oct 29, 2024 at 20:20 UTC 09500f7ddeb974730972fd9284bd93c08a557cf6
3 files changed +84 -12
vllm/model_executor/layers/quantization/bitsandbytes.py
+31 -4
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional
3 import torch
4
5 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
6 + UnquantizedLinearMethod,
7 set_weight_attrs)
8 from vllm.model_executor.layers.quantization.base_config import (
9 QuantizationConfig)
@@ -23,7 +24,7 @@ class BitsAndBytesConfig(QuantizationConfig):
24 bnb_4bit_use_double_quant: bool = False,
25 llm_int8_enable_fp32_cpu_offload: bool = False,
26 llm_int8_has_fp16_weight: bool = False,
26 - llm_int8_skip_modules: Optional[Any] = None,
27 + llm_int8_skip_modules: Optional[List[str]] = None,
28 llm_int8_threshold: float = 0.0,
29 ) -> None:
30
@@ -34,11 +35,15 @@ class BitsAndBytesConfig(QuantizationConfig):
35 self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant
36 self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload
37 self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight
37 - self.llm_int8_skip_modules = llm_int8_skip_modules
38 + self.llm_int8_skip_modules = llm_int8_skip_modules or []
39 self.llm_int8_threshold = llm_int8_threshold
40
41 def __repr__(self) -> str:
41 - return "BitsAndBytesConfig"
42 + return (f"BitsAndBytesConfig(load_in_8bit={self.load_in_8bit}, "
43 + f"load_in_4bit={self.load_in_4bit}, "
44 + f"bnb_4bit_compute_dtype={self.bnb_4bit_compute_dtype}, "
45 + f"bnb_4bit_quant_type={self.bnb_4bit_quant_type}, "
46 + f"llm_int8_skip_modules={self.llm_int8_skip_modules})")
47
48 @classmethod
49 def get_name(self) -> str:
@@ -102,8 +107,10 @@ class BitsAndBytesConfig(QuantizationConfig):
107 llm_int8_threshold=llm_int8_threshold)
108
109 def get_quant_method(self, layer: torch.nn.Module,
105 - prefix: str) -> Optional["BitsAndBytesLinearMethod"]:
110 + prefix: str) -> Optional["LinearMethodBase"]:
111 if isinstance(layer, LinearBase):
112 + if is_layer_skipped_bnb(prefix, self.llm_int8_skip_modules):
113 + return UnquantizedLinearMethod()
114 return BitsAndBytesLinearMethod(self)
115 return None
116
@@ -111,6 +118,10 @@ class BitsAndBytesConfig(QuantizationConfig):
118 return []
119
120
121 +def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
122 + return any(module_name in prefix for module_name in llm_int8_skip_modules)
123 +
124 +
125 class BitsAndBytesLinearMethod(LinearMethodBase):
126 """Linear method for BitsAndBytes.
127
@@ -211,6 +222,11 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
222 from bitsandbytes import MatmulLtState, matmul
223
224 original_type = x.dtype
225 + original_shape = x.shape
226 + reshape_after_matmul = False
227 + if x.ndim > 2:
228 + x = x.reshape(-1, x.size(-1))
229 + reshape_after_matmul = True
230 bf_x = x.to(torch.bfloat16)
231
232 qweight = layer.qweight
@@ -265,6 +281,9 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
281
282 out = out.to(original_type)
283
284 + if reshape_after_matmul:
285 + out = out.view(*original_shape[:-1], out.size(-1))
286 +
287 if bias is not None:
288 out += bias
289
@@ -282,6 +301,11 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
301 from bitsandbytes import matmul_4bit
302
303 original_type = x.dtype
304 + original_shape = x.shape
305 + reshape_after_matmul = False
306 + if x.ndim > 2:
307 + x = x.reshape(-1, x.size(-1))
308 + reshape_after_matmul = True
309 bf_x = x.to(torch.bfloat16)
310
311 qweight = layer.qweight
@@ -310,6 +334,9 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
334
335 out = out.to(original_type)
336
337 + if reshape_after_matmul:
338 + out = out.view(*original_shape[:-1], out.size(-1))
339 +
340 if bias is not None:
341 out += bias
342
vllm/model_executor/model_loader/loader.py
+16 -3
@@ -899,6 +899,19 @@ class BitsAndBytesModelLoader(BaseModelLoader):
899 return self._unquantized_generator(hf_weights_files, use_safetensors,
900 quant_state_dict), quant_state_dict
901
902 + def _is_8bit_weight_name(self, weight_name: str):
903 + quantized_suffix = {".scb", ".weight_format"}
904 + return any(weight_name.lower().endswith(suffix)
905 + for suffix in quantized_suffix)
906 +
907 + def _is_4bit_weight_name(self, weight_name: str):
908 + quantized_suffix = {
909 + "absmax", "quant_map", "nested_absmax", "nested_quant_map",
910 + "bitsandbytes"
911 + }
912 + suffix = weight_name.split(".")[-1]
913 + return any(q_suffix in suffix for q_suffix in quantized_suffix)
914 +
915 def _quantized_8bit_generator(self, hf_weights_files, use_safetensors,
916 quant_state_dict) -> Generator:
917 for weight_name, weight_tensor in self._hf_weight_iter(
@@ -912,7 +925,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
925 for weight_name, weight_tensor in self._hf_weight_iter(
926 hf_weights_files, use_safetensors):
927
915 - if not weight_name.endswith((".weight", ".bias")):
928 + if self._is_8bit_weight_name(weight_name):
929 continue
930
931 qweight_name = weight_name.replace(".weight", ".qweight")
@@ -932,7 +945,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
945 use_safetensors)
946 temp_state_dict = {}
947 for weight_name, weight_tensor in weight_iterator:
935 - if weight_name.endswith((".weight", ".bias")):
948 + if not self._is_4bit_weight_name(weight_name):
949 continue
950 # bitsandbytes library requires
951 # weight.quant_state.bitsandbytes__* in CPU
@@ -956,7 +969,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
969 for weight_name, weight_tensor in self._hf_weight_iter(
970 hf_weights_files, use_safetensors):
971
959 - if not weight_name.endswith((".weight", ".bias")):
972 + if self._is_4bit_weight_name(weight_name):
973 continue
974
975 if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
vllm/model_executor/models/mllama.py
+37 -5
@@ -325,7 +325,10 @@ class MllamaPrecomputedPositionEmbedding(nn.Module):
325 # TODO: support other attention backends for attention in vision model
326 class MllamaVisionSdpaAttention(nn.Module):
327
328 - def __init__(self, config: config_mllama.MllamaVisionConfig):
328 + def __init__(self,
329 + config: config_mllama.MllamaVisionConfig,
330 + quant_config: Optional[QuantizationConfig] = None,
331 + prefix: str = ""):
332 super().__init__()
333
334 model_parallel_size = get_tensor_model_parallel_world_size()
@@ -341,12 +344,16 @@ class MllamaVisionSdpaAttention(nn.Module):
344 self.head_dim,
345 self.num_heads,
346 bias=False,
347 + quant_config=quant_config,
348 + prefix=f"{prefix}.qkv_proj",
349 )
350 self.o_proj = RowParallelLinear(
351 self.num_heads * self.head_dim,
352 self.embed_dim,
353 bias=False,
354 input_is_parallel=True,
355 + quant_config=quant_config,
356 + prefix=f"{prefix}.o_proj",
357 )
358
359 def forward(
@@ -393,7 +400,8 @@ class MllamaVisionEncoderLayer(nn.Module):
400 self.is_gated = is_gated
401 self.intermediate_size = config.intermediate_size
402
396 - self.self_attn = MllamaVisionSdpaAttention(config)
403 + self.self_attn = MllamaVisionSdpaAttention(
404 + config, quant_config=quant_config, prefix=f"{prefix}.self_attn")
405 self.mlp = CLIPMLP(config,
406 quant_config=quant_config,
407 prefix=f"{prefix}.mlp")
@@ -1002,6 +1010,7 @@ class MllamaForCausalLM(nn.Module):
1010 org_num_embeddings=config.vocab_size,
1011 padding_size=DEFAULT_VOCAB_PADDING_SIZE,
1012 quant_config=quant_config,
1013 + prefix=f"{prefix}.lm_head",
1014 )
1015
1016 def forward(
@@ -1037,6 +1046,26 @@ class MllamaForCausalLM(nn.Module):
1046 @INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
1047 @INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
1048 class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1049 + # BitandBytes specific attributes
1050 + default_bitsandbytes_target_modules = [
1051 + ".gate_proj.",
1052 + ".down_proj.",
1053 + ".up_proj.",
1054 + ".q_proj.",
1055 + ".k_proj.",
1056 + ".v_proj.",
1057 + ".o_proj.",
1058 + ]
1059 + # in TP, these weights are partitioned along the column dimension (dim=-1)
1060 + column_parallel_weights_modules = [".down_proj.", ".o_proj."]
1061 + bitsandbytes_stacked_params_mapping = {
1062 + # shard_name, weight_name, index
1063 + "q_proj": ("qkv_proj", 0),
1064 + "k_proj": ("qkv_proj", 1),
1065 + "v_proj": ("qkv_proj", 2),
1066 + "gate_proj": ("gate_up_proj", 0),
1067 + "up_proj": ("gate_up_proj", 1),
1068 + }
1069
1070 def __init__(self,
1071 config: config_mllama.MllamaConfig,
@@ -1061,10 +1090,13 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1090 quant_config=quant_config,
1091 prefix="language_model",
1092 )
1064 - self.multi_modal_projector = nn.Linear(
1093 + self.multi_modal_projector = ColumnParallelLinear(
1094 config.vision_config.vision_output_dim,
1095 config.text_config.hidden_size,
1096 bias=True,
1097 + quant_config=quant_config,
1098 + gather_output=True,
1099 + prefix="multi_modal_projector",
1100 )
1101 self.logits_processor = LogitsProcessor(config.output_hidden_states,
1102 config.text_config.vocab_size)
@@ -1128,7 +1160,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1160 raise ValueError("No images provided.")
1161 max_num_tiles = max(
1162 max([len(x) for x in y[0]]) for y in pixel_values)
1131 - device = self.multi_modal_projector.weight.device
1163 + device = next(self.multi_modal_projector.parameters()).device
1164 bsz = len(pixel_values)
1165 out_num_tiles = []
1166 out_images = torch.zeros(
@@ -1204,7 +1236,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1236 cross_attention_states = self.vision_model(pixel_values,
1237 aspect_ratio_ids,
1238 aspect_ratio_mask)
1207 - cross_attention_states = self.multi_modal_projector(
1239 + cross_attention_states, _ = self.multi_modal_projector(
1240 cross_attention_states)
1241
1242 bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)