A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Add BNB quantization support for Mllama (#9720)
Isotr0py committed
Oct 29, 2024 at 20:20 UTC
09500f7ddeb974730972fd9284bd93c08a557cf6
3 files changed
+84
-12
vllm/model_executor/layers/quantization/bitsandbytes.py
+31
-4
@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional
3
import torch
4
5
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
6
+ UnquantizedLinearMethod,
7
set_weight_attrs)
8
from vllm.model_executor.layers.quantization.base_config import (
9
QuantizationConfig)
@@ -23,7 +24,7 @@ class BitsAndBytesConfig(QuantizationConfig):
24
bnb_4bit_use_double_quant: bool = False,
25
llm_int8_enable_fp32_cpu_offload: bool = False,
26
llm_int8_has_fp16_weight: bool = False,
26
- llm_int8_skip_modules: Optional[Any] = None,
27
+ llm_int8_skip_modules: Optional[List[str]] = None,
28
llm_int8_threshold: float = 0.0,
29
) -> None:
30
@@ -34,11 +35,15 @@ class BitsAndBytesConfig(QuantizationConfig):
35
self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant
36
self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload
37
self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight
37
- self.llm_int8_skip_modules = llm_int8_skip_modules
38
+ self.llm_int8_skip_modules = llm_int8_skip_modules or []
39
self.llm_int8_threshold = llm_int8_threshold
40
41
def __repr__(self) -> str:
41
- return "BitsAndBytesConfig"
42
+ return (f"BitsAndBytesConfig(load_in_8bit={self.load_in_8bit}, "
43
+ f"load_in_4bit={self.load_in_4bit}, "
44
+ f"bnb_4bit_compute_dtype={self.bnb_4bit_compute_dtype}, "
45
+ f"bnb_4bit_quant_type={self.bnb_4bit_quant_type}, "
46
+ f"llm_int8_skip_modules={self.llm_int8_skip_modules})")
47
48
@classmethod
49
def get_name(self) -> str:
@@ -102,8 +107,10 @@ class BitsAndBytesConfig(QuantizationConfig):
107
llm_int8_threshold=llm_int8_threshold)
108
109
def get_quant_method(self, layer: torch.nn.Module,
105
- prefix: str) -> Optional["BitsAndBytesLinearMethod"]:
110
+ prefix: str) -> Optional["LinearMethodBase"]:
111
if isinstance(layer, LinearBase):
112
+ if is_layer_skipped_bnb(prefix, self.llm_int8_skip_modules):
113
+ return UnquantizedLinearMethod()
114
return BitsAndBytesLinearMethod(self)
115
return None
116
@@ -111,6 +118,10 @@ class BitsAndBytesConfig(QuantizationConfig):
118
return []
119
120
121
+def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
122
+ return any(module_name in prefix for module_name in llm_int8_skip_modules)
123
+
124
+
125
class BitsAndBytesLinearMethod(LinearMethodBase):
126
"""Linear method for BitsAndBytes.
127
@@ -211,6 +222,11 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
222
from bitsandbytes import MatmulLtState, matmul
223
224
original_type = x.dtype
225
+ original_shape = x.shape
226
+ reshape_after_matmul = False
227
+ if x.ndim > 2:
228
+ x = x.reshape(-1, x.size(-1))
229
+ reshape_after_matmul = True
230
bf_x = x.to(torch.bfloat16)
231
232
qweight = layer.qweight
@@ -265,6 +281,9 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
281
282
out = out.to(original_type)
283
284
+ if reshape_after_matmul:
285
+ out = out.view(*original_shape[:-1], out.size(-1))
286
+
287
if bias is not None:
288
out += bias
289
@@ -282,6 +301,11 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
301
from bitsandbytes import matmul_4bit
302
303
original_type = x.dtype
304
+ original_shape = x.shape
305
+ reshape_after_matmul = False
306
+ if x.ndim > 2:
307
+ x = x.reshape(-1, x.size(-1))
308
+ reshape_after_matmul = True
309
bf_x = x.to(torch.bfloat16)
310
311
qweight = layer.qweight
@@ -310,6 +334,9 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
334
335
out = out.to(original_type)
336
337
+ if reshape_after_matmul:
338
+ out = out.view(*original_shape[:-1], out.size(-1))
339
+
340
if bias is not None:
341
out += bias
342
vllm/model_executor/model_loader/loader.py
+16
-3
@@ -899,6 +899,19 @@ class BitsAndBytesModelLoader(BaseModelLoader):
899
return self._unquantized_generator(hf_weights_files, use_safetensors,
900
quant_state_dict), quant_state_dict
901
902
+ def _is_8bit_weight_name(self, weight_name: str):
903
+ quantized_suffix = {".scb", ".weight_format"}
904
+ return any(weight_name.lower().endswith(suffix)
905
+ for suffix in quantized_suffix)
906
+
907
+ def _is_4bit_weight_name(self, weight_name: str):
908
+ quantized_suffix = {
909
+ "absmax", "quant_map", "nested_absmax", "nested_quant_map",
910
+ "bitsandbytes"
911
+ }
912
+ suffix = weight_name.split(".")[-1]
913
+ return any(q_suffix in suffix for q_suffix in quantized_suffix)
914
+
915
def _quantized_8bit_generator(self, hf_weights_files, use_safetensors,
916
quant_state_dict) -> Generator:
917
for weight_name, weight_tensor in self._hf_weight_iter(
@@ -912,7 +925,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
925
for weight_name, weight_tensor in self._hf_weight_iter(
926
hf_weights_files, use_safetensors):
927
915
- if not weight_name.endswith((".weight", ".bias")):
928
+ if self._is_8bit_weight_name(weight_name):
929
continue
930
931
qweight_name = weight_name.replace(".weight", ".qweight")
@@ -932,7 +945,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
945
use_safetensors)
946
temp_state_dict = {}
947
for weight_name, weight_tensor in weight_iterator:
935
- if weight_name.endswith((".weight", ".bias")):
948
+ if not self._is_4bit_weight_name(weight_name):
949
continue
950
# bitsandbytes library requires
951
# weight.quant_state.bitsandbytes__* in CPU
@@ -956,7 +969,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
969
for weight_name, weight_tensor in self._hf_weight_iter(
970
hf_weights_files, use_safetensors):
971
959
- if not weight_name.endswith((".weight", ".bias")):
972
+ if self._is_4bit_weight_name(weight_name):
973
continue
974
975
if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
vllm/model_executor/models/mllama.py
+37
-5
@@ -325,7 +325,10 @@ class MllamaPrecomputedPositionEmbedding(nn.Module):
325
# TODO: support other attention backends for attention in vision model
326
class MllamaVisionSdpaAttention(nn.Module):
327
328
- def __init__(self, config: config_mllama.MllamaVisionConfig):
328
+ def __init__(self,
329
+ config: config_mllama.MllamaVisionConfig,
330
+ quant_config: Optional[QuantizationConfig] = None,
331
+ prefix: str = ""):
332
super().__init__()
333
334
model_parallel_size = get_tensor_model_parallel_world_size()
@@ -341,12 +344,16 @@ class MllamaVisionSdpaAttention(nn.Module):
344
self.head_dim,
345
self.num_heads,
346
bias=False,
347
+ quant_config=quant_config,
348
+ prefix=f"{prefix}.qkv_proj",
349
)
350
self.o_proj = RowParallelLinear(
351
self.num_heads * self.head_dim,
352
self.embed_dim,
353
bias=False,
354
input_is_parallel=True,
355
+ quant_config=quant_config,
356
+ prefix=f"{prefix}.o_proj",
357
)
358
359
def forward(
@@ -393,7 +400,8 @@ class MllamaVisionEncoderLayer(nn.Module):
400
self.is_gated = is_gated
401
self.intermediate_size = config.intermediate_size
402
396
- self.self_attn = MllamaVisionSdpaAttention(config)
403
+ self.self_attn = MllamaVisionSdpaAttention(
404
+ config, quant_config=quant_config, prefix=f"{prefix}.self_attn")
405
self.mlp = CLIPMLP(config,
406
quant_config=quant_config,
407
prefix=f"{prefix}.mlp")
@@ -1002,6 +1010,7 @@ class MllamaForCausalLM(nn.Module):
1010
org_num_embeddings=config.vocab_size,
1011
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
1012
quant_config=quant_config,
1013
+ prefix=f"{prefix}.lm_head",
1014
)
1015
1016
def forward(
@@ -1037,6 +1046,26 @@ class MllamaForCausalLM(nn.Module):
1046
@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
1047
@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
1048
class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1049
+ # BitandBytes specific attributes
1050
+ default_bitsandbytes_target_modules = [
1051
+ ".gate_proj.",
1052
+ ".down_proj.",
1053
+ ".up_proj.",
1054
+ ".q_proj.",
1055
+ ".k_proj.",
1056
+ ".v_proj.",
1057
+ ".o_proj.",
1058
+ ]
1059
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
1060
+ column_parallel_weights_modules = [".down_proj.", ".o_proj."]
1061
+ bitsandbytes_stacked_params_mapping = {
1062
+ # shard_name, weight_name, index
1063
+ "q_proj": ("qkv_proj", 0),
1064
+ "k_proj": ("qkv_proj", 1),
1065
+ "v_proj": ("qkv_proj", 2),
1066
+ "gate_proj": ("gate_up_proj", 0),
1067
+ "up_proj": ("gate_up_proj", 1),
1068
+ }
1069
1070
def __init__(self,
1071
config: config_mllama.MllamaConfig,
@@ -1061,10 +1090,13 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1090
quant_config=quant_config,
1091
prefix="language_model",
1092
)
1064
- self.multi_modal_projector = nn.Linear(
1093
+ self.multi_modal_projector = ColumnParallelLinear(
1094
config.vision_config.vision_output_dim,
1095
config.text_config.hidden_size,
1096
bias=True,
1097
+ quant_config=quant_config,
1098
+ gather_output=True,
1099
+ prefix="multi_modal_projector",
1100
)
1101
self.logits_processor = LogitsProcessor(config.output_hidden_states,
1102
config.text_config.vocab_size)
@@ -1128,7 +1160,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1160
raise ValueError("No images provided.")
1161
max_num_tiles = max(
1162
max([len(x) for x in y[0]]) for y in pixel_values)
1131
- device = self.multi_modal_projector.weight.device
1163
+ device = next(self.multi_modal_projector.parameters()).device
1164
bsz = len(pixel_values)
1165
out_num_tiles = []
1166
out_images = torch.zeros(
@@ -1204,7 +1236,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1236
cross_attention_states = self.vision_model(pixel_values,
1237
aspect_ratio_ids,
1238
aspect_ratio_mask)
1207
- cross_attention_states = self.multi_modal_projector(
1239
+ cross_attention_states, _ = self.multi_modal_projector(
1240
cross_attention_states)
1241
1242
bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)