@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f4117c38e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

support bitsandbytes quantization with more models (#9148)

chenqianfzh committed Oct 8, 2024 at 18:52 UTC 2f4117c38e101ee63b65521c93b22efe3526f77e
10 files changed +164 -27
tests/quantization/test_bitsandbytes.py
+7 -6
@@ -9,22 +9,22 @@ import pytest
9 import torch
10
11 from tests.quantization.utils import is_quant_method_supported
12 -
13 -from ..utils import fork_new_process_for_each_test
12 +from tests.utils import fork_new_process_for_each_test
13
14 models_4bit_to_test = [
16 - ('huggyllama/llama-7b', 'quantize model inflight'),
15 + ("facebook/opt-125m", "quantize opt model inflight"),
16 ]
17
18 models_pre_qaunt_4bit_to_test = [
20 - ('lllyasviel/omost-llama-3-8b-4bits',
21 - 'read pre-quantized 4-bit NF4 model'),
19 ('PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed',
20 'read pre-quantized 4-bit FP4 model'),
21 + ('poedator/opt-125m-bnb-4bit', 'read pre-quantized 4-bit NF4 opt model'),
22 ]
23
24 models_pre_quant_8bit_to_test = [
27 - ('meta-llama/Llama-Guard-3-8B-INT8', 'read pre-quantized 8-bit model'),
25 + ('meta-llama/Llama-Guard-3-8B-INT8',
26 + 'read pre-quantized llama 8-bit model'),
27 + ("yec019/fbopt-350m-8bit", "read pre-quantized 8-bit opt model"),
28 ]
29
30
@@ -133,6 +133,7 @@ def validate_generated_texts(hf_runner,
133 hf_str = hf_log["generated_text"]
134 vllm_str = vllm_log["generated_text"]
135 prompt = hf_log["prompt"]
136 +
137 assert hf_str == vllm_str, (f"Model: {model_name}"
138 f"Mismatch between HF and vLLM outputs:\n"
139 f"Prompt: {prompt}\n"
vllm/model_executor/layers/linear.py
+25 -1
@@ -336,8 +336,12 @@ class ColumnParallelLinear(LinearBase):
336 if is_gguf_weight and isinstance(param, UninitializedParameter):
337 param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
338
339 + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
340 +
341 param_data = param.data
340 - if output_dim is not None:
342 + # bitsandbytes loads the weights of the specific portion
343 + # no need to narrow here
344 + if output_dim is not None and not use_bitsandbytes_4bit:
345 shard_size = param_data.shape[output_dim]
346 start_idx = tp_rank * shard_size
347 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
@@ -821,6 +825,9 @@ class QKVParallelLinear(ColumnParallelLinear):
825 ("v", (self.total_num_heads + self.total_num_kv_heads) *
826 self.head_size, self.total_num_kv_heads * self.head_size),
827 ]
828 + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
829 + False)
830 +
831 packed_dim = getattr(param, "packed_dim", None)
832 for shard_id, shard_offset, shard_size in shard_offsets:
833 # Special case for Quantized Weights.
@@ -834,6 +841,23 @@ class QKVParallelLinear(ColumnParallelLinear):
841 shard_size, shard_offset = adjust_marlin_shard(
842 param, shard_size, shard_offset)
843
844 + if use_bitsandbytes_4bit:
845 + orig_qkv_offsets = {
846 + "q": (0, self.total_num_heads * self.head_size),
847 + "k": (self.total_num_heads * self.head_size,
848 + self.total_num_kv_heads * self.head_size),
849 + "v":
850 + ((self.total_num_heads + self.total_num_kv_heads) *
851 + self.head_size,
852 + self.total_num_kv_heads * self.head_size),
853 + "total":
854 + ((self.total_num_heads + 2 * self.total_num_kv_heads) *
855 + self.head_size, 0)
856 + }
857 +
858 + shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(
859 + param, orig_qkv_offsets, shard_id)
860 +
861 loaded_weight_shard = loaded_weight.narrow(
862 output_dim, shard_offset, shard_size)
863 self.weight_loader(param, loaded_weight_shard, shard_id)
vllm/model_executor/layers/quantization/bitsandbytes.py
+2 -2
@@ -108,7 +108,7 @@ class BitsAndBytesConfig(QuantizationConfig):
108 return None
109
110 def get_scaled_act_names(self) -> List[str]:
111 - return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
111 + return []
112
113
114 class BitsAndBytesLinearMethod(LinearMethodBase):
@@ -236,7 +236,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
236 if generation == 0 or generation == 1:
237 matmul_states[i] = MatmulLtState()
238 matmul_states[i].CB = qweight[offsets[i]:offsets[i + 1]]
239 - matmul_states[i].SCB = quant_states[i]
239 + matmul_states[i].SCB = quant_states[i].to(x.device)
240 matmul_states[i].threshold = (
241 self.quant_config.llm_int8_threshold)
242 matmul_states[i].has_fp16_weights = (
vllm/model_executor/model_loader/loader.py
+44 -18
@@ -736,15 +736,26 @@ class ShardedStateLoader(BaseModelLoader):
736 class BitsAndBytesModelLoader(BaseModelLoader):
737 """Model loader to load model weights with BitAndBytes quantization."""
738
739 - # TODO: these module names are for Llama only,
740 - # change so that it works with other models as well
739 + possible_config_file_names = ["adapter_config.json"]
740 +
741 default_target_modules = [
742 - "gate_proj", "down_proj", "up_proj", "q_proj", "k_proj", "v_proj",
743 - "o_proj"
742 + ".gate_proj.",
743 + ".down_proj.",
744 + ".up_proj.",
745 + ".q_proj.",
746 + ".k_proj.",
747 + ".v_proj.",
748 + ".o_proj.",
749 + '.fc1.',
750 + '.fc2.',
751 + '.dense.',
752 + '.query_key_value.',
753 + '.qkv_proj.',
754 + '.dense_h_to_4h.',
755 + '.dense_4h_to_h.',
756 + '.out_proj.',
757 ]
758
746 - possible_config_file_names = ["adapter_config.json"]
747 -
759 def __init__(self, load_config: LoadConfig):
760 super().__init__(load_config)
761
@@ -754,7 +765,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
765 if (not load_config.model_loader_extra_config
766 or "qlora_adapter_name_or_path"
767 not in load_config.model_loader_extra_config):
757 - self.target_modules = self.default_target_modules
768 + self.target_modules = []
769 return
770
771 qlora_adapter = load_config.model_loader_extra_config[
@@ -901,10 +912,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
912 for weight_name, weight_tensor in self._hf_weight_iter(
913 hf_weights_files, use_safetensors):
914
904 - if not weight_name.endswith(".weight"):
915 + if not weight_name.endswith((".weight", ".bias")):
916 continue
917
918 qweight_name = weight_name.replace(".weight", ".qweight")
919 +
920 if qweight_name in quant_state_dict:
921 set_weight_attrs(weight_tensor, {"load_in_8bit": True})
922 yield qweight_name, weight_tensor
@@ -920,7 +932,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
932 use_safetensors)
933 temp_state_dict = {}
934 for weight_name, weight_tensor in weight_iterator:
923 - if weight_name.endswith(".weight"):
935 + if weight_name.endswith((".weight", ".bias")):
936 continue
937 # bitsandbytes library requires
938 # weight.quant_state.bitsandbytes__* in CPU
@@ -943,9 +955,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
955 # pre quantized weights would have a quant_state
956 for weight_name, weight_tensor in self._hf_weight_iter(
957 hf_weights_files, use_safetensors):
946 - # Filter out all weights whose suffix is not ".weight"
947 - if not weight_name.endswith(".weight"):
958 +
959 + if not weight_name.endswith((".weight", ".bias")):
960 continue
961 +
962 if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
963 in temp_state_dict) or \
964 (f"{weight_name}.quant_state.bitsandbytes__fp4" \
@@ -965,15 +978,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
978
979 for weight_name, weight_tensor in self._hf_weight_iter(
980 hf_weights_files, use_safetensors):
968 - if any(target_module in weight_name
969 - for target_module in self.target_modules):
981 +
982 + if any(target_module in weight_name for target_module in
983 + self.target_modules) and weight_name.endswith(".weight"):
984 weight_name = weight_name.replace(".weight", ".qweight")
985
972 - # weight partitions of different modules occur at
973 - # different dimensions
974 - # TODO: these module names are for Llama only,
975 - # change so that it works with other models as well
976 - if 'down_proj' in weight_name or 'o_proj' in weight_name:
986 + if any(module in weight_name
987 + for module in self.column_parallel_weights_modules):
988 +
989 total_size = weight_tensor.size(-1)
990 start_index = total_size // tp_size * tp_rank
991 end_index = total_size // tp_size * (tp_rank + 1)
@@ -1022,6 +1034,20 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1034 f"Model {type(model).__name__} does not support BitsAndBytes "
1035 "quantization yet.")
1036
1037 + if len(self.target_modules) == 0:
1038 + if hasattr(model, 'default_bitsandbytes_target_modules'):
1039 + self.target_modules = model.default_bitsandbytes_target_modules
1040 + else:
1041 + self.target_modules = self.default_target_modules
1042 +
1043 + if hasattr(model, 'column_parallel_weights_modules'):
1044 + self.column_parallel_weights_modules = \
1045 + model.column_parallel_weights_modules
1046 + else:
1047 + self.column_parallel_weights_modules = []
1048 +
1049 + self.model_type = type(model).__name__
1050 +
1051 logger.info("Loading weights with BitsAndBytes quantization. "
1052 " May take a while ...")
1053
vllm/model_executor/models/falcon.py
+11
@@ -391,6 +391,17 @@ class FalconModel(nn.Module):
391
392 class FalconForCausalLM(nn.Module, SupportsPP):
393
394 + # BitandBytes specific attributes
395 + bitsandbytes_stacked_params_mapping = {}
396 + default_bitsandbytes_target_modules = [
397 + ".query_key_value.",
398 + ".dense.",
399 + ".dense_h_to_4h.",
400 + ".dense_4h_to_h.",
401 + ]
402 + # in TP, these weights are partitioned along the column dimension (dim=-1)
403 + column_parallel_weights_modules = [".dense_4h_to_h.", ".dense."]
404 +
405 def __init__(
406 self,
407 config: FalconConfig,
vllm/model_executor/models/gemma.py
+22
@@ -332,6 +332,28 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332 "gate_up_proj",
333 "down_proj",
334 ]
335 +
336 + # BitandBytes specific attributes
337 + default_bitsandbytes_target_modules = [
338 + ".gate_proj.",
339 + ".down_proj.",
340 + ".up_proj.",
341 + ".q_proj.",
342 + ".k_proj.",
343 + ".v_proj.",
344 + ".o_proj.",
345 + ]
346 + # in TP, these weights are partitioned along the column dimension (dim=-1)
347 + column_parallel_weights_modules = [".down_proj.", ".o_proj."]
348 + bitsandbytes_stacked_params_mapping = {
349 + # shard_name, weight_name, index
350 + "q_proj": ("qkv_proj", 0),
351 + "k_proj": ("qkv_proj", 1),
352 + "v_proj": ("qkv_proj", 2),
353 + "gate_proj": ("gate_up_proj", 0),
354 + "up_proj": ("gate_up_proj", 1),
355 + }
356 +
357 # Gemma does not apply LoRA to the embedding layer.
358 embedding_modules = {}
359 embedding_padding_modules = []
vllm/model_executor/models/gemma2.py
+13
@@ -375,6 +375,19 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
375 # Gemma does not apply LoRA to the embedding layer.
376 embedding_modules = {}
377 embedding_padding_modules = []
378 +
379 + # BitandBytes specific attributes
380 + default_bitsandbytes_target_modules = [
381 + ".gate_proj.",
382 + ".down_proj.",
383 + ".up_proj.",
384 + ".q_proj.",
385 + ".k_proj.",
386 + ".v_proj.",
387 + ".o_proj.",
388 + ]
389 + # in TP, these weights are partitioned along the column dimension (dim=-1)
390 + column_parallel_weights_modules = [".down_proj.", ".o_proj."]
391 bitsandbytes_stacked_params_mapping = {
392 # shard_name, weight_name, index
393 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/llama.py
+13
@@ -449,6 +449,19 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
449 "lm_head": "output_embeddings"
450 }
451 embedding_padding_modules = ["lm_head"]
452 +
453 + # BitandBytes specific attributes
454 + default_bitsandbytes_target_modules = [
455 + ".gate_proj.",
456 + ".down_proj.",
457 + ".up_proj.",
458 + ".q_proj.",
459 + ".k_proj.",
460 + ".v_proj.",
461 + ".o_proj.",
462 + ]
463 + # in TP, these weights are partitioned along the column dimension (dim=-1)
464 + column_parallel_weights_modules = [".down_proj.", ".o_proj."]
465 bitsandbytes_stacked_params_mapping = {
466 # shard_name, weight_name, index
467 "q_proj": ("qkv_proj", 0),
vllm/model_executor/models/opt.py
+13
@@ -315,6 +315,19 @@ class OPTModel(nn.Module):
315
316 class OPTForCausalLM(nn.Module, SupportsPP):
317
318 + # BitandBytes specific attributes
319 + bitsandbytes_stacked_params_mapping = {
320 + # shard_name, weight_name, index
321 + "q_proj": ("qkv_proj", 0),
322 + "k_proj": ("qkv_proj", 1),
323 + "v_proj": ("qkv_proj", 2),
324 + }
325 + default_bitsandbytes_target_modules = [
326 + ".q_proj.", ".k_proj.", ".v_proj.", ".out_proj.", ".fc1.", ".fc2."
327 + ]
328 + # in TP, these weights are partitioned along the column dimension (dim=-1)
329 + column_parallel_weights_modules = [".out_proj.", ".fc2."]
330 +
331 def __init__(
332 self,
333 config: OPTConfig,
vllm/model_executor/models/phi.py
+14
@@ -260,6 +260,20 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
260 "fc1",
261 "fc2",
262 ]
263 +
264 + # BitandBytes specific attributes
265 + bitsandbytes_stacked_params_mapping = {
266 + # shard_name, weight_name, index
267 + "q_proj": ("qkv_proj", 0),
268 + "k_proj": ("qkv_proj", 1),
269 + "v_proj": ("qkv_proj", 2),
270 + }
271 + default_bitsandbytes_target_modules = [
272 + ".q_proj.", ".k_proj.", ".v_proj.", ".fc1.", ".fc2.", ".dense."
273 + ]
274 + # in TP, these weights are partitioned along the column dimension (dim=-1)
275 + column_parallel_weights_modules = [".fc2.", ".dense."]
276 +
277 embedding_modules = {}
278 embedding_padding_modules = []
279